From 9e99a55b317f5da66f5110891b154084b337a031 Mon Sep 17 00:00:00 2001
From: Alexander Eichner <github@aeichner.de>
Date: Fri, 18 Aug 2023 13:32:15 +0200
Subject: [PATCH 001/237] [virtio] Fix implementation of vpm_ioread32()

The current implementation of vpm_ioread32() erroneously reads only 16
bits of data, which fails when used with the (stricter) virtio device
emulation in VirtualBox.

Fix by using the correct readl()/inl() I/O wrappers.

Reworded-by: Michael Brown <mcb30@ipxe.org>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/bus/virtio-pci.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/drivers/bus/virtio-pci.c b/src/drivers/bus/virtio-pci.c
index 8b34c7277..3fc93a906 100644
--- a/src/drivers/bus/virtio-pci.c
+++ b/src/drivers/bus/virtio-pci.c
@@ -230,10 +230,10 @@ u32 vpm_ioread32(struct virtio_pci_modern_device *vdev,
     uint32_t data;
     switch (region->flags & VIRTIO_PCI_REGION_TYPE_MASK) {
     case VIRTIO_PCI_REGION_MEMORY:
-        data = readw(region->base + offset);
+        data = readl(region->base + offset);
         break;
     case VIRTIO_PCI_REGION_PORT:
-        data = inw(region->base + offset);
+        data = inl(region->base + offset);
         break;
     case VIRTIO_PCI_REGION_PCI_CONFIG:
         prep_pci_cfg_cap(vdev, region, offset, 4);

From 0aa2e4ec963597794dd8f8b36f77f4d0cf4e03c8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 5 Sep 2023 12:46:39 +0100
Subject: [PATCH 002/237] [librm] Use explicit operand size when pushing a
 label address

We currently use "push $1f" within inline assembly to push the address
of the real-mode code fragment, relying on the assembler to treat this
as "pushl" for 32-bit code or "pushq" for 64-bit code.

As of binutils commit 5cc0077 ("x86: further adjust extend-to-32bit-
address conditions"), first included in binutils-2.41, this implicit
operand size is no longer calculated as expected and 64-bit builds
will fail with

  Error: operand size mismatch for `push'

Fix by adding an explicit operand size to the "push" instruction.

Originally-fixed-by: Justin Cano <jstncno@gmail.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/include/librm.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/arch/x86/include/librm.h b/src/arch/x86/include/librm.h
index 5196d390f..40f075439 100644
--- a/src/arch/x86/include/librm.h
+++ b/src/arch/x86/include/librm.h
@@ -250,8 +250,10 @@ extern void remove_user_from_rm_stack ( userptr_t data, size_t size );
 /* CODE_DEFAULT: restore default .code32/.code64 directive */
 #ifdef __x86_64__
 #define CODE_DEFAULT ".code64"
+#define STACK_DEFAULT "q"
 #else
 #define CODE_DEFAULT ".code32"
+#define STACK_DEFAULT "l"
 #endif
 
 /* LINE_SYMBOL: declare a symbol for the current source code line */
@@ -268,7 +270,7 @@ extern void remove_user_from_rm_stack ( userptr_t data, size_t size );
 
 /* REAL_CODE: declare a fragment of code that executes in real mode */
 #define REAL_CODE( asm_code_str )			\
-	"push $1f\n\t"					\
+	"push" STACK_DEFAULT " $1f\n\t"			\
 	"call real_call\n\t"				\
 	TEXT16_CODE ( "\n1:\n\t"			\
 		      asm_code_str			\
@@ -277,7 +279,7 @@ extern void remove_user_from_rm_stack ( userptr_t data, size_t size );
 
 /* PHYS_CODE: declare a fragment of code that executes in flat physical mode */
 #define PHYS_CODE( asm_code_str )			\
-	"push $1f\n\t"					\
+	"push" STACK_DEFAULT " $1f\n\t"			\
 	"call phys_call\n\t"				\
 	".section \".text.phys\", \"ax\", @progbits\n\t"\
 	"\n" LINE_SYMBOL "\n\t"				\

From eeb7cd56e54e2bc649626988872c170fba37c163 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 13 Sep 2023 14:30:25 +0100
Subject: [PATCH 003/237] [netdevice] Remove netdev_priv() helper function

Some network device drivers use the trivial netdev_priv() helper
function while others use the netdev->priv pointer directly.

Standardise on direct use of netdev->priv, in order to free up the
function name netdev_priv() for reuse.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/3c90x.c            | 20 +++++++-------
 src/drivers/net/atl1e.c            | 18 ++++++-------
 src/drivers/net/b44.c              | 16 ++++++------
 src/drivers/net/bnxt/bnxt.c        | 18 ++++++-------
 src/drivers/net/eepro100.c         |  2 +-
 src/drivers/net/efi/snpnet.c       |  6 ++---
 src/drivers/net/etherfabric.c      | 14 +++++-----
 src/drivers/net/forcedeth.c        | 18 ++++++-------
 src/drivers/net/igbvf/igbvf_main.c | 18 ++++++-------
 src/drivers/net/jme.c              |  2 +-
 src/drivers/net/pcnet32.c          | 16 ++++++------
 src/drivers/net/phantom/phantom.c  | 18 ++++++-------
 src/drivers/net/sfc/efx_common.c   |  4 +--
 src/drivers/net/sfc/efx_hunt.c     | 16 ++++++------
 src/drivers/net/sfc/sfc_hunt.c     | 10 +++----
 src/drivers/net/sis190.c           | 38 +++++++++++++--------------
 src/drivers/net/skge.c             | 42 +++++++++++++++---------------
 src/drivers/net/sky2.c             | 30 ++++++++++-----------
 src/drivers/net/tg3/tg3.c          | 16 ++++++------
 src/drivers/net/tg3/tg3_hw.c       |  2 +-
 src/drivers/net/vmxnet3.c          | 26 +++++++++---------
 src/drivers/net/vxge/vxge_main.c   | 14 +++++-----
 src/include/ipxe/netdevice.h       | 11 --------
 23 files changed, 182 insertions(+), 193 deletions(-)

diff --git a/src/drivers/net/3c90x.c b/src/drivers/net/3c90x.c
index 63e07777f..1b8190c46 100644
--- a/src/drivers/net/3c90x.c
+++ b/src/drivers/net/3c90x.c
@@ -272,7 +272,7 @@ static int a3c90x_setup_tx_ring(struct INF_3C90X *p)
  */
 static void a3c90x_process_tx_packets(struct net_device *netdev)
 {
-	struct INF_3C90X *p = netdev_priv(netdev);
+	struct INF_3C90X *p = netdev->priv;
 	unsigned int downlist_ptr;
 
 	DBGP("a3c90x_process_tx_packets\n");
@@ -320,7 +320,7 @@ static void a3c90x_free_tx_ring(struct INF_3C90X *p)
 static int a3c90x_transmit(struct net_device *netdev,
 			   struct io_buffer *iob)
 {
-	struct INF_3C90X *inf_3c90x = netdev_priv(netdev);
+	struct INF_3C90X *inf_3c90x = netdev->priv;
 	struct TXD *tx_cur_desc;
 	struct TXD *tx_prev_desc;
 
@@ -518,7 +518,7 @@ static void a3c90x_process_rx_packets(struct net_device *netdev)
 {
 	int i;
 	unsigned int rx_status;
-	struct INF_3C90X *p = netdev_priv(netdev);
+	struct INF_3C90X *p = netdev->priv;
 	struct RXD *rx_cur_desc;
 
 	DBGP("a3c90x_process_rx_packets\n");
@@ -567,7 +567,7 @@ static void a3c90x_process_rx_packets(struct net_device *netdev)
  */
 static void a3c90x_poll(struct net_device *netdev)
 {
-	struct INF_3C90X *p = netdev_priv(netdev);
+	struct INF_3C90X *p = netdev->priv;
 	uint16_t raw_status, int_status;
 
 	DBGP("a3c90x_poll\n");
@@ -611,7 +611,7 @@ static void a3c90x_free_resources(struct INF_3C90X *p)
 static void a3c90x_remove(struct pci_device *pci)
 {
 	struct net_device *netdev = pci_get_drvdata(pci);
-	struct INF_3C90X *inf_3c90x = netdev_priv(netdev);
+	struct INF_3C90X *inf_3c90x = netdev->priv;
 
 	DBGP("a3c90x_remove\n");
 
@@ -628,7 +628,7 @@ static void a3c90x_remove(struct pci_device *pci)
 
 static void a3c90x_irq(struct net_device *netdev, int enable)
 {
-	struct INF_3C90X *p = netdev_priv(netdev);
+	struct INF_3C90X *p = netdev->priv;
 
 	DBGP("a3c90x_irq\n");
 
@@ -657,7 +657,7 @@ static void a3c90x_hw_start(struct net_device *netdev)
 	unsigned int cfg;
 	unsigned int mopt;
 	unsigned short linktype;
-	struct INF_3C90X *inf_3c90x = netdev_priv(netdev);
+	struct INF_3C90X *inf_3c90x = netdev->priv;
 
 	DBGP("a3c90x_hw_start\n");
 
@@ -796,7 +796,7 @@ static void a3c90x_hw_start(struct net_device *netdev)
 static int a3c90x_open(struct net_device *netdev)
 {
 	int rc;
-	struct INF_3C90X *inf_3c90x = netdev_priv(netdev);
+	struct INF_3C90X *inf_3c90x = netdev->priv;
 
 	DBGP("a3c90x_open\n");
 
@@ -845,7 +845,7 @@ static int a3c90x_open(struct net_device *netdev)
  */
 static void a3c90x_close(struct net_device *netdev)
 {
-	struct INF_3C90X *inf_3c90x = netdev_priv(netdev);
+	struct INF_3C90X *inf_3c90x = netdev->priv;
 
 	DBGP("a3c90x_close\n");
 
@@ -895,7 +895,7 @@ static int a3c90x_probe(struct pci_device *pci)
 	pci_set_drvdata(pci, netdev);
 	netdev->dev = &pci->dev;
 
-	inf_3c90x = netdev_priv(netdev);
+	inf_3c90x = netdev->priv;
 	memset(inf_3c90x, 0, sizeof(*inf_3c90x));
 
 	adjust_pci_device(pci);
diff --git a/src/drivers/net/atl1e.c b/src/drivers/net/atl1e.c
index 0f0df5326..1acbb3ca8 100644
--- a/src/drivers/net/atl1e.c
+++ b/src/drivers/net/atl1e.c
@@ -173,7 +173,7 @@ static int atl1e_check_link(struct atl1e_adapter *adapter)
 static int atl1e_mdio_read(struct net_device *netdev, int phy_id __unused,
 			   int reg_num)
 {
-	struct atl1e_adapter *adapter = netdev_priv(netdev);
+	struct atl1e_adapter *adapter = netdev->priv;
 	u16 result;
 
 	atl1e_read_phy_reg(&adapter->hw, reg_num & MDIO_REG_ADDR_MASK, &result);
@@ -183,7 +183,7 @@ static int atl1e_mdio_read(struct net_device *netdev, int phy_id __unused,
 static void atl1e_mdio_write(struct net_device *netdev, int phy_id __unused,
 			     int reg_num, int val)
 {
-	struct atl1e_adapter *adapter = netdev_priv(netdev);
+	struct atl1e_adapter *adapter = netdev->priv;
 
 	atl1e_write_phy_reg(&adapter->hw, reg_num & MDIO_REG_ADDR_MASK, val);
 }
@@ -841,7 +841,7 @@ fatal_err:
  */
 static void atl1e_poll(struct net_device *netdev)
 {
-	struct atl1e_adapter *adapter = netdev_priv(netdev);
+	struct atl1e_adapter *adapter = netdev->priv;
 	struct atl1e_hw *hw = &adapter->hw;
 	int max_ints = 64;
 	u32 status;
@@ -963,7 +963,7 @@ static void atl1e_tx_queue(struct atl1e_adapter *adapter, u16 count __unused,
 
 static int atl1e_xmit_frame(struct net_device *netdev, struct io_buffer *iob)
 {
-	struct atl1e_adapter *adapter = netdev_priv(netdev);
+	struct atl1e_adapter *adapter = netdev->priv;
 	u16 tpd_req = 1;
 	struct atl1e_tpd_desc *tpd;
 
@@ -1013,7 +1013,7 @@ int atl1e_up(struct atl1e_adapter *adapter)
 
 void atl1e_irq(struct net_device *netdev, int enable)
 {
-	struct atl1e_adapter *adapter = netdev_priv(netdev);
+	struct atl1e_adapter *adapter = netdev->priv;
 
 	if (enable)
 		atl1e_irq_enable(adapter);
@@ -1051,7 +1051,7 @@ void atl1e_down(struct atl1e_adapter *adapter)
  */
 static int atl1e_open(struct net_device *netdev)
 {
-	struct atl1e_adapter *adapter = netdev_priv(netdev);
+	struct atl1e_adapter *adapter = netdev->priv;
 	int err;
 
 	/* allocate rx/tx dma buffer & descriptors */
@@ -1086,7 +1086,7 @@ err_up:
  */
 static void atl1e_close(struct net_device *netdev)
 {
-	struct atl1e_adapter *adapter = netdev_priv(netdev);
+	struct atl1e_adapter *adapter = netdev->priv;
 
 	atl1e_down(adapter);
 	atl1e_free_ring_resources(adapter);
@@ -1138,7 +1138,7 @@ static int atl1e_probe(struct pci_device *pdev)
 
 	atl1e_init_netdev(netdev, pdev);
 
-	adapter = netdev_priv(netdev);
+	adapter = netdev->priv;
 	adapter->bd_number = cards_found;
 	adapter->netdev = netdev;
 	adapter->pdev = pdev;
@@ -1227,7 +1227,7 @@ err:
 static void atl1e_remove(struct pci_device *pdev)
 {
 	struct net_device *netdev = pci_get_drvdata(pdev);
-	struct atl1e_adapter *adapter = netdev_priv(netdev);
+	struct atl1e_adapter *adapter = netdev->priv;
 
 	unregister_netdev(netdev);
 	atl1e_free_ring_resources(adapter);
diff --git a/src/drivers/net/b44.c b/src/drivers/net/b44.c
index eaf6d35ce..1ca7e2e53 100644
--- a/src/drivers/net/b44.c
+++ b/src/drivers/net/b44.c
@@ -622,7 +622,7 @@ static void b44_load_mac_and_phy_addr(struct b44_private *bp)
 
 static void b44_set_rx_mode(struct net_device *netdev)
 {
-	struct b44_private *bp = netdev_priv(netdev);
+	struct b44_private *bp = netdev->priv;
 	unsigned char zero[6] = { 0, 0, 0, 0, 0, 0 };
 	u32 val;
 	int i;
@@ -667,7 +667,7 @@ static int b44_probe(struct pci_device *pci)
 	netdev->dev = &pci->dev;
 
 	/* Set up private data */
-	bp = netdev_priv(netdev);
+	bp = netdev->priv;
 	memset(bp, 0, sizeof(*bp));
 	bp->netdev = netdev;
 	bp->pci = pci;
@@ -712,7 +712,7 @@ static int b44_probe(struct pci_device *pci)
 static void b44_remove(struct pci_device *pci)
 {
 	struct net_device *netdev = pci_get_drvdata(pci);
-	struct b44_private *bp = netdev_priv(netdev);
+	struct b44_private *bp = netdev->priv;
 
 	ssb_core_disable(bp);
 	unregister_netdev(netdev);
@@ -729,7 +729,7 @@ static void b44_remove(struct pci_device *pci)
  */
 static void b44_irq(struct net_device *netdev, int enable)
 {
-	struct b44_private *bp = netdev_priv(netdev);
+	struct b44_private *bp = netdev->priv;
 
 	/* Interrupt mask specifies which events generate interrupts */
 	bw32(bp, B44_IMASK, enable ? IMASK_DEF : IMASK_DISABLE);
@@ -743,7 +743,7 @@ static void b44_irq(struct net_device *netdev, int enable)
  */
 static int b44_open(struct net_device *netdev)
 {
-	struct b44_private *bp = netdev_priv(netdev);
+	struct b44_private *bp = netdev->priv;
 	int rc;
 
 	rc = b44_init_tx_ring(bp);
@@ -769,7 +769,7 @@ static int b44_open(struct net_device *netdev)
  */
 static void b44_close(struct net_device *netdev)
 {
-	struct b44_private *bp = netdev_priv(netdev);
+	struct b44_private *bp = netdev->priv;
 
 	b44_chip_reset(bp, B44_FULL_RESET);
 	b44_free_tx_ring(bp);
@@ -785,7 +785,7 @@ static void b44_close(struct net_device *netdev)
  */
 static int b44_transmit(struct net_device *netdev, struct io_buffer *iobuf)
 {
-	struct b44_private *bp = netdev_priv(netdev);
+	struct b44_private *bp = netdev->priv;
 	u32 cur = bp->tx_cur;
 	u32 ctrl;
 
@@ -905,7 +905,7 @@ static void b44_process_rx_packets(struct b44_private *bp)
  */
 static void b44_poll(struct net_device *netdev)
 {
-	struct b44_private *bp = netdev_priv(netdev);
+	struct b44_private *bp = netdev->priv;
 	u32 istat;
 
 	/* Interrupt status */
diff --git a/src/drivers/net/bnxt/bnxt.c b/src/drivers/net/bnxt/bnxt.c
index e3876503f..605aea328 100644
--- a/src/drivers/net/bnxt/bnxt.c
+++ b/src/drivers/net/bnxt/bnxt.c
@@ -307,7 +307,7 @@ void bnxt_set_txq ( struct bnxt *bp, int entry, dma_addr_t mapping, int len )
 
 static void bnxt_tx_complete ( struct net_device *dev, u16 hw_idx )
 {
-	struct bnxt *bp = netdev_priv ( dev );
+	struct bnxt *bp = dev->priv;
 	struct io_buffer *iob;
 
 	iob = bp->tx.iob[hw_idx];
@@ -484,7 +484,7 @@ void bnxt_rx_process ( struct net_device *dev, struct bnxt *bp,
 static int bnxt_rx_complete ( struct net_device *dev,
 		struct rx_pkt_cmpl *rx_cmp )
 {
-	struct bnxt *bp = netdev_priv ( dev );
+	struct bnxt *bp = dev->priv;
 	struct rx_pkt_cmpl_hi *rx_cmp_hi;
 	u8  cmpl_bit = bp->cq.completion_bit;
 
@@ -1927,7 +1927,7 @@ int bnxt_hwrm_run ( hwrm_func_t cmds[], struct bnxt *bp )
 
 static int bnxt_open ( struct net_device *dev )
 {
-	struct bnxt *bp = netdev_priv ( dev );
+	struct bnxt *bp = dev->priv;
 
 	DBGP ( "%s\n", __func__ );
 	bnxt_mm_nic ( bp );
@@ -1952,7 +1952,7 @@ static void bnxt_tx_adjust_pkt ( struct bnxt *bp, struct io_buffer *iob )
 
 static int bnxt_tx ( struct net_device *dev, struct io_buffer *iob )
 {
-	struct bnxt *bp = netdev_priv ( dev );
+	struct bnxt *bp = dev->priv;
 	u16 len, entry;
 	dma_addr_t mapping;
 
@@ -2009,7 +2009,7 @@ void bnxt_link_evt ( struct bnxt *bp, struct hwrm_async_event_cmpl *evt )
 
 static void bnxt_service_cq ( struct net_device *dev )
 {
-	struct bnxt *bp = netdev_priv ( dev );
+	struct bnxt *bp = dev->priv;
 	struct cmpl_base *cmp;
 	struct tx_cmpl *tx;
 	u16 old_cid = bp->cq.cons_id;
@@ -2057,7 +2057,7 @@ static void bnxt_service_cq ( struct net_device *dev )
 
 static void bnxt_service_nq ( struct net_device *dev )
 {
-	struct bnxt *bp = netdev_priv ( dev );
+	struct bnxt *bp = dev->priv;
 	struct nq_base *nqp;
 	u16 old_cid = bp->nq.cons_id;
 	int done = SERVICE_NEXT_NQ_BD;
@@ -2102,7 +2102,7 @@ static void bnxt_poll ( struct net_device *dev )
 
 static void bnxt_close ( struct net_device *dev )
 {
-	struct bnxt *bp = netdev_priv ( dev );
+	struct bnxt *bp = dev->priv;
 
 	DBGP ( "%s\n", __func__ );
 	bnxt_down_nic (bp);
@@ -2143,7 +2143,7 @@ static int bnxt_init_one ( struct pci_device *pci )
 	netdev_init ( netdev, &bnxt_netdev_ops );
 
 	/* Driver private area for this device */
-	bp = netdev_priv ( netdev );
+	bp = netdev->priv;
 
 	/* Set PCI driver private data */
 	pci_set_drvdata ( pci, netdev );
@@ -2197,7 +2197,7 @@ disable_pdev:
 static void bnxt_remove_one ( struct pci_device *pci )
 {
 	struct net_device *netdev = pci_get_drvdata ( pci );
-	struct bnxt *bp = netdev_priv ( netdev );
+	struct bnxt *bp = netdev->priv;
 
 	DBGP ( "%s\n", __func__ );
 	/* Unregister network device */
diff --git a/src/drivers/net/eepro100.c b/src/drivers/net/eepro100.c
index 1a802b590..a0551a89b 100644
--- a/src/drivers/net/eepro100.c
+++ b/src/drivers/net/eepro100.c
@@ -690,7 +690,7 @@ static void ifec_reset ( struct net_device *netdev )
  */
 static void ifec_free ( struct net_device *netdev )
 {
-	struct ifec_private *priv = netdev_priv ( netdev );
+	struct ifec_private *priv = netdev->priv;
 	int i;
 
 	DBGP ( "ifec_free\n" );
diff --git a/src/drivers/net/efi/snpnet.c b/src/drivers/net/efi/snpnet.c
index 69ec6f5e5..3b09d4918 100644
--- a/src/drivers/net/efi/snpnet.c
+++ b/src/drivers/net/efi/snpnet.c
@@ -97,7 +97,7 @@ static const char * snpnet_mac_text ( EFI_MAC_ADDRESS *mac, size_t len ) {
  * @v netdev		Network device
  */
 static void snpnet_dump_mode ( struct net_device *netdev ) {
-	struct snp_nic *snp = netdev_priv ( netdev );
+	struct snp_nic *snp = netdev->priv;
 	EFI_SIMPLE_NETWORK_MODE *mode = snp->snp->Mode;
 	size_t mac_len = mode->HwAddressSize;
 	unsigned int i;
@@ -136,7 +136,7 @@ static void snpnet_dump_mode ( struct net_device *netdev ) {
  * @v netdev		Network device
  */
 static void snpnet_check_link ( struct net_device *netdev ) {
-	struct snp_nic *snp = netdev_priv ( netdev );
+	struct snp_nic *snp = netdev->priv;
 	EFI_SIMPLE_NETWORK_MODE *mode = snp->snp->Mode;
 
 	/* Do nothing unless media presence detection is supported */
@@ -160,7 +160,7 @@ static void snpnet_check_link ( struct net_device *netdev ) {
  */
 static int snpnet_transmit ( struct net_device *netdev,
 			     struct io_buffer *iobuf ) {
-	struct snp_nic *snp = netdev_priv ( netdev );
+	struct snp_nic *snp = netdev->priv;
 	EFI_STATUS efirc;
 	int rc;
 
diff --git a/src/drivers/net/etherfabric.c b/src/drivers/net/etherfabric.c
index e43d4336e..b40596bea 100644
--- a/src/drivers/net/etherfabric.c
+++ b/src/drivers/net/etherfabric.c
@@ -3725,7 +3725,7 @@ efab_receive ( struct efab_nic *efab, unsigned int id, int len, int drop )
 static int
 efab_transmit ( struct net_device *netdev, struct io_buffer *iob )
 {
-	struct efab_nic *efab = netdev_priv ( netdev );
+	struct efab_nic *efab = netdev->priv;
 	struct efab_tx_queue *tx_queue = &efab->tx_queue;
 	int fill_level, space;
 	falcon_tx_desc_t *txd;
@@ -3844,7 +3844,7 @@ falcon_handle_event ( struct efab_nic *efab, falcon_event_t *evt )
 static void
 efab_poll ( struct net_device *netdev )
 {
-	struct efab_nic *efab = netdev_priv ( netdev );
+	struct efab_nic *efab = netdev->priv;
 	struct efab_ev_queue *ev_queue = &efab->ev_queue;
 	struct efab_rx_queue *rx_queue = &efab->rx_queue;
 	falcon_event_t *evt;
@@ -3883,7 +3883,7 @@ efab_poll ( struct net_device *netdev )
 static void
 efab_irq ( struct net_device *netdev, int enable )
 {
-	struct efab_nic *efab = netdev_priv ( netdev );
+	struct efab_nic *efab = netdev->priv;
 	struct efab_ev_queue *ev_queue = &efab->ev_queue;
 
 	switch ( enable ) {
@@ -4032,7 +4032,7 @@ efab_init_mac ( struct efab_nic *efab )
 static void
 efab_close ( struct net_device *netdev )
 {
-	struct efab_nic *efab = netdev_priv ( netdev );
+	struct efab_nic *efab = netdev->priv;
 
 	falcon_fini_resources ( efab );
 	efab_free_resources ( efab );
@@ -4043,7 +4043,7 @@ efab_close ( struct net_device *netdev )
 static int
 efab_open ( struct net_device *netdev )
 {
-	struct efab_nic *efab = netdev_priv ( netdev );
+	struct efab_nic *efab = netdev->priv;
 	struct efab_rx_queue *rx_queue = &efab->rx_queue;
 	int rc;
 
@@ -4104,7 +4104,7 @@ static void
 efab_remove ( struct pci_device *pci )
 {
 	struct net_device *netdev = pci_get_drvdata ( pci );
-	struct efab_nic *efab = netdev_priv ( netdev );
+	struct efab_nic *efab = netdev->priv;
 
 	if ( efab->membase ) {
 		falcon_reset ( efab );
@@ -4143,7 +4143,7 @@ efab_probe ( struct pci_device *pci )
 	pci_set_drvdata ( pci, netdev );
 	netdev->dev = &pci->dev;
 
-	efab = netdev_priv ( netdev );
+	efab = netdev->priv;
 	memset ( efab, 0, sizeof ( *efab ) );
 	efab->netdev = netdev;
 
diff --git a/src/drivers/net/forcedeth.c b/src/drivers/net/forcedeth.c
index 7fba08a08..ec3a5bdb2 100644
--- a/src/drivers/net/forcedeth.c
+++ b/src/drivers/net/forcedeth.c
@@ -677,7 +677,7 @@ set_speed:
 static int
 forcedeth_open ( struct net_device *netdev )
 {
-	struct forcedeth_private *priv = netdev_priv ( netdev );
+	struct forcedeth_private *priv = netdev->priv;
 	void *ioaddr = priv->mmio_addr;
 	int i;
 	int rc;
@@ -794,7 +794,7 @@ err_init_rings:
 static int
 forcedeth_transmit ( struct net_device *netdev, struct io_buffer *iobuf )
 {
-	struct forcedeth_private *priv = netdev_priv ( netdev );
+	struct forcedeth_private *priv = netdev->priv;
 	void *ioaddr = priv->mmio_addr;
 	struct ring_desc *tx_curr_desc;
 	u32 size = iob_len ( iobuf );
@@ -853,7 +853,7 @@ forcedeth_transmit ( struct net_device *netdev, struct io_buffer *iobuf )
 static void
 nv_process_tx_packets ( struct net_device *netdev )
 {
-	struct forcedeth_private *priv = netdev_priv ( netdev );
+	struct forcedeth_private *priv = netdev->priv;
 	struct ring_desc *tx_curr_desc;
 	u32 flaglen;
 
@@ -899,7 +899,7 @@ nv_process_tx_packets ( struct net_device *netdev )
 static void
 nv_process_rx_packets ( struct net_device *netdev )
 {
-	struct forcedeth_private *priv = netdev_priv ( netdev );
+	struct forcedeth_private *priv = netdev->priv;
 	struct io_buffer *curr_iob;
 	struct ring_desc *rx_curr_desc;
 	u32 flags, len;
@@ -960,7 +960,7 @@ nv_process_rx_packets ( struct net_device *netdev )
 static void
 forcedeth_link_status ( struct net_device *netdev )
 {
-	struct forcedeth_private *priv = netdev_priv ( netdev );
+	struct forcedeth_private *priv = netdev->priv;
 	void *ioaddr = priv->mmio_addr;
 
 	/* Clear the MII link change status by reading the MIIStatus register */
@@ -981,7 +981,7 @@ forcedeth_link_status ( struct net_device *netdev )
 static void
 forcedeth_poll ( struct net_device *netdev )
 {
-	struct forcedeth_private *priv = netdev_priv ( netdev );
+	struct forcedeth_private *priv = netdev->priv;
 	void *ioaddr = priv->mmio_addr;
 	u32 status;
 
@@ -1018,7 +1018,7 @@ forcedeth_poll ( struct net_device *netdev )
 static void
 forcedeth_close ( struct net_device *netdev )
 {
-	struct forcedeth_private *priv = netdev_priv ( netdev );
+	struct forcedeth_private *priv = netdev->priv;
 
 	DBGP ( "forcedeth_close\n" );
 
@@ -1045,7 +1045,7 @@ forcedeth_close ( struct net_device *netdev )
 static void
 forcedeth_irq ( struct net_device *netdev, int action )
 {
-	struct forcedeth_private *priv = netdev_priv ( netdev );
+	struct forcedeth_private *priv = netdev->priv;
 
 	DBGP ( "forcedeth_irq\n" );
 
@@ -1814,7 +1814,7 @@ forcedeth_probe ( struct pci_device *pdev )
 	netdev->dev = &pdev->dev;
 
 	/* Get a reference to our private data */
-	priv = netdev_priv ( netdev );
+	priv = netdev->priv;
 
 	/* We'll need these set up for the rest of the routines */
 	priv->pci_dev = pdev;
diff --git a/src/drivers/net/igbvf/igbvf_main.c b/src/drivers/net/igbvf/igbvf_main.c
index a5ed0c451..862ad6a24 100644
--- a/src/drivers/net/igbvf/igbvf_main.c
+++ b/src/drivers/net/igbvf/igbvf_main.c
@@ -179,7 +179,7 @@ static void igbvf_irq_enable ( struct igbvf_adapter *adapter )
  **/
 static void igbvf_irq ( struct net_device *netdev, int enable )
 {
-	struct igbvf_adapter *adapter = netdev_priv ( netdev );
+	struct igbvf_adapter *adapter = netdev->priv;
 
 	DBG ( "igbvf_irq\n" );
 
@@ -197,7 +197,7 @@ static void igbvf_irq ( struct net_device *netdev, int enable )
  **/
 static void igbvf_process_tx_packets ( struct net_device *netdev )
 {
-	struct igbvf_adapter *adapter = netdev_priv ( netdev );
+	struct igbvf_adapter *adapter = netdev->priv;
 	uint32_t i;
 	uint32_t tx_status;
 	union e1000_adv_tx_desc *tx_curr_desc;
@@ -243,7 +243,7 @@ static void igbvf_process_tx_packets ( struct net_device *netdev )
  **/
 static void igbvf_process_rx_packets ( struct net_device *netdev )
 {
-	struct igbvf_adapter *adapter = netdev_priv ( netdev );
+	struct igbvf_adapter *adapter = netdev->priv;
 	struct e1000_hw *hw = &adapter->hw;
 	uint32_t i;
 	uint32_t rx_status;
@@ -306,7 +306,7 @@ static void igbvf_process_rx_packets ( struct net_device *netdev )
  */
 static void igbvf_poll ( struct net_device *netdev )
 {
-	struct igbvf_adapter *adapter = netdev_priv ( netdev );
+	struct igbvf_adapter *adapter = netdev->priv;
 	uint32_t rx_status;
 	union e1000_adv_rx_desc *rx_curr_desc;
 
@@ -612,7 +612,7 @@ int igbvf_setup_rx_resources ( struct igbvf_adapter *adapter )
  **/
 static int igbvf_open ( struct net_device *netdev )
 {
-	struct igbvf_adapter *adapter = netdev_priv ( netdev );
+	struct igbvf_adapter *adapter = netdev->priv;
 	int err;
 
 	DBG ("igbvf_open\n");
@@ -667,7 +667,7 @@ err_setup_tx:
  **/
 static void igbvf_close ( struct net_device *netdev )
 {
-	struct igbvf_adapter *adapter = netdev_priv ( netdev );
+	struct igbvf_adapter *adapter = netdev->priv;
         struct e1000_hw *hw = &adapter->hw;
         uint32_t rxdctl;
 
@@ -698,7 +698,7 @@ static void igbvf_close ( struct net_device *netdev )
  */
 static int igbvf_transmit ( struct net_device *netdev, struct io_buffer *iobuf )
 {
-	struct igbvf_adapter *adapter = netdev_priv ( netdev );
+	struct igbvf_adapter *adapter = netdev->priv;
 	struct e1000_hw *hw = &adapter->hw;
 	uint32_t tx_curr = adapter->tx_tail;
 	union e1000_adv_tx_desc *tx_curr_desc;
@@ -810,7 +810,7 @@ int igbvf_probe ( struct pci_device *pdev )
 	netdev->dev = &pdev->dev;
 
 	/* Initialize driver private storage */
-	adapter = netdev_priv ( netdev );
+	adapter = netdev->priv;
 	memset ( adapter, 0, ( sizeof ( *adapter ) ) );
 
 	adapter->pdev = pdev;
@@ -924,7 +924,7 @@ err_alloc_etherdev:
 void igbvf_remove ( struct pci_device *pdev )
 {
 	struct net_device *netdev = pci_get_drvdata ( pdev );
-	struct igbvf_adapter *adapter = netdev_priv ( netdev );
+	struct igbvf_adapter *adapter = netdev->priv;
 
 	DBG ( "igbvf_remove\n" );
 
diff --git a/src/drivers/net/jme.c b/src/drivers/net/jme.c
index c7307728d..298109c29 100644
--- a/src/drivers/net/jme.c
+++ b/src/drivers/net/jme.c
@@ -1153,7 +1153,7 @@ jme_reload_eeprom(struct jme_adapter *jme)
 static void
 jme_load_macaddr(struct net_device *netdev)
 {
-	struct jme_adapter *jme = netdev_priv(netdev);
+	struct jme_adapter *jme = netdev->priv;
 	unsigned char macaddr[6];
 	u32 val;
 
diff --git a/src/drivers/net/pcnet32.c b/src/drivers/net/pcnet32.c
index c0dea86a8..7da884e5c 100644
--- a/src/drivers/net/pcnet32.c
+++ b/src/drivers/net/pcnet32.c
@@ -690,7 +690,7 @@ pcnet32_hw_start ( struct pcnet32_private *priv )
 static int
 pcnet32_open ( struct net_device *netdev )
 {
-	struct pcnet32_private *priv = netdev_priv ( netdev );
+	struct pcnet32_private *priv = netdev->priv;
 	unsigned long ioaddr = priv->pci_dev->ioaddr;
 	int rc;
 	u16 val;
@@ -754,7 +754,7 @@ err_setup_tx:
 static int
 pcnet32_transmit ( struct net_device *netdev, struct io_buffer *iobuf )
 {
-	struct pcnet32_private *priv = netdev_priv ( netdev );
+	struct pcnet32_private *priv = netdev->priv;
 	unsigned long ioaddr = priv->pci_dev->ioaddr;
 	uint32_t tx_len = iob_len ( iobuf );
 	struct pcnet32_tx_desc *tx_curr_desc;
@@ -802,7 +802,7 @@ pcnet32_transmit ( struct net_device *netdev, struct io_buffer *iobuf )
 static void
 pcnet32_process_tx_packets ( struct net_device *netdev )
 {
-	struct pcnet32_private *priv = netdev_priv ( netdev );
+	struct pcnet32_private *priv = netdev->priv;
 	struct pcnet32_tx_desc *tx_curr_desc;
 
 	DBGP ( "pcnet32_process_tx_packets\n" );
@@ -848,7 +848,7 @@ pcnet32_process_tx_packets ( struct net_device *netdev )
 static void
 pcnet32_process_rx_packets ( struct net_device *netdev )
 {
-	struct pcnet32_private *priv = netdev_priv ( netdev );
+	struct pcnet32_private *priv = netdev->priv;
 	struct pcnet32_rx_desc *rx_curr_desc;
 	u16 status;
 	u32 len;
@@ -913,7 +913,7 @@ pcnet32_process_rx_packets ( struct net_device *netdev )
 static void
 pcnet32_poll ( struct net_device *netdev )
 {
-	struct pcnet32_private *priv = netdev_priv ( netdev );
+	struct pcnet32_private *priv = netdev->priv;
 	unsigned long ioaddr = priv->pci_dev->ioaddr;
 	u16 status;
 
@@ -946,7 +946,7 @@ pcnet32_poll ( struct net_device *netdev )
 static void
 pcnet32_close ( struct net_device *netdev )
 {
-	struct pcnet32_private *priv = netdev_priv ( netdev );
+	struct pcnet32_private *priv = netdev->priv;
 	unsigned long ioaddr = priv->pci_dev->ioaddr;
 
 	DBGP ( "pcnet32_close\n" );
@@ -1003,7 +1003,7 @@ static void pcnet32_irq_disable ( struct pcnet32_private *priv )
 static void
 pcnet32_irq ( struct net_device *netdev, int action )
 {
-	struct pcnet32_private *priv = netdev_priv ( netdev );
+	struct pcnet32_private *priv = netdev->priv;
 
 	DBGP ( "pcnet32_irq\n" );
 
@@ -1061,7 +1061,7 @@ pcnet32_probe ( struct pci_device *pdev )
 	netdev->dev = &pdev->dev;
 
 	/* Get a reference to our private data */
-	priv = netdev_priv ( netdev );
+	priv = netdev->priv;
 
 	/* We'll need these set up for the rest of the routines */
 	priv->pci_dev = pdev;
diff --git a/src/drivers/net/phantom/phantom.c b/src/drivers/net/phantom/phantom.c
index 843459059..e5fd1f31f 100644
--- a/src/drivers/net/phantom/phantom.c
+++ b/src/drivers/net/phantom/phantom.c
@@ -1062,7 +1062,7 @@ static inline int phantom_del_macaddr ( struct phantom_nic *phantom,
  * @v netdev		Network device
  */
 static void phantom_poll_link_state ( struct net_device *netdev ) {
-	struct phantom_nic *phantom = netdev_priv ( netdev );
+	struct phantom_nic *phantom = netdev->priv;
 	uint32_t xg_state_p3;
 	unsigned int link;
 
@@ -1109,7 +1109,7 @@ static void phantom_poll_link_state ( struct net_device *netdev ) {
  * @v netdev		Net device
  */
 static void phantom_refill_rx_ring ( struct net_device *netdev ) {
-	struct phantom_nic *phantom = netdev_priv ( netdev );
+	struct phantom_nic *phantom = netdev->priv;
 	struct io_buffer *iobuf;
 	struct phantom_rds rds;
 	unsigned int handle;
@@ -1160,7 +1160,7 @@ static void phantom_refill_rx_ring ( struct net_device *netdev ) {
  * @ret rc		Return status code
  */
 static int phantom_open ( struct net_device *netdev ) {
-	struct phantom_nic *phantom = netdev_priv ( netdev );
+	struct phantom_nic *phantom = netdev->priv;
 	int rc;
 
 	/* Allocate and zero descriptor rings */
@@ -1220,7 +1220,7 @@ static int phantom_open ( struct net_device *netdev ) {
  * @v netdev		Net device
  */
 static void phantom_close ( struct net_device *netdev ) {
-	struct phantom_nic *phantom = netdev_priv ( netdev );
+	struct phantom_nic *phantom = netdev->priv;
 	struct io_buffer *iobuf;
 	unsigned int i;
 
@@ -1258,7 +1258,7 @@ static void phantom_close ( struct net_device *netdev ) {
  */
 static int phantom_transmit ( struct net_device *netdev,
 			      struct io_buffer *iobuf ) {
-	struct phantom_nic *phantom = netdev_priv ( netdev );
+	struct phantom_nic *phantom = netdev->priv;
 	union phantom_cds cds;
 	int index;
 
@@ -1297,7 +1297,7 @@ static int phantom_transmit ( struct net_device *netdev,
  * @v netdev	Network device
  */
 static void phantom_poll ( struct net_device *netdev ) {
-	struct phantom_nic *phantom = netdev_priv ( netdev );
+	struct phantom_nic *phantom = netdev->priv;
 	struct io_buffer *iobuf;
 	unsigned int irq_vector;
 	unsigned int irq_state;
@@ -1434,7 +1434,7 @@ static void phantom_poll ( struct net_device *netdev ) {
  * @v enable	Interrupts should be enabled
  */
 static void phantom_irq ( struct net_device *netdev, int enable ) {
-	struct phantom_nic *phantom = netdev_priv ( netdev );
+	struct phantom_nic *phantom = netdev->priv;
 
 	phantom_writel ( phantom, ( enable ? 1 : 0 ),
 			 phantom->sds_irq_mask_crb );
@@ -2070,7 +2070,7 @@ static int phantom_probe ( struct pci_device *pci ) {
 		goto err_alloc_etherdev;
 	}
 	netdev_init ( netdev, &phantom_operations );
-	phantom = netdev_priv ( netdev );
+	phantom = netdev->priv;
 	pci_set_drvdata ( pci, netdev );
 	netdev->dev = &pci->dev;
 	memset ( phantom, 0, sizeof ( *phantom ) );
@@ -2161,7 +2161,7 @@ static int phantom_probe ( struct pci_device *pci ) {
  */
 static void phantom_remove ( struct pci_device *pci ) {
 	struct net_device *netdev = pci_get_drvdata ( pci );
-	struct phantom_nic *phantom = netdev_priv ( netdev );
+	struct phantom_nic *phantom = netdev->priv;
 
 	unregister_settings ( &phantom->settings );
 	unregister_netdev ( netdev );
diff --git a/src/drivers/net/sfc/efx_common.c b/src/drivers/net/sfc/efx_common.c
index ad572b1da..2b7a88a5e 100644
--- a/src/drivers/net/sfc/efx_common.c
+++ b/src/drivers/net/sfc/efx_common.c
@@ -70,7 +70,7 @@ efx_readl(struct efx_nic *efx, efx_dword_t *value, unsigned int reg)
  ******************************************************************************/
 void efx_probe(struct net_device *netdev, enum efx_revision revision)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 	struct pci_device *pci = container_of(netdev->dev,
 					      struct pci_device, dev);
 	unsigned int reg = PCI_BASE_ADDRESS_0;
@@ -97,7 +97,7 @@ void efx_probe(struct net_device *netdev, enum efx_revision revision)
 
 void efx_remove(struct net_device *netdev)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 
 	iounmap(efx->membase);
 	efx->membase = NULL;
diff --git a/src/drivers/net/sfc/efx_hunt.c b/src/drivers/net/sfc/efx_hunt.c
index 0bce3e45a..abe3e8320 100644
--- a/src/drivers/net/sfc/efx_hunt.c
+++ b/src/drivers/net/sfc/efx_hunt.c
@@ -100,7 +100,7 @@ efx_hunt_notify_tx_desc(struct efx_nic *efx)
 int
 efx_hunt_transmit(struct net_device *netdev, struct io_buffer *iob)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 	struct efx_tx_queue *txq = &efx->txq;
 	int fill_level, space;
 	efx_tx_desc_t *txd;
@@ -155,7 +155,7 @@ efx_hunt_transmit_done(struct efx_nic *efx, int id)
 
 int efx_hunt_tx_init(struct net_device *netdev, dma_addr_t *dma_addr)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 	struct efx_tx_queue *txq = &efx->txq;
 	size_t bytes;
 
@@ -270,7 +270,7 @@ efx_hunt_receive(struct efx_nic *efx, unsigned int id, int len, int drop)
 
 int efx_hunt_rx_init(struct net_device *netdev, dma_addr_t *dma_addr)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 	struct efx_rx_queue *rxq = &efx->rxq;
 	size_t bytes;
 
@@ -294,7 +294,7 @@ int efx_hunt_rx_init(struct net_device *netdev, dma_addr_t *dma_addr)
  ******************************************************************************/
 int efx_hunt_ev_init(struct net_device *netdev, dma_addr_t *dma_addr)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 	struct efx_ev_queue *evq = &efx->evq;
 	size_t bytes;
 
@@ -404,7 +404,7 @@ efx_hunt_handle_event(struct efx_nic *efx, efx_event_t *evt)
 
 void efx_hunt_poll(struct net_device *netdev)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 	struct efx_ev_queue *evq = &efx->evq;
 	efx_event_t *evt;
 	int budget = 10;
@@ -443,7 +443,7 @@ void efx_hunt_poll(struct net_device *netdev)
 
 void efx_hunt_irq(struct net_device *netdev, int enable)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 
 	efx->int_en = enable;
 
@@ -465,7 +465,7 @@ void efx_hunt_irq(struct net_device *netdev, int enable)
  ******************************************************************************/
 int efx_hunt_open(struct net_device *netdev)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 	efx_dword_t cmd;
 
 	/* Set interrupt moderation to 0*/
@@ -486,7 +486,7 @@ int efx_hunt_open(struct net_device *netdev)
 
 void efx_hunt_close(struct net_device *netdev)
 {
-	struct efx_nic *efx = netdev_priv(netdev);
+	struct efx_nic *efx = netdev->priv;
 	struct efx_rx_queue *rxq = &efx->rxq;
 	struct efx_tx_queue *txq = &efx->txq;
 	int i;
diff --git a/src/drivers/net/sfc/sfc_hunt.c b/src/drivers/net/sfc/sfc_hunt.c
index a37670ae2..43ac229ab 100644
--- a/src/drivers/net/sfc/sfc_hunt.c
+++ b/src/drivers/net/sfc/sfc_hunt.c
@@ -1043,7 +1043,7 @@ static void hunt_ev_fini(struct hunt_nic *hunt)
 static void
 hunt_poll(struct net_device *netdev)
 {
-	struct hunt_nic *hunt = netdev_priv(netdev);
+	struct hunt_nic *hunt = netdev->priv;
 
 	/* If called while already polling, return immediately */
 	if (hunt->efx.state & EFX_STATE_POLLING)
@@ -1071,7 +1071,7 @@ hunt_poll(struct net_device *netdev)
  ******************************************************************************/
 static int hunt_open(struct net_device *netdev)
 {
-	struct hunt_nic *hunt = netdev_priv(netdev);
+	struct hunt_nic *hunt = netdev->priv;
 	int rc;
 
 	/* Allocate VIs */
@@ -1133,7 +1133,7 @@ fail2:
 
 static void hunt_close(struct net_device *netdev)
 {
-	struct hunt_nic *hunt = netdev_priv(netdev);
+	struct hunt_nic *hunt = netdev->priv;
 
 	/* Stop datapath */
 	efx_hunt_close(netdev);
@@ -1187,7 +1187,7 @@ hunt_probe(struct pci_device *pci)
 	netdev->dev = &pci->dev;
 	netdev->state |= NETDEV_IRQ_UNSUPPORTED;
 
-	hunt = netdev_priv(netdev);
+	hunt = netdev->priv;
 	memset(hunt, 0, sizeof(*hunt));
 	efx = &hunt->efx;
 
@@ -1290,7 +1290,7 @@ fail1:
 static void hunt_remove(struct pci_device *pci)
 {
 	struct net_device *netdev = pci_get_drvdata(pci);
-	struct hunt_nic *hunt = netdev_priv(netdev);
+	struct hunt_nic *hunt = netdev->priv;
 
 	if (!(hunt->flags &
 	      (1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_NO_ACTIVE_PORT))) {
diff --git a/src/drivers/net/sis190.c b/src/drivers/net/sis190.c
index 0e4f0762e..034cac9e0 100644
--- a/src/drivers/net/sis190.c
+++ b/src/drivers/net/sis190.c
@@ -107,14 +107,14 @@ static int mdio_read(void *ioaddr, int phy_id, int reg)
 
 static void __mdio_write(struct net_device *dev, int phy_id, int reg, int val)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 
 	mdio_write(tp->mmio_addr, phy_id, reg, val);
 }
 
 static int __mdio_read(struct net_device *dev, int phy_id, int reg)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 
 	return mdio_read(tp->mmio_addr, phy_id, reg);
 }
@@ -343,7 +343,7 @@ static void sis190_process_tx(struct sis190_private *tp)
  */
 static void sis190_poll(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void  *ioaddr = tp->mmio_addr;
 	u32 status;
 
@@ -374,7 +374,7 @@ static inline void sis190_init_ring_indexes(struct sis190_private *tp)
 
 static int sis190_init_ring(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 
 	sis190_init_ring_indexes(tp);
 
@@ -395,7 +395,7 @@ err:
 
 static void sis190_set_rx_mode(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void *ioaddr = tp->mmio_addr;
 	u32 mc_filter[2];	/* Multicast hash filter */
 	u16 rx_mode;
@@ -419,7 +419,7 @@ static void sis190_soft_reset(void  *ioaddr)
 
 static void sis190_hw_start(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void *ioaddr = tp->mmio_addr;
 
 	sis190_soft_reset(ioaddr);
@@ -548,7 +548,7 @@ static void sis190_phy_task(struct sis190_private *tp)
 
 static int sis190_open(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	int rc;
 
 	/* Allocate TX ring */
@@ -587,7 +587,7 @@ error:
 
 static void sis190_down(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void  *ioaddr = tp->mmio_addr;
 
 	do {
@@ -597,7 +597,7 @@ static void sis190_down(struct net_device *dev)
 
 static void sis190_free(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	int i;
 
 	free_phys(tp->TxDescRing, TX_RING_BYTES);
@@ -630,7 +630,7 @@ static void sis190_close(struct net_device *dev)
 
 static int sis190_transmit(struct net_device *dev, struct io_buffer *iob)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void  *ioaddr = tp->mmio_addr;
 	u32 len, entry;
 	struct TxDesc *desc;
@@ -804,7 +804,7 @@ static void sis190_mii_probe_88e1111_fixup(struct sis190_private *tp)
  */
 static int sis190_mii_probe(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	struct mii_if_info *mii_if = &tp->mii_if;
 	void *ioaddr = tp->mmio_addr;
 	int phy_id;
@@ -858,7 +858,7 @@ out:
 
 static void sis190_mii_remove(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 
 	sis190_free_phy(&tp->first_phy);
 }
@@ -879,7 +879,7 @@ static int sis190_init_board(struct pci_device *pdev, struct net_device **netdev
 
 	dev->dev = &pdev->dev;
 
-	tp = netdev_priv(dev);
+	tp = dev->priv;
 	memset(tp, 0, sizeof(*tp));
 
 	tp->dev = dev;
@@ -916,7 +916,7 @@ static void sis190_set_rgmii(struct sis190_private *tp, u8 reg)
 static int sis190_get_mac_addr_from_eeprom(struct pci_device *pdev __unused,
 						     struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void *ioaddr = tp->mmio_addr;
 	u16 sig;
 	int i;
@@ -955,7 +955,7 @@ static int sis190_get_mac_addr_from_eeprom(struct pci_device *pdev __unused,
 static int sis190_get_mac_addr_from_apc(struct pci_device *pdev,
 					struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	struct pci_device *isa_bridge = NULL;
 	struct device *d;
 	u8 reg, tmp8;
@@ -1018,7 +1018,7 @@ static int sis190_get_mac_addr_from_apc(struct pci_device *pdev,
  */
 static inline void sis190_init_rxfilter(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void *ioaddr = tp->mmio_addr;
 	u16 ctl;
 	int i;
@@ -1057,7 +1057,7 @@ static int sis190_get_mac_addr(struct pci_device *pdev,
 
 static void sis190_set_speed_auto(struct net_device *dev)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void *ioaddr = tp->mmio_addr;
 	int phy_id = tp->mii_if.phy_id;
 	int val;
@@ -1082,7 +1082,7 @@ static void sis190_set_speed_auto(struct net_device *dev)
 
 static void sis190_irq(struct net_device *dev, int enable)
 {
-	struct sis190_private *tp = netdev_priv(dev);
+	struct sis190_private *tp = dev->priv;
 	void *ioaddr = tp->mmio_addr;
 
 	SIS_W32(IntrStatus, 0xffffffff);
@@ -1116,7 +1116,7 @@ static int sis190_probe(struct pci_device *pdev)
 
 	pci_set_drvdata(pdev, dev);
 
-	tp = netdev_priv(dev);
+	tp = dev->priv;
 
 	rc = sis190_get_mac_addr(pdev, dev);
 	if (rc < 0)
diff --git a/src/drivers/net/skge.c b/src/drivers/net/skge.c
index 5aa5e2a6a..cc7f0b91b 100755
--- a/src/drivers/net/skge.c
+++ b/src/drivers/net/skge.c
@@ -213,7 +213,7 @@ static void skge_led(struct skge_port *skge, enum led_mode mode)
  *
  * static int skge_get_eeprom_len(struct net_device *dev)
  * {
- * 	struct skge_port *skge = netdev_priv(dev);
+ * 	struct skge_port *skge = dev->priv;
  * 	u32 reg2;
  *
  * 	pci_read_config_dword(skge->hw->pdev, PCI_DEV_REG2, &reg2);
@@ -248,7 +248,7 @@ static void skge_led(struct skge_port *skge, enum led_mode mode)
  * static int skge_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
  * 			   u8 *data)
  * {
- * 	struct skge_port *skge = netdev_priv(dev);
+ * 	struct skge_port *skge = dev->priv;
  * 	struct pci_dev *pdev = skge->hw->pdev;
  * 	int cap = pci_find_capability(pdev, PCI_CAP_ID_VPD);
  * 	int length = eeprom->len;
@@ -274,7 +274,7 @@ static void skge_led(struct skge_port *skge, enum led_mode mode)
  * static int skge_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
  * 			   u8 *data)
  * {
- * 	struct skge_port *skge = netdev_priv(dev);
+ * 	struct skge_port *skge = dev->priv;
  * 	struct pci_dev *pdev = skge->hw->pdev;
  * 	int cap = pci_find_capability(pdev, PCI_CAP_ID_VPD);
  * 	int length = eeprom->len;
@@ -415,7 +415,7 @@ static void skge_link_down(struct skge_port *skge)
 static void xm_link_down(struct skge_hw *hw, int port)
 {
 	struct net_device *dev = hw->dev[port];
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 
 	xm_write16(hw, port, XM_IMSK, XM_IMSK_DISABLE);
 
@@ -553,7 +553,7 @@ static const u16 fiber_pause_map[] = {
 static void bcom_check_link(struct skge_hw *hw, int port)
 {
 	struct net_device *dev = hw->dev[port];
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	u16 status;
 
 	/* read twice because of latch */
@@ -751,7 +751,7 @@ static void xm_phy_init(struct skge_port *skge)
 
 static int xm_check_link(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_hw *hw = skge->hw;
 	int port = skge->port;
 	u16 status;
@@ -852,7 +852,7 @@ static void xm_link_timer(struct skge_port *skge)
 static void genesis_mac_init(struct skge_hw *hw, int port)
 {
 	struct net_device *dev = hw->dev[port];
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	int i;
 	u32 r;
 	const u8 zero[6]  = { 0 };
@@ -1209,7 +1209,7 @@ static u16 gm_phy_read(struct skge_hw *hw, int port, u16 reg)
 /* Marvell Phy Initialization */
 static void yukon_init(struct skge_hw *hw, int port)
 {
-	struct skge_port *skge = netdev_priv(hw->dev[port]);
+	struct skge_port *skge = hw->dev[port]->priv;
 	u16 ctrl, ct1000, adv;
 
 	if (skge->autoneg == AUTONEG_ENABLE) {
@@ -1325,7 +1325,7 @@ static int is_yukon_lite_a0(struct skge_hw *hw)
 
 static void yukon_mac_init(struct skge_hw *hw, int port)
 {
-	struct skge_port *skge = netdev_priv(hw->dev[port]);
+	struct skge_port *skge = hw->dev[port]->priv;
 	int i;
 	u32 reg;
 	const u8 *addr = hw->dev[port]->ll_addr;
@@ -1691,7 +1691,7 @@ static void skge_qset(struct skge_port *skge, u16 q,
 
 void skge_free(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 
 	free(skge->rx_ring.start);
 	skge->rx_ring.start = NULL;
@@ -1706,7 +1706,7 @@ void skge_free(struct net_device *dev)
 
 static int skge_up(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_hw *hw = skge->hw;
 	int port = skge->port;
 	u32 chunk, ram_addr;
@@ -1789,7 +1789,7 @@ static void skge_rx_stop(struct skge_hw *hw, int port)
 
 static void skge_down(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_hw *hw = skge->hw;
 	int port = skge->port;
 
@@ -1862,7 +1862,7 @@ static inline int skge_tx_avail(const struct skge_ring *ring)
 
 static int skge_xmit_frame(struct net_device *dev, struct io_buffer *iob)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_hw *hw = skge->hw;
 	struct skge_element *e;
 	struct skge_tx_desc *td;
@@ -1908,7 +1908,7 @@ static int skge_xmit_frame(struct net_device *dev, struct io_buffer *iob)
 /* Free all buffers in transmit ring */
 static void skge_tx_clean(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_element *e;
 
 	for (e = skge->tx_ring.to_clean; e != skge->tx_ring.to_use; e = e->next) {
@@ -1939,7 +1939,7 @@ static inline int bad_phy_status(const struct skge_hw *hw, u32 status)
 /* Free all buffers in Tx ring which are no longer owned by device */
 static void skge_tx_done(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_ring *ring = &skge->tx_ring;
 	struct skge_element *e;
 
@@ -1961,7 +1961,7 @@ static void skge_tx_done(struct net_device *dev)
 
 static void skge_rx_refill(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_ring *ring = &skge->rx_ring;
 	struct skge_element *e;
 	struct io_buffer *iob;
@@ -2003,7 +2003,7 @@ static void skge_rx_refill(struct net_device *dev)
 
 static void skge_rx_done(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_ring *ring = &skge->rx_ring;
 	struct skge_rx_desc *rd;
 	struct skge_element *e;
@@ -2050,7 +2050,7 @@ static void skge_rx_done(struct net_device *dev)
 
 static void skge_poll(struct net_device *dev)
 {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_hw *hw = skge->hw;
 	u32 status;
 
@@ -2085,7 +2085,7 @@ static void skge_phyirq(struct skge_hw *hw)
 
 	for (port = 0; port < hw->ports; port++) {
 		struct net_device *dev = hw->dev[port];
-		struct skge_port *skge = netdev_priv(dev);
+		struct skge_port *skge = dev->priv;
 
 		if (hw->chip_id != CHIP_ID_GENESIS)
 			yukon_phy_intr(skge);
@@ -2302,7 +2302,7 @@ static struct net_device *skge_devinit(struct skge_hw *hw, int port,
 
 	dev->dev = &hw->pdev->dev;
 
-	skge = netdev_priv(dev);
+	skge = dev->priv;
 	skge->netdev = dev;
 	skge->hw = hw;
 
@@ -2446,7 +2446,7 @@ static void skge_remove(struct pci_device *pdev)
  * This is a iPXE Network Driver API function.
  */
 static void skge_net_irq ( struct net_device *dev, int enable ) {
-	struct skge_port *skge = netdev_priv(dev);
+	struct skge_port *skge = dev->priv;
 	struct skge_hw *hw = skge->hw;
 
 	if (enable)
diff --git a/src/drivers/net/sky2.c b/src/drivers/net/sky2.c
index 9d612c997..26396585f 100644
--- a/src/drivers/net/sky2.c
+++ b/src/drivers/net/sky2.c
@@ -296,7 +296,7 @@ static const u16 gm_fc_disable[] = {
 
 static void sky2_phy_init(struct sky2_hw *hw, unsigned port)
 {
-	struct sky2_port *sky2 = netdev_priv(hw->dev[port]);
+	struct sky2_port *sky2 = hw->dev[port]->priv;
 	u16 ctrl, ct1000, adv, pg, ledctrl, ledover, reg;
 
 	if (sky2->autoneg == AUTONEG_ENABLE &&
@@ -1128,7 +1128,7 @@ static void sky2_free_rings(struct sky2_port *sky2)
 /* Bring up network interface. */
 static int sky2_up(struct net_device *dev)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	struct sky2_hw *hw = sky2->hw;
 	unsigned port = sky2->port;
 	u32 imask, ramsize;
@@ -1237,7 +1237,7 @@ static inline int tx_avail(const struct sky2_port *sky2)
  */
 static int sky2_xmit_frame(struct net_device *dev, struct io_buffer *iob)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	struct sky2_hw *hw = sky2->hw;
 	struct sky2_tx_le *le = NULL;
 	struct tx_ring_info *re;
@@ -1303,7 +1303,7 @@ static void sky2_tx_complete(struct sky2_port *sky2, u16 done)
 /* Cleanup all untransmitted buffers, assume transmitter not running */
 static void sky2_tx_clean(struct net_device *dev)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 
 	sky2_tx_complete(sky2, sky2->tx_prod);
 }
@@ -1311,7 +1311,7 @@ static void sky2_tx_clean(struct net_device *dev)
 /* Network shutdown */
 static void sky2_down(struct net_device *dev)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	struct sky2_hw *hw = sky2->hw;
 	unsigned port = sky2->port;
 	u16 ctrl;
@@ -1511,7 +1511,7 @@ static int sky2_autoneg_done(struct sky2_port *sky2, u16 aux)
 static void sky2_phy_intr(struct sky2_hw *hw, unsigned port)
 {
 	struct net_device *dev = hw->dev[port];
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	u16 istatus, phystat;
 
 	istatus = gm_phy_read(hw, port, PHY_MARV_INT_STAT);
@@ -1570,7 +1570,7 @@ static struct io_buffer *receive_new(struct sky2_port *sky2,
 static struct io_buffer *sky2_receive(struct net_device *dev,
 				      u16 length, u32 status)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	struct rx_ring_info *re = sky2->rx_ring + sky2->rx_next;
 	struct io_buffer *iob = NULL;
 	u16 count = (status & GMR_FS_LEN) >> 16;
@@ -1634,7 +1634,7 @@ error:
 /* Transmit complete */
 static inline void sky2_tx_done(struct net_device *dev, u16 last)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 
 	sky2_tx_complete(sky2, last);
 }
@@ -1700,10 +1700,10 @@ static void sky2_status_intr(struct sky2_hw *hw, u16 idx)
 	sky2_write32(hw, STAT_CTRL, SC_STAT_CLR_IRQ);
 
 	if (rx[0])
-		sky2_rx_update(netdev_priv(hw->dev[0]), Q_R1);
+		sky2_rx_update(hw->dev[0]->priv, Q_R1);
 
 	if (rx[1])
-		sky2_rx_update(netdev_priv(hw->dev[1]), Q_R2);
+		sky2_rx_update(hw->dev[1]->priv, Q_R2);
 }
 
 static void sky2_hw_error(struct sky2_hw *hw, unsigned port, u32 status)
@@ -1809,7 +1809,7 @@ static void sky2_le_error(struct sky2_hw *hw, unsigned port,
 			  u16 q, unsigned ring_size __unused)
 {
 	struct net_device *dev = hw->dev[port];
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	int idx;
 	const u64 *le = (q == Q_R1 || q == Q_R2)
 		? (u64 *) sky2->rx_le : (u64 *) sky2->tx_le;
@@ -1853,7 +1853,7 @@ static void sky2_err_intr(struct sky2_hw *hw, u32 status)
 
 static void sky2_poll(struct net_device *dev)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	struct sky2_hw *hw = sky2->hw;
 	u32 status = sky2_read32(hw, B0_Y2_SP_EISR);
 	u16 idx;
@@ -2152,7 +2152,7 @@ static u32 sky2_supported_modes(const struct sky2_hw *hw)
 
 static void sky2_set_multicast(struct net_device *dev)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	struct sky2_hw *hw = sky2->hw;
 	unsigned port = sky2->port;
 	u16 reg;
@@ -2189,7 +2189,7 @@ static struct net_device *sky2_init_netdev(struct sky2_hw *hw,
 
 	dev->dev = &hw->pdev->dev;
 
-	sky2 = netdev_priv(dev);
+	sky2 = dev->priv;
 	sky2->netdev = dev;
 	sky2->hw = hw;
 
@@ -2241,7 +2241,7 @@ static const char *sky2_name(u8 chipid, char *buf, int sz)
 
 static void sky2_net_irq(struct net_device *dev, int enable)
 {
-	struct sky2_port *sky2 = netdev_priv(dev);
+	struct sky2_port *sky2 = dev->priv;
 	struct sky2_hw *hw = sky2->hw;
 
 	u32 imask = sky2_read32(hw, B0_IMSK);
diff --git a/src/drivers/net/tg3/tg3.c b/src/drivers/net/tg3/tg3.c
index cec599c1c..559c2d638 100644
--- a/src/drivers/net/tg3/tg3.c
+++ b/src/drivers/net/tg3/tg3.c
@@ -233,7 +233,7 @@ int tg3_init_rings(struct tg3 *tp)
 static int tg3_open(struct net_device *dev)
 {	DBGP("%s\n", __func__);
 
-	struct tg3 *tp = netdev_priv(dev);
+	struct tg3 *tp = dev->priv;
 	struct tg3_rx_prodring_set *tpr = &tp->prodring;
 	int err = 0;
 
@@ -299,7 +299,7 @@ static void __unused tw32_mailbox2(struct tg3 *tp, uint32_t reg, uint32_t val)
 static int tg3_transmit(struct net_device *dev, struct io_buffer *iob)
 {	DBGP("%s\n", __func__);
 
-	struct tg3 *tp = netdev_priv(dev);
+	struct tg3 *tp = dev->priv;
 	u32 len, entry;
 	dma_addr_t mapping;
 
@@ -333,7 +333,7 @@ static int tg3_transmit(struct net_device *dev, struct io_buffer *iob)
 static void tg3_tx_complete(struct net_device *dev)
 {	DBGP("%s\n", __func__);
 
-	struct tg3 *tp = netdev_priv(dev);
+	struct tg3 *tp = dev->priv;
 	u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
 	u32 sw_idx = tp->tx_cons;
 
@@ -427,7 +427,7 @@ static void tg3_refill_prod_ring(struct tg3 *tp)
 static void tg3_rx_complete(struct net_device *dev)
 {	DBGP("%s\n", __func__);
 
-	struct tg3 *tp = netdev_priv(dev);
+	struct tg3 *tp = dev->priv;
 
 	u32 sw_idx = tp->rx_rcb_ptr;
 	u16 hw_idx;
@@ -478,7 +478,7 @@ static void tg3_rx_complete(struct net_device *dev)
 static void tg3_poll(struct net_device *dev)
 {	DBGP("%s\n", __func__);
 
-	struct tg3 *tp = netdev_priv(dev);
+	struct tg3 *tp = dev->priv;
 
 	/* ACK interrupts */
 	/*
@@ -496,7 +496,7 @@ static void tg3_poll(struct net_device *dev)
 static void tg3_close(struct net_device *dev)
 {	DBGP("%s\n", __func__);
 
-	struct tg3 *tp = netdev_priv(dev);
+	struct tg3 *tp = dev->priv;
 
 	DBGP("%s\n", __func__);
 
@@ -511,7 +511,7 @@ static void tg3_close(struct net_device *dev)
 static void tg3_irq(struct net_device *dev, int enable)
 {	DBGP("%s\n", __func__);
 
-	struct tg3 *tp = netdev_priv(dev);
+	struct tg3 *tp = dev->priv;
 
 	DBGP("%s: %d\n", __func__, enable);
 
@@ -735,7 +735,7 @@ static int tg3_init_one(struct pci_device *pdev)
 
 	dev->dev = &pdev->dev;
 
-	tp = netdev_priv(dev);
+	tp = dev->priv;
 	tp->pdev = pdev;
 	tp->dev = dev;
 	tp->rx_mode = TG3_DEF_RX_MODE;
diff --git a/src/drivers/net/tg3/tg3_hw.c b/src/drivers/net/tg3/tg3_hw.c
index 798f8519f..9a70413b6 100644
--- a/src/drivers/net/tg3/tg3_hw.c
+++ b/src/drivers/net/tg3/tg3_hw.c
@@ -1717,7 +1717,7 @@ int tg3_get_device_address(struct tg3 *tp)
 static void __tg3_set_rx_mode(struct net_device *dev)
 {	DBGP("%s\n", __func__);
 
-	struct tg3 *tp = netdev_priv(dev);
+	struct tg3 *tp = dev->priv;
 	u32 rx_mode;
 
 	rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
diff --git a/src/drivers/net/vmxnet3.c b/src/drivers/net/vmxnet3.c
index 63bcf0e01..3800d6b72 100644
--- a/src/drivers/net/vmxnet3.c
+++ b/src/drivers/net/vmxnet3.c
@@ -90,7 +90,7 @@ static inline uint32_t vmxnet3_command ( struct vmxnet3_nic *vmxnet,
  */
 static int vmxnet3_transmit ( struct net_device *netdev,
 			      struct io_buffer *iobuf ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	struct vmxnet3_tx_desc *tx_desc;
 	unsigned int fill;
 	unsigned int desc_idx;
@@ -139,7 +139,7 @@ static int vmxnet3_transmit ( struct net_device *netdev,
  * @v netdev		Network device
  */
 static void vmxnet3_poll_tx ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	struct vmxnet3_tx_comp *tx_comp;
 	struct io_buffer *iobuf;
 	unsigned int comp_idx;
@@ -188,7 +188,7 @@ static void vmxnet3_poll_tx ( struct net_device *netdev ) {
  * @v netdev		Network device
  */
 static void vmxnet3_flush_tx ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	unsigned int i;
 
 	for ( i = 0 ; i < VMXNET3_NUM_TX_DESC ; i++ ) {
@@ -206,7 +206,7 @@ static void vmxnet3_flush_tx ( struct net_device *netdev ) {
  * @v netdev		Network device
  */
 static void vmxnet3_refill_rx ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	struct vmxnet3_rx_desc *rx_desc;
 	struct io_buffer *iobuf;
 	unsigned int orig_rx_prod = vmxnet->count.rx_prod;
@@ -261,7 +261,7 @@ static void vmxnet3_refill_rx ( struct net_device *netdev ) {
  * @v netdev		Network device
  */
 static void vmxnet3_poll_rx ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	struct vmxnet3_rx_comp *rx_comp;
 	struct io_buffer *iobuf;
 	unsigned int comp_idx;
@@ -315,7 +315,7 @@ static void vmxnet3_poll_rx ( struct net_device *netdev ) {
  * @v netdev		Network device
  */
 static void vmxnet3_flush_rx ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	struct io_buffer *iobuf;
 	unsigned int i;
 
@@ -333,7 +333,7 @@ static void vmxnet3_flush_rx ( struct net_device *netdev ) {
  * @v netdev		Network device
  */
 static void vmxnet3_check_link ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	uint32_t state;
 	int link_up;
 	unsigned int link_speed;
@@ -360,7 +360,7 @@ static void vmxnet3_check_link ( struct net_device *netdev ) {
  * @v netdev		Network device
  */
 static void vmxnet3_poll_events ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	uint32_t events;
 
 	/* Do nothing unless there are events to process */
@@ -424,7 +424,7 @@ static void vmxnet3_poll ( struct net_device *netdev ) {
  * @v enable		Interrupts should be enabled
  */
 static void vmxnet3_irq ( struct net_device *netdev, int enable ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 
 	DBGC ( vmxnet, "VMXNET3 %p %s IRQ not implemented\n",
 	       vmxnet, ( enable ? "enable" : "disable" ) );
@@ -456,7 +456,7 @@ static void vmxnet3_set_ll_addr ( struct vmxnet3_nic *vmxnet,
  * @ret rc		Return status code
  */
 static int vmxnet3_open ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 	struct vmxnet3_shared *shared;
 	struct vmxnet3_queues *queues;
 	uint64_t shared_bus;
@@ -554,7 +554,7 @@ static int vmxnet3_open ( struct net_device *netdev ) {
  * @v netdev		Network device
  */
 static void vmxnet3_close ( struct net_device *netdev ) {
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 
 	vmxnet3_command ( vmxnet, VMXNET3_CMD_QUIESCE_DEV );
 	vmxnet3_command ( vmxnet, VMXNET3_CMD_RESET_DEV );
@@ -633,7 +633,7 @@ static int vmxnet3_probe ( struct pci_device *pci ) {
 		goto err_alloc_etherdev;
 	}
 	netdev_init ( netdev, &vmxnet3_operations );
-	vmxnet = netdev_priv ( netdev );
+	vmxnet = netdev->priv;
 	pci_set_drvdata ( pci, netdev );
 	netdev->dev = &pci->dev;
 	memset ( vmxnet, 0, sizeof ( *vmxnet ) );
@@ -699,7 +699,7 @@ static int vmxnet3_probe ( struct pci_device *pci ) {
  */
 static void vmxnet3_remove ( struct pci_device *pci ) {
 	struct net_device *netdev = pci_get_drvdata ( pci );
-	struct vmxnet3_nic *vmxnet = netdev_priv ( netdev );
+	struct vmxnet3_nic *vmxnet = netdev->priv;
 
 	unregister_netdev ( netdev );
 	iounmap ( vmxnet->vd );
diff --git a/src/drivers/net/vxge/vxge_main.c b/src/drivers/net/vxge/vxge_main.c
index 631928318..e323701f2 100644
--- a/src/drivers/net/vxge/vxge_main.c
+++ b/src/drivers/net/vxge/vxge_main.c
@@ -186,7 +186,7 @@ vxge_xmit(struct net_device *dev, struct io_buffer *iobuf)
 
 	vxge_trace();
 
-	vdev = (struct vxgedev *)netdev_priv(dev);
+	vdev = (struct vxgedev *)dev->priv;
 
 	if (!is_vxge_card_up(vdev)) {
 		vxge_debug(VXGE_ERR,
@@ -235,7 +235,7 @@ static void vxge_poll(struct net_device *ndev)
 
 	vxge_debug(VXGE_POLL, "%s:%d \n", __func__, __LINE__);
 
-	vdev = (struct vxgedev *)netdev_priv(ndev);
+	vdev = (struct vxgedev *)ndev->priv;
 	hldev = (struct __vxge_hw_device *)pci_get_drvdata(vdev->pdev);
 
 	if (!is_vxge_card_up(vdev))
@@ -263,7 +263,7 @@ static void vxge_irq(struct net_device *netdev __unused, int action)
 	vxge_debug(VXGE_INFO,
 		"%s:%d action(%d)\n", __func__, __LINE__, action);
 
-	vdev = (struct vxgedev *)netdev_priv(netdev);
+	vdev = (struct vxgedev *)netdev->priv;
 	hldev = (struct __vxge_hw_device *)pci_get_drvdata(vdev->pdev);
 
 	switch (action) {
@@ -297,7 +297,7 @@ vxge_open(struct net_device *dev)
 	vxge_debug(VXGE_INFO, "%s: %s:%d\n",
 			VXGE_DRIVER_NAME, __func__, __LINE__);
 
-	vdev = (struct vxgedev *)netdev_priv(dev);
+	vdev = (struct vxgedev *)dev->priv;
 	hldev = (struct __vxge_hw_device *)pci_get_drvdata(vdev->pdev);
 
 	/* make sure you have link off by default every time Nic is
@@ -369,7 +369,7 @@ static void vxge_close(struct net_device *dev)
 	vxge_debug(VXGE_INFO, "%s: %s:%d\n",
 		dev->name, __func__, __LINE__);
 
-	vdev = (struct vxgedev *)netdev_priv(dev);
+	vdev = (struct vxgedev *)dev->priv;
 	hldev = (struct __vxge_hw_device *)pci_get_drvdata(vdev->pdev);
 
 	if (!is_vxge_card_up(vdev))
@@ -420,7 +420,7 @@ int vxge_device_register(struct __vxge_hw_device *hldev,
 
 	vxge_debug(VXGE_INFO, "%s:%d  netdev registering\n",
 		__func__, __LINE__);
-	vdev = netdev_priv(ndev);
+	vdev = ndev->priv;
 	memset(vdev, 0, sizeof(struct vxgedev));
 
 	vdev->ndev = ndev;
@@ -683,7 +683,7 @@ vxge_remove(struct pci_device *pdev)
 		return;
 
 	ndev = hldev->ndev;
-	vdev = netdev_priv(ndev);
+	vdev = ndev->priv;
 
 	iounmap(vdev->bar0);
 
diff --git a/src/include/ipxe/netdevice.h b/src/include/ipxe/netdevice.h
index af932c259..a65dbfd23 100644
--- a/src/include/ipxe/netdevice.h
+++ b/src/include/ipxe/netdevice.h
@@ -568,17 +568,6 @@ netdev_put ( struct net_device *netdev ) {
 	ref_put ( &netdev->refcnt );
 }
 
-/**
- * Get driver private area for this network device
- *
- * @v netdev		Network device
- * @ret priv		Driver private area for this network device
- */
-static inline __attribute__ (( always_inline )) void *
-netdev_priv ( struct net_device *netdev ) {
-        return netdev->priv;
-}
-
 /**
  * Get per-netdevice configuration settings block
  *

From ae4e85bde97c9b216736a5087039f3309a628d24 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 13 Sep 2023 16:29:59 +0100
Subject: [PATCH 004/237] [netdevice] Allocate private data for each network
 upper-layer driver

Allow network upper-layer drivers (such as LLDP, which attaches to
each network device in order to provide a corresponding LLDP settings
block) to specify a size for private data, which will be allocated as
part of the network device structure (as with the existing private
data allocated for the underlying device driver).

This will allow network upper-layer drivers to be simplified by
omitting memory allocation and freeing code.  If the upper-layer
driver requires a reference counter (e.g. for interface
initialisation), then it may use the network device's existing
reference counter, since this is now the reference counter for the
containing block of memory.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/interface/pxe/pxe_call.c     |  3 +-
 src/arch/x86/interface/vmware/guestinfo.c |  8 ++-
 src/core/cachedhcp.c                      |  3 +-
 src/drivers/net/netfront.c                |  4 +-
 src/include/ipxe/netdevice.h              | 13 +++-
 src/interface/efi/efi_snp.c               |  9 ++-
 src/net/fcoe.c                            |  9 ++-
 src/net/infiniband/xsigo.c                |  4 +-
 src/net/ipv6.c                            |  4 +-
 src/net/lldp.c                            |  3 +-
 src/net/neighbour.c                       |  3 +-
 src/net/netdevice.c                       | 72 +++++++++++++++++++----
 src/net/vlan.c                            |  9 ++-
 13 files changed, 110 insertions(+), 34 deletions(-)

diff --git a/src/arch/x86/interface/pxe/pxe_call.c b/src/arch/x86/interface/pxe/pxe_call.c
index 671182991..0e8d5c5a8 100644
--- a/src/arch/x86/interface/pxe/pxe_call.c
+++ b/src/arch/x86/interface/pxe/pxe_call.c
@@ -375,9 +375,10 @@ int pxe_start_nbp ( void ) {
  * Notify BIOS of existence of network device
  *
  * @v netdev		Network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int pxe_notify ( struct net_device *netdev ) {
+static int pxe_notify ( struct net_device *netdev, void *priv __unused ) {
 
 	/* Do nothing if we already have a network device */
 	if ( pxe_netdev )
diff --git a/src/arch/x86/interface/vmware/guestinfo.c b/src/arch/x86/interface/vmware/guestinfo.c
index a0530c8d1..b52c2e87b 100644
--- a/src/arch/x86/interface/vmware/guestinfo.c
+++ b/src/arch/x86/interface/vmware/guestinfo.c
@@ -207,9 +207,11 @@ struct init_fn guestinfo_init_fn __init_fn ( INIT_NORMAL ) = {
  * Create per-netdevice GuestInfo settings
  *
  * @v netdev		Network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int guestinfo_net_probe ( struct net_device *netdev ) {
+static int guestinfo_net_probe ( struct net_device *netdev,
+				 void *priv __unused ) {
 	struct settings *settings;
 	int rc;
 
@@ -247,8 +249,10 @@ static int guestinfo_net_probe ( struct net_device *netdev ) {
  * Remove per-netdevice GuestInfo settings
  *
  * @v netdev		Network device
+ * @v priv		Private data
  */
-static void guestinfo_net_remove ( struct net_device *netdev ) {
+static void guestinfo_net_remove ( struct net_device *netdev,
+				   void *priv __unused ) {
 	struct settings *parent = netdev_settings ( netdev );
 	struct settings *settings;
 
diff --git a/src/core/cachedhcp.c b/src/core/cachedhcp.c
index 60213f02d..57226e165 100644
--- a/src/core/cachedhcp.c
+++ b/src/core/cachedhcp.c
@@ -295,9 +295,10 @@ struct startup_fn cachedhcp_startup_fn __startup_fn ( STARTUP_LATE ) = {
  * Apply cached DHCPACK to network device, if applicable
  *
  * @v netdev		Network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int cachedhcp_probe ( struct net_device *netdev ) {
+static int cachedhcp_probe ( struct net_device *netdev, void *priv __unused ) {
 
 	/* Apply cached DHCPACK to network device, if applicable */
 	return cachedhcp_apply ( &cached_dhcpack, netdev );
diff --git a/src/drivers/net/netfront.c b/src/drivers/net/netfront.c
index 90930a5a3..12713c5b4 100644
--- a/src/drivers/net/netfront.c
+++ b/src/drivers/net/netfront.c
@@ -1056,9 +1056,11 @@ struct xen_driver netfront_driver __xen_driver = {
  * Inhibit emulated PCI devices
  *
  * @v netdev		Network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int netfront_net_probe ( struct net_device *netdev ) {
+static int netfront_net_probe ( struct net_device *netdev,
+				void *priv __unused ) {
 	struct netfront_nic *netfront;
 
 	/* Inhibit emulated PCI devices matching an existing netfront device */
diff --git a/src/include/ipxe/netdevice.h b/src/include/ipxe/netdevice.h
index a65dbfd23..caa83b44b 100644
--- a/src/include/ipxe/netdevice.h
+++ b/src/include/ipxe/netdevice.h
@@ -473,22 +473,27 @@ struct net_device {
 struct net_driver {
 	/** Name */
 	const char *name;
+	/** Size of private data */
+	size_t priv_len;
 	/** Probe device
 	 *
 	 * @v netdev		Network device
+	 * @v priv		Private data
 	 * @ret rc		Return status code
 	 */
-	int ( * probe ) ( struct net_device *netdev );
+	int ( * probe ) ( struct net_device *netdev, void *priv );
 	/** Notify of device or link state change
 	 *
 	 * @v netdev		Network device
+	 * @v priv		Private data
 	 */
-	void ( * notify ) ( struct net_device *netdev );
+	void ( * notify ) ( struct net_device *netdev, void *priv );
 	/** Remove device
 	 *
 	 * @v netdev		Network device
+	 * @v priv		Private data
 	 */
-	void ( * remove ) ( struct net_device *netdev );
+	void ( * remove ) ( struct net_device *netdev, void *priv );
 };
 
 /** Network driver table */
@@ -688,6 +693,8 @@ netdev_rx_frozen ( struct net_device *netdev ) {
 	return ( netdev->state & NETDEV_RX_FROZEN );
 }
 
+extern void * netdev_priv ( struct net_device *netdev,
+			    struct net_driver *driver );
 extern void netdev_rx_freeze ( struct net_device *netdev );
 extern void netdev_rx_unfreeze ( struct net_device *netdev );
 extern void netdev_link_err ( struct net_device *netdev, int rc );
diff --git a/src/interface/efi/efi_snp.c b/src/interface/efi/efi_snp.c
index c4f7d4ea8..8443be997 100644
--- a/src/interface/efi/efi_snp.c
+++ b/src/interface/efi/efi_snp.c
@@ -1777,9 +1777,10 @@ static struct efi_snp_device * efi_snp_demux ( struct net_device *netdev ) {
  * Create SNP device
  *
  * @v netdev		Network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int efi_snp_probe ( struct net_device *netdev ) {
+static int efi_snp_probe ( struct net_device *netdev, void *priv __unused ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	struct efi_device *efidev;
 	struct efi_snp_device *snpdev;
@@ -2017,8 +2018,9 @@ static int efi_snp_probe ( struct net_device *netdev ) {
  * Handle SNP device or link state change
  *
  * @v netdev		Network device
+ * @v priv		Private data
  */
-static void efi_snp_notify ( struct net_device *netdev ) {
+static void efi_snp_notify ( struct net_device *netdev, void *priv __unused ) {
 	struct efi_snp_device *snpdev;
 
 	/* Locate SNP device */
@@ -2042,8 +2044,9 @@ static void efi_snp_notify ( struct net_device *netdev ) {
  * Destroy SNP device
  *
  * @v netdev		Network device
+ * @v priv		Private data
  */
-static void efi_snp_remove ( struct net_device *netdev ) {
+static void efi_snp_remove ( struct net_device *netdev, void *priv __unused ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	struct efi_snp_device *snpdev;
 	int leak = efi_shutdown_in_progress;
diff --git a/src/net/fcoe.c b/src/net/fcoe.c
index f910eeead..70804dd03 100644
--- a/src/net/fcoe.c
+++ b/src/net/fcoe.c
@@ -1110,9 +1110,10 @@ static void fcoe_expired ( struct retry_timer *timer, int over __unused ) {
  * Create FCoE port
  *
  * @v netdev		Network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int fcoe_probe ( struct net_device *netdev ) {
+static int fcoe_probe ( struct net_device *netdev, void *priv __unused ) {
 	struct ll_protocol *ll_protocol = netdev->ll_protocol;
 	struct fcoe_port *fcoe;
 	int rc;
@@ -1162,8 +1163,9 @@ static int fcoe_probe ( struct net_device *netdev ) {
  * Handle FCoE port device or link state change
  *
  * @v netdev		Network device
+ * @v priv		Private data
  */
-static void fcoe_notify ( struct net_device *netdev ) {
+static void fcoe_notify ( struct net_device *netdev, void *priv __unused ) {
 	struct fcoe_port *fcoe;
 
 	/* Sanity check */
@@ -1185,8 +1187,9 @@ static void fcoe_notify ( struct net_device *netdev ) {
  * Destroy FCoE port
  *
  * @v netdev		Network device
+ * @v priv		Private data
  */
-static void fcoe_remove ( struct net_device *netdev ) {
+static void fcoe_remove ( struct net_device *netdev, void *priv __unused ) {
 	struct fcoe_port *fcoe;
 
 	/* Sanity check */
diff --git a/src/net/infiniband/xsigo.c b/src/net/infiniband/xsigo.c
index 4f5c618d7..5e805fa02 100644
--- a/src/net/infiniband/xsigo.c
+++ b/src/net/infiniband/xsigo.c
@@ -1829,8 +1829,10 @@ struct ib_driver xsigo_ib_driver __ib_driver = {
  * Handle device or link status change
  *
  * @v netdev		Network device
+ * @v priv		Private data
  */
-static void xsigo_net_notify ( struct net_device *netdev ) {
+static void xsigo_net_notify ( struct net_device *netdev,
+			       void *priv __unused ) {
 	struct xsigo_device *xdev;
 	struct ib_device *ibdev;
 	struct xsigo_manager *xcm;
diff --git a/src/net/ipv6.c b/src/net/ipv6.c
index ef5e51daa..a0173dfb1 100644
--- a/src/net/ipv6.c
+++ b/src/net/ipv6.c
@@ -1224,9 +1224,11 @@ struct ipv6_settings {
  * Register IPv6 link-local address settings
  *
  * @v netdev		Network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int ipv6_register_settings ( struct net_device *netdev ) {
+static int ipv6_register_settings ( struct net_device *netdev,
+				    void *priv __unused ) {
 	struct settings *parent = netdev_settings ( netdev );
 	struct ipv6_settings *ipv6set;
 	int rc;
diff --git a/src/net/lldp.c b/src/net/lldp.c
index 72e3ecdf6..2ef32cb0c 100644
--- a/src/net/lldp.c
+++ b/src/net/lldp.c
@@ -296,9 +296,10 @@ struct net_protocol lldp_protocol __net_protocol = {
  * Create LLDP settings block
  *
  * @v netdev		Network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int lldp_probe ( struct net_device *netdev ) {
+static int lldp_probe ( struct net_device *netdev, void *priv __unused ) {
 	struct lldp_settings *lldpset;
 	int rc;
 
diff --git a/src/net/neighbour.c b/src/net/neighbour.c
index 7f66d9992..13a8bc3ba 100644
--- a/src/net/neighbour.c
+++ b/src/net/neighbour.c
@@ -383,8 +383,9 @@ int neighbour_define ( struct net_device *netdev,
  * Update neighbour cache on network device state change or removal
  *
  * @v netdev		Network device
+ * @v priv		Private data
  */
-static void neighbour_flush ( struct net_device *netdev ) {
+static void neighbour_flush ( struct net_device *netdev, void *priv __unused ) {
 	struct neighbour *neighbour;
 	struct neighbour *tmp;
 
diff --git a/src/net/netdevice.c b/src/net/netdevice.c
index 915178218..a9ed18134 100644
--- a/src/net/netdevice.c
+++ b/src/net/netdevice.c
@@ -109,6 +109,51 @@ static int netdev_has_ll_addr ( struct net_device *netdev ) {
 	return 0;
 }
 
+/**
+ * Get offset of network device driver private data
+ *
+ * @v driver		Upper-layer driver, or NULL for device driver
+ * @ret offset		Offset of driver private data
+ */
+static size_t netdev_priv_offset ( struct net_driver *driver ) {
+	struct net_device *netdev;
+	unsigned int num_configs;
+	size_t offset;
+
+	/* Allow space for network device */
+	offset = sizeof ( *netdev );
+
+	/* Allow space for configurations */
+	num_configs = table_num_entries ( NET_DEVICE_CONFIGURATORS );
+	offset += ( num_configs * sizeof ( netdev->configs[0] ) );
+
+	/* Place variable-length device driver private data at end */
+	if ( ! driver )
+		driver = table_end ( NET_DRIVERS );
+
+	/* Allow space for preceding upper-layer drivers' private data */
+	for_each_table_entry_continue_reverse ( driver, NET_DRIVERS ) {
+		offset += driver->priv_len;
+	}
+
+	/* Sanity check */
+	assert ( ( offset & ( sizeof ( void * ) - 1 ) ) == 0 );
+
+	return offset;
+}
+
+/**
+ * Get network device driver private data
+ *
+ * @v netdev		Network device
+ * @v driver		Upper-layer driver, or NULL for device driver
+ * @ret priv		Driver private data
+ */
+void * netdev_priv ( struct net_device *netdev, struct net_driver *driver ) {
+
+	return ( ( ( void * ) netdev ) + netdev_priv_offset ( driver ) );
+}
+
 /**
  * Notify drivers of network device or link state change
  *
@@ -116,10 +161,12 @@ static int netdev_has_ll_addr ( struct net_device *netdev ) {
  */
 static void netdev_notify ( struct net_device *netdev ) {
 	struct net_driver *driver;
+	void *priv;
 
 	for_each_table_entry ( driver, NET_DRIVERS ) {
+		priv = netdev_priv ( netdev, driver );
 		if ( driver->notify )
-			driver->notify ( netdev );
+			driver->notify ( netdev, priv );
 	}
 }
 
@@ -675,14 +722,8 @@ struct net_device * alloc_netdev ( size_t priv_len ) {
 	struct net_device *netdev;
 	struct net_device_configurator *configurator;
 	struct net_device_configuration *config;
-	unsigned int num_configs;
-	size_t confs_len;
-	size_t total_len;
 
-	num_configs = table_num_entries ( NET_DEVICE_CONFIGURATORS );
-	confs_len = ( num_configs * sizeof ( netdev->configs[0] ) );
-	total_len = ( sizeof ( *netdev ) + confs_len + priv_len );
-	netdev = zalloc ( total_len );
+	netdev = zalloc ( netdev_priv_offset ( NULL ) + priv_len );
 	if ( netdev ) {
 		ref_init ( &netdev->refcnt, free_netdev );
 		netdev->link_rc = -EUNKNOWN_LINK_STATUS;
@@ -701,8 +742,7 @@ struct net_device * alloc_netdev ( size_t priv_len ) {
 				    &netdev->refcnt );
 			config++;
 		}
-		netdev->priv = ( ( ( void * ) netdev ) + sizeof ( *netdev ) +
-				 confs_len );
+		netdev->priv = netdev_priv ( netdev, NULL );
 	}
 	return netdev;
 }
@@ -722,6 +762,7 @@ int register_netdev ( struct net_device *netdev ) {
 	struct net_device *duplicate;
 	unsigned int i;
 	uint32_t seed;
+	void *priv;
 	int rc;
 
 	/* Set initial link-layer address, if not already set */
@@ -784,7 +825,9 @@ int register_netdev ( struct net_device *netdev ) {
 
 	/* Probe device */
 	for_each_table_entry ( driver, NET_DRIVERS ) {
-		if ( driver->probe && ( rc = driver->probe ( netdev ) ) != 0 ) {
+		priv = netdev_priv ( netdev, driver );
+		if ( driver->probe &&
+		     ( rc = driver->probe ( netdev, priv ) ) != 0 ) {
 			DBGC ( netdev, "NETDEV %s could not add %s device: "
 			       "%s\n", netdev->name, driver->name,
 			       strerror ( rc ) );
@@ -796,8 +839,9 @@ int register_netdev ( struct net_device *netdev ) {
 
  err_probe:
 	for_each_table_entry_continue_reverse ( driver, NET_DRIVERS ) {
+		priv = netdev_priv ( netdev, driver );
 		if ( driver->remove )
-			driver->remove ( netdev );
+			driver->remove ( netdev, priv );
 	}
 	clear_settings ( netdev_settings ( netdev ) );
 	unregister_settings ( netdev_settings ( netdev ) );
@@ -896,14 +940,16 @@ void netdev_close ( struct net_device *netdev ) {
  */
 void unregister_netdev ( struct net_device *netdev ) {
 	struct net_driver *driver;
+	void *priv;
 
 	/* Ensure device is closed */
 	netdev_close ( netdev );
 
 	/* Remove device */
 	for_each_table_entry_reverse ( driver, NET_DRIVERS ) {
+		priv = netdev_priv ( netdev, driver );
 		if ( driver->remove )
-			driver->remove ( netdev );
+			driver->remove ( netdev, priv );
 	}
 
 	/* Unregister per-netdev configuration settings */
diff --git a/src/net/vlan.c b/src/net/vlan.c
index d73a95711..c61bb850e 100644
--- a/src/net/vlan.c
+++ b/src/net/vlan.c
@@ -470,9 +470,10 @@ void vlan_auto ( const void *ll_addr, unsigned int tag ) {
  * Create automatic VLAN device
  *
  * @v trunk		Trunk network device
+ * @v priv		Private data
  * @ret rc		Return status code
  */
-static int vlan_probe ( struct net_device *trunk ) {
+static int vlan_probe ( struct net_device *trunk, void *priv __unused ) {
 	int rc;
 
 	/* Do nothing unless an automatic VLAN exists */
@@ -498,8 +499,9 @@ static int vlan_probe ( struct net_device *trunk ) {
  * Handle trunk network device link state change
  *
  * @v trunk		Trunk network device
+ * @v priv		Private data
  */
-static void vlan_notify ( struct net_device *trunk ) {
+static void vlan_notify ( struct net_device *trunk, void *priv __unused ) {
 	struct net_device *netdev;
 	struct vlan_device *vlan;
 
@@ -538,8 +540,9 @@ static int vlan_remove_first ( struct net_device *trunk ) {
  * Destroy all VLAN devices for a given trunk
  *
  * @v trunk		Trunk network device
+ * @v priv		Private data
  */
-static void vlan_remove ( struct net_device *trunk ) {
+static void vlan_remove ( struct net_device *trunk, void *priv __unused ) {
 
 	/* Remove all VLAN devices attached to this trunk, safe
 	 * against arbitrary net device removal.

From cc1e27e525201f5ee7fcc098f47f04ec26814289 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 13 Sep 2023 20:23:59 +0100
Subject: [PATCH 005/237] [lldp] Use driver-private data to hold LLDP settings
 block

Simplify the LLDP code by using driver-private data to hold the LLDP
settings block, instead of using a separate allocation.  This avoids
the need to maintain a list of LLDP settings blocks (since the LLDP
settings block pointer can always be obtained using netdev_priv()) and
obviates several failure paths.

Any recorded LLDP data is now freed when the network device is
unregistered, since there is no longer a dedicated reference counter
for the LLDP settings block.  To minimise surprise, we also now
explicitly unregister the settings block.  This is not strictly
necessary (since the block will be automatically unregistered when the
parent network device settings block is unregistered), but it
maintains symmetry between lldp_probe() and lldp_remove().

The overall reduction in the size of the LLDP code is around 15%.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/lldp.c | 94 +++++++++++++++++---------------------------------
 1 file changed, 31 insertions(+), 63 deletions(-)

diff --git a/src/net/lldp.c b/src/net/lldp.c
index 2ef32cb0c..a854d0ace 100644
--- a/src/net/lldp.c
+++ b/src/net/lldp.c
@@ -40,12 +40,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** An LLDP settings block */
 struct lldp_settings {
-	/** Reference counter */
-	struct refcnt refcnt;
 	/** Settings interface */
 	struct settings settings;
-	/** List of LLDP settings blocks */
-	struct list_head list;
 	/** Name */
 	const char *name;
 	/** LLDP data */
@@ -54,45 +50,12 @@ struct lldp_settings {
 	size_t len;
 };
 
+/* Forward declaration */
+struct net_driver lldp_driver __net_driver;
+
 /** LLDP settings scope */
 static const struct settings_scope lldp_settings_scope;
 
-/** List of LLDP settings blocks */
-static LIST_HEAD ( lldp_settings );
-
-/**
- * Free LLDP settings block
- *
- * @v refcnt		Reference counter
- */
-static void lldp_free ( struct refcnt *refcnt ) {
-	struct lldp_settings *lldpset =
-		container_of ( refcnt, struct lldp_settings, refcnt );
-
-	DBGC ( lldpset, "LLDP %s freed\n", lldpset->name );
-	list_del ( &lldpset->list );
-	free ( lldpset->data );
-	free ( lldpset );
-}
-
-/**
- * Find LLDP settings block
- *
- * @v netdev		Network device
- * @ret lldpset		LLDP settings block
- */
-static struct lldp_settings * lldp_find ( struct net_device *netdev ) {
-	struct lldp_settings *lldpset;
-
-	/* Find matching LLDP settings block */
-	list_for_each_entry ( lldpset, &lldp_settings, list ) {
-		if ( netdev_settings ( netdev ) == lldpset->settings.parent )
-			return lldpset;
-	}
-
-	return NULL;
-}
-
 /**
  * Check applicability of LLDP setting
  *
@@ -246,13 +209,7 @@ static int lldp_rx ( struct io_buffer *iobuf, struct net_device *netdev,
 	int rc;
 
 	/* Find matching LLDP settings block */
-	lldpset = lldp_find ( netdev );
-	if ( ! lldpset ) {
-		DBGC ( netdev, "LLDP %s has no \"%s\" settings block\n",
-		       netdev->name, LLDP_SETTINGS_NAME );
-		rc = -ENOENT;
-		goto err_find;
-	}
+	lldpset = netdev_priv ( netdev, &lldp_driver );
 
 	/* Create trimmed copy of received LLDP data */
 	len = iob_len ( iobuf );
@@ -280,7 +237,6 @@ static int lldp_rx ( struct io_buffer *iobuf, struct net_device *netdev,
 
 	free ( data );
  err_alloc:
- err_find:
 	free_iob ( iobuf );
 	return rc;
 }
@@ -299,24 +255,18 @@ struct net_protocol lldp_protocol __net_protocol = {
  * @v priv		Private data
  * @ret rc		Return status code
  */
-static int lldp_probe ( struct net_device *netdev, void *priv __unused ) {
-	struct lldp_settings *lldpset;
+static int lldp_probe ( struct net_device *netdev, void *priv ) {
+	struct lldp_settings *lldpset = priv;
 	int rc;
 
-	/* Allocate LLDP settings block */
-	lldpset = zalloc ( sizeof ( *lldpset ) );
-	if ( ! lldpset ) {
-		rc = -ENOMEM;
-		goto err_alloc;
-	}
-	ref_init ( &lldpset->refcnt, lldp_free );
+	/* Initialise LLDP settings block */
 	settings_init ( &lldpset->settings, &lldp_settings_operations,
-			&lldpset->refcnt, &lldp_settings_scope );
-	list_add_tail ( &lldpset->list, &lldp_settings );
+			&netdev->refcnt, &lldp_settings_scope );
 	lldpset->name = netdev->name;
 
 	/* Register settings */
-	if ( ( rc = register_settings ( &lldpset->settings, netdev_settings ( netdev ),
+	if ( ( rc = register_settings ( &lldpset->settings,
+					netdev_settings ( netdev ),
 					LLDP_SETTINGS_NAME ) ) != 0 ) {
 		DBGC ( lldpset, "LLDP %s could not register settings: %s\n",
 		       lldpset->name, strerror ( rc ) );
@@ -324,18 +274,36 @@ static int lldp_probe ( struct net_device *netdev, void *priv __unused ) {
 	}
 	DBGC ( lldpset, "LLDP %s registered\n", lldpset->name );
 
-	ref_put ( &lldpset->refcnt );
 	return 0;
 
 	unregister_settings ( &lldpset->settings );
  err_register:
-	ref_put ( &lldpset->refcnt );
- err_alloc:
+	assert ( lldpset->data == NULL );
 	return rc;
 }
 
+/**
+ * Remove LLDP settings block
+ *
+ * @v netdev		Network device
+ * @v priv		Private data
+ */
+static void lldp_remove ( struct net_device *netdev __unused, void *priv ) {
+	struct lldp_settings *lldpset = priv;
+
+	/* Unregister settings */
+	unregister_settings ( &lldpset->settings );
+	DBGC ( lldpset, "LLDP %s unregistered\n", lldpset->name );
+
+	/* Free any LLDP data */
+	free ( lldpset->data );
+	lldpset->data = NULL;
+}
+
 /** LLDP driver */
 struct net_driver lldp_driver __net_driver = {
 	.name = "LLDP",
+	.priv_len = sizeof ( struct lldp_settings ),
 	.probe = lldp_probe,
+	.remove = lldp_remove,
 };

From 8b1d34badf321668f830c78e6803d718446b57ef Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 13 Sep 2023 22:21:08 +0100
Subject: [PATCH 006/237] [ipv6] Use driver-private data to hold link-local
 IPv6 settings block

Simplify the IPv6 link-local settings code by using driver-private
data to hold the settings block, instead of using a separate
allocation.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/ipv6.c | 39 ++++++++++-----------------------------
 1 file changed, 10 insertions(+), 29 deletions(-)

diff --git a/src/net/ipv6.c b/src/net/ipv6.c
index a0173dfb1..8ee0804d3 100644
--- a/src/net/ipv6.c
+++ b/src/net/ipv6.c
@@ -1212,14 +1212,6 @@ static struct settings_operations ipv6_settings_operations = {
 	.fetch = ipv6_fetch,
 };
 
-/** IPv6 link-local address settings */
-struct ipv6_settings {
-	/** Reference counter */
-	struct refcnt refcnt;
-	/** Settings interface */
-	struct settings settings;
-};
-
 /**
  * Register IPv6 link-local address settings
  *
@@ -1227,37 +1219,26 @@ struct ipv6_settings {
  * @v priv		Private data
  * @ret rc		Return status code
  */
-static int ipv6_register_settings ( struct net_device *netdev,
-				    void *priv __unused ) {
+static int ipv6_register_settings ( struct net_device *netdev, void *priv ) {
 	struct settings *parent = netdev_settings ( netdev );
-	struct ipv6_settings *ipv6set;
+	struct settings *settings = priv;
 	int rc;
 
-	/* Allocate and initialise structure */
-	ipv6set = zalloc ( sizeof ( *ipv6set ) );
-	if ( ! ipv6set ) {
-		rc = -ENOMEM;
-		goto err_alloc;
-	}
-	ref_init ( &ipv6set->refcnt, NULL );
-	settings_init ( &ipv6set->settings, &ipv6_settings_operations,
-			&ipv6set->refcnt, &ipv6_settings_scope );
-	ipv6set->settings.order = IPV6_ORDER_LINK_LOCAL;
-
-	/* Register settings */
-	if ( ( rc = register_settings ( &ipv6set->settings, parent,
+	/* Initialise and register settings */
+	settings_init ( settings, &ipv6_settings_operations,
+			&netdev->refcnt, &ipv6_settings_scope );
+	settings->order = IPV6_ORDER_LINK_LOCAL;
+	if ( ( rc = register_settings ( settings, parent,
 					IPV6_SETTINGS_NAME ) ) != 0 )
-		goto err_register;
+		return rc;
 
- err_register:
-	ref_put ( &ipv6set->refcnt );
- err_alloc:
-	return rc;
+	return 0;
 }
 
 /** IPv6 network device driver */
 struct net_driver ipv6_driver __net_driver = {
 	.name = "IPv6",
+	.priv_len = sizeof ( struct settings ),
 	.probe = ipv6_register_settings,
 };
 

From 8cbf248198f3bc66c52b2340b4decf293af8af47 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 13 Sep 2023 22:43:24 +0100
Subject: [PATCH 007/237] [vmware] Use driver-private data to hold GuestInfo
 settings block

Simplify the per-netdevice GuestInfo settings code by using
driver-private data to hold the settings block, instead of using a
separate allocation.

The settings block (if existent) will be automatically unregistered
when the parent network device settings block is unregistered, and no
longer needs to be separately freed.  The guestinfo_net_remove()
function may therefore be omitted completely.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/interface/vmware/guestinfo.c | 48 ++++-------------------
 1 file changed, 7 insertions(+), 41 deletions(-)

diff --git a/src/arch/x86/interface/vmware/guestinfo.c b/src/arch/x86/interface/vmware/guestinfo.c
index b52c2e87b..4134515c1 100644
--- a/src/arch/x86/interface/vmware/guestinfo.c
+++ b/src/arch/x86/interface/vmware/guestinfo.c
@@ -210,66 +210,32 @@ struct init_fn guestinfo_init_fn __init_fn ( INIT_NORMAL ) = {
  * @v priv		Private data
  * @ret rc		Return status code
  */
-static int guestinfo_net_probe ( struct net_device *netdev,
-				 void *priv __unused ) {
-	struct settings *settings;
+static int guestinfo_net_probe ( struct net_device *netdev, void *priv ) {
+	struct settings *settings = priv;
 	int rc;
 
 	/* Do nothing unless we have a GuestInfo channel available */
 	if ( guestinfo_channel < 0 )
 		return 0;
 
-	/* Allocate and initialise settings block */
-	settings = zalloc ( sizeof ( *settings ) );
-	if ( ! settings ) {
-		rc = -ENOMEM;
-		goto err_alloc;
-	}
-	settings_init ( settings, &guestinfo_settings_operations, NULL, NULL );
-
-	/* Register settings */
+	/* Initialise and register settings */
+	settings_init ( settings, &guestinfo_settings_operations,
+			&netdev->refcnt, NULL );
 	if ( ( rc = register_settings ( settings, netdev_settings ( netdev ),
 					"vmware" ) ) != 0 ) {
 		DBGC ( settings, "GuestInfo %p could not register for %s: %s\n",
 		       settings, netdev->name, strerror ( rc ) );
-		goto err_register;
+		return rc;
 	}
 	DBGC ( settings, "GuestInfo %p registered for %s\n",
 	       settings, netdev->name );
 
 	return 0;
-
- err_register:
-	free ( settings );
- err_alloc:
-	return rc;
-}
-
-/**
- * Remove per-netdevice GuestInfo settings
- *
- * @v netdev		Network device
- * @v priv		Private data
- */
-static void guestinfo_net_remove ( struct net_device *netdev,
-				   void *priv __unused ) {
-	struct settings *parent = netdev_settings ( netdev );
-	struct settings *settings;
-
-	list_for_each_entry ( settings, &parent->children, siblings ) {
-		if ( settings->op == &guestinfo_settings_operations ) {
-			DBGC ( settings, "GuestInfo %p unregistered for %s\n",
-			       settings, netdev->name );
-			unregister_settings ( settings );
-			free ( settings );
-			return;
-		}
-	}
 }
 
 /** GuestInfo per-netdevice driver */
 struct net_driver guestinfo_net_driver __net_driver = {
 	.name = "GuestInfo",
+	.priv_len = sizeof ( struct settings ),
 	.probe = guestinfo_net_probe,
-	.remove = guestinfo_net_remove,
 };

From cac3a584dc8acea1522669f1ed16e0979fb92252 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 13 Sep 2023 23:00:57 +0100
Subject: [PATCH 008/237] [fcoe] Use driver-private data to hold FCoE port
 structure

Simplify the FCoE code by using driver-private data to hold the FCoE
port for each network device, instead of using a separate allocation.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/fcoe.c | 85 +++++++++++++-------------------------------------
 1 file changed, 21 insertions(+), 64 deletions(-)

diff --git a/src/net/fcoe.c b/src/net/fcoe.c
index 70804dd03..9f3ddf88b 100644
--- a/src/net/fcoe.c
+++ b/src/net/fcoe.c
@@ -69,10 +69,6 @@ FEATURE ( FEATURE_PROTOCOL, "FCoE", DHCP_EB_FEATURE_FCOE, 1 );
 
 /** An FCoE port */
 struct fcoe_port {
-	/** Reference count */
-	struct refcnt refcnt;
-	/** List of FCoE ports */
-	struct list_head list;
 	/** Transport interface */
 	struct interface transport;
 	/** Network device */
@@ -115,6 +111,7 @@ enum fcoe_flags {
 	FCOE_VLAN_TIMED_OUT = 0x0020,
 };
 
+struct net_driver fcoe_driver __net_driver;
 struct net_protocol fcoe_protocol __net_protocol;
 struct net_protocol fip_protocol __net_protocol;
 
@@ -152,9 +149,6 @@ static uint8_t default_fcf_mac[ETH_ALEN] =
 /** Maximum number of missing discovery advertisements */
 #define FCOE_MAX_FIP_MISSING_KEEPALIVES 4
 
-/** List of FCoE ports */
-static LIST_HEAD ( fcoe_ports );
-
 /******************************************************************************
  *
  * FCoE protocol
@@ -162,22 +156,6 @@ static LIST_HEAD ( fcoe_ports );
  ******************************************************************************
  */
 
-/**
- * Identify FCoE port by network device
- *
- * @v netdev		Network device
- * @ret fcoe		FCoE port, or NULL
- */
-static struct fcoe_port * fcoe_demux ( struct net_device *netdev ) {
-	struct fcoe_port *fcoe;
-
-	list_for_each_entry ( fcoe, &fcoe_ports, list ) {
-		if ( fcoe->netdev == netdev )
-			return fcoe;
-	}
-	return NULL;
-}
-
 /**
  * Reset FCoE port
  *
@@ -348,7 +326,8 @@ static int fcoe_rx ( struct io_buffer *iobuf, struct net_device *netdev,
 	int rc;
 
 	/* Identify FCoE port */
-	if ( ( fcoe = fcoe_demux ( netdev ) ) == NULL ) {
+	fcoe = netdev_priv ( netdev, &fcoe_driver );
+	if ( ! fcoe->netdev ) {
 		DBG ( "FCoE received frame for net device %s missing FCoE "
 		      "port\n", netdev->name );
 		rc = -ENOTCONN;
@@ -448,9 +427,6 @@ static void fcoe_close ( struct fcoe_port *fcoe, int rc ) {
 
 	stop_timer ( &fcoe->timer );
 	intf_shutdown ( &fcoe->transport, rc );
-	netdev_put ( fcoe->netdev );
-	list_del ( &fcoe->list );
-	ref_put ( &fcoe->refcnt );
 }
 
 /**
@@ -947,7 +923,8 @@ static int fcoe_fip_rx ( struct io_buffer *iobuf,
 	int rc;
 
 	/* Identify FCoE port */
-	if ( ( fcoe = fcoe_demux ( netdev ) ) == NULL ) {
+	fcoe = netdev_priv ( netdev, &fcoe_driver );
+	if ( ! fcoe->netdev ) {
 		DBG ( "FCoE received FIP frame for net device %s missing FCoE "
 		      "port\n", netdev->name );
 		rc = -ENOTCONN;
@@ -1113,29 +1090,21 @@ static void fcoe_expired ( struct retry_timer *timer, int over __unused ) {
  * @v priv		Private data
  * @ret rc		Return status code
  */
-static int fcoe_probe ( struct net_device *netdev, void *priv __unused ) {
+static int fcoe_probe ( struct net_device *netdev, void *priv ) {
 	struct ll_protocol *ll_protocol = netdev->ll_protocol;
-	struct fcoe_port *fcoe;
-	int rc;
+	struct fcoe_port *fcoe = priv;
 
 	/* Sanity check */
 	if ( ll_protocol->ll_proto != htons ( ARPHRD_ETHER ) ) {
 		/* Not an error; simply skip this net device */
 		DBG ( "FCoE skipping non-Ethernet device %s\n", netdev->name );
-		rc = 0;
-		goto err_non_ethernet;
+		return 0;
 	}
 
-	/* Allocate and initialise structure */
-	fcoe = zalloc ( sizeof ( *fcoe ) );
-	if ( ! fcoe ) {
-		rc = -ENOMEM;
-		goto err_zalloc;
-	}
-	ref_init ( &fcoe->refcnt, NULL );
-	intf_init ( &fcoe->transport, &fcoe_transport_desc, &fcoe->refcnt );
-	timer_init ( &fcoe->timer, fcoe_expired, &fcoe->refcnt );
-	fcoe->netdev = netdev_get ( netdev );
+	/* Initialise structure */
+	intf_init ( &fcoe->transport, &fcoe_transport_desc, &netdev->refcnt );
+	timer_init ( &fcoe->timer, fcoe_expired, &netdev->refcnt );
+	fcoe->netdev = netdev;
 
 	/* Construct node and port names */
 	fcoe->node_wwn.fcoe.authority = htons ( FCOE_AUTHORITY_IEEE );
@@ -1149,14 +1118,7 @@ static int fcoe_probe ( struct net_device *netdev, void *priv __unused ) {
 	       fc_ntoa ( &fcoe->node_wwn.fc ) );
 	DBGC ( fcoe, " port %s\n", fc_ntoa ( &fcoe->port_wwn.fc ) );
 
-	/* Transfer reference to port list */
-	list_add ( &fcoe->list, &fcoe_ports );
 	return 0;
-
-	netdev_put ( fcoe->netdev );
- err_zalloc:
- err_non_ethernet:
-	return rc;
 }
 
 /**
@@ -1165,15 +1127,12 @@ static int fcoe_probe ( struct net_device *netdev, void *priv __unused ) {
  * @v netdev		Network device
  * @v priv		Private data
  */
-static void fcoe_notify ( struct net_device *netdev, void *priv __unused ) {
-	struct fcoe_port *fcoe;
+static void fcoe_notify ( struct net_device *netdev, void *priv ) {
+	struct fcoe_port *fcoe = priv;
 
-	/* Sanity check */
-	if ( ( fcoe = fcoe_demux ( netdev ) ) == NULL ) {
-		DBG ( "FCoE notification for net device %s missing FCoE "
-		      "port\n", netdev->name );
+	/* Skip non-FCoE net devices */
+	if ( ! fcoe->netdev )
 		return;
-	}
 
 	/* Reset the FCoE link if necessary */
 	if ( ! ( netdev_is_open ( netdev ) &&
@@ -1189,15 +1148,12 @@ static void fcoe_notify ( struct net_device *netdev, void *priv __unused ) {
  * @v netdev		Network device
  * @v priv		Private data
  */
-static void fcoe_remove ( struct net_device *netdev, void *priv __unused ) {
-	struct fcoe_port *fcoe;
+static void fcoe_remove ( struct net_device *netdev __unused, void *priv ) {
+	struct fcoe_port *fcoe = priv;
 
-	/* Sanity check */
-	if ( ( fcoe = fcoe_demux ( netdev ) ) == NULL ) {
-		DBG ( "FCoE removal of net device %s missing FCoE port\n",
-		      netdev->name );
+	/* Skip non-FCoE net devices */
+	if ( ! fcoe->netdev )
 		return;
-	}
 
 	/* Close FCoE device */
 	fcoe_close ( fcoe, 0 );
@@ -1206,6 +1162,7 @@ static void fcoe_remove ( struct net_device *netdev, void *priv __unused ) {
 /** FCoE driver */
 struct net_driver fcoe_driver __net_driver = {
 	.name = "FCoE",
+	.priv_len = sizeof ( struct fcoe_port ),
 	.probe = fcoe_probe,
 	.notify = fcoe_notify,
 	.remove = fcoe_remove,

From 56cc61a168820c7cbbe23418388129ec11699a8c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 15 Sep 2023 16:10:07 +0100
Subject: [PATCH 009/237] [eap] Define a supplicant model for EAP and EAPoL

Extend the EAP model to include a record of whether or not EAP
authentication has completed (successfully or otherwise), and to
provide a method for transmitting EAP responses.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/eap.h   |  21 ++++++-
 src/include/ipxe/eapol.h |  13 ++++-
 src/net/80211/wpa.c      |  11 ++--
 src/net/eap.c            |  43 +++++++++-----
 src/net/eapol.c          | 123 +++++++++++++++++++++++++++++++++++++--
 5 files changed, 184 insertions(+), 27 deletions(-)

diff --git a/src/include/ipxe/eap.h b/src/include/ipxe/eap.h
index 6fe70189b..e5f606553 100644
--- a/src/include/ipxe/eap.h
+++ b/src/include/ipxe/eap.h
@@ -64,6 +64,25 @@ union eap_packet {
  */
 #define EAP_BLOCK_TIMEOUT ( 45 * TICKS_PER_SEC )
 
-extern int eap_rx ( struct net_device *netdev, const void *data, size_t len );
+/** An EAP supplicant */
+struct eap_supplicant {
+	/** Network device */
+	struct net_device *netdev;
+	/** Authentication outcome is final */
+	int done;
+	/**
+	 * Transmit EAP response
+	 *
+	 * @v supplicant	EAP supplicant
+	 * @v data		Response data
+	 * @v len		Length of response data
+	 * @ret rc		Return status code
+	 */
+	int ( * tx ) ( struct eap_supplicant *supplicant,
+		       const void *data, size_t len );
+};
+
+extern int eap_rx ( struct eap_supplicant *supplicant,
+		    const void *data, size_t len );
 
 #endif /* _IPXE_EAP_H */
diff --git a/src/include/ipxe/eapol.h b/src/include/ipxe/eapol.h
index 952d6c752..f6009a2ff 100644
--- a/src/include/ipxe/eapol.h
+++ b/src/include/ipxe/eapol.h
@@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <stdint.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/tables.h>
+#include <ipxe/eap.h>
 
 /** EAPoL header */
 struct eapol_header {
@@ -32,6 +33,12 @@ struct eapol_header {
 /** EAPoL key */
 #define EAPOL_TYPE_KEY 5
 
+/** An EAPoL supplicant */
+struct eapol_supplicant {
+	/** EAP supplicant */
+	struct eap_supplicant eap;
+};
+
 /** An EAPoL handler */
 struct eapol_handler {
 	/** Type */
@@ -39,15 +46,15 @@ struct eapol_handler {
 	/**
 	 * Process received packet
 	 *
+	 * @v supplicant	EAPoL supplicant
 	 * @v iobuf		I/O buffer
-	 * @v netdev		Network device
 	 * @v ll_source		Link-layer source address
 	 * @ret rc		Return status code
 	 *
 	 * This method takes ownership of the I/O buffer.
 	 */
-	int ( * rx ) ( struct io_buffer *iobuf, struct net_device *netdev,
-		       const void *ll_source );
+	int ( * rx ) ( struct eapol_supplicant *supplicant,
+		       struct io_buffer *iobuf, const void *ll_source );
 };
 
 /** EAPoL handler table */
diff --git a/src/net/80211/wpa.c b/src/net/80211/wpa.c
index 1484d0e80..17c11b8ed 100644
--- a/src/net/80211/wpa.c
+++ b/src/net/80211/wpa.c
@@ -761,13 +761,14 @@ static int wpa_handle_1_of_2 ( struct wpa_common_ctx *ctx,
 /**
  * Handle receipt of EAPOL-Key frame for WPA
  *
- * @v iob	I/O buffer
- * @v netdev	Network device
- * @v ll_source	Source link-layer address
+ * @v supplicant	EAPoL supplicant
+ * @v iob		I/O buffer
+ * @v ll_source		Source link-layer address
  */
-static int eapol_key_rx ( struct io_buffer *iob, struct net_device *netdev,
-			  const void *ll_source )
+static int eapol_key_rx ( struct eapol_supplicant *supplicant,
+			  struct io_buffer *iob, const void *ll_source )
 {
+	struct net_device *netdev = supplicant->eap.netdev;
 	struct net80211_device *dev = net80211_get ( netdev );
 	struct eapol_header *eapol;
 	struct eapol_key_pkt *pkt;
diff --git a/src/net/eap.c b/src/net/eap.c
index 8d1d540fb..beaeb61d2 100644
--- a/src/net/eap.c
+++ b/src/net/eap.c
@@ -36,10 +36,11 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /**
  * Handle EAP Request-Identity
  *
- * @v netdev		Network device
+ * @v supplicant	EAP supplicant
  * @ret rc		Return status code
  */
-static int eap_rx_request_identity ( struct net_device *netdev ) {
+static int eap_rx_request_identity ( struct eap_supplicant *supplicant ) {
+	struct net_device *netdev = supplicant->netdev;
 
 	/* Treat Request-Identity as blocking the link */
 	DBGC ( netdev, "EAP %s Request-Identity blocking link\n",
@@ -52,13 +53,14 @@ static int eap_rx_request_identity ( struct net_device *netdev ) {
 /**
  * Handle EAP Request
  *
- * @v netdev		Network device
+ * @v supplicant	EAP supplicant
  * @v req		EAP request
  * @v len		Length of EAP request
  * @ret rc		Return status code
  */
-static int eap_rx_request ( struct net_device *netdev,
+static int eap_rx_request ( struct eap_supplicant *supplicant,
 			    const struct eap_request *req, size_t len ) {
+	struct net_device *netdev = supplicant->netdev;
 
 	/* Sanity check */
 	if ( len < sizeof ( *req ) ) {
@@ -67,10 +69,13 @@ static int eap_rx_request ( struct net_device *netdev,
 		return -EINVAL;
 	}
 
+	/* Mark authentication as incomplete */
+	supplicant->done = 0;
+
 	/* Handle according to type */
 	switch ( req->type ) {
 	case EAP_TYPE_IDENTITY:
-		return eap_rx_request_identity ( netdev );
+		return eap_rx_request_identity ( supplicant );
 	default:
 		DBGC ( netdev, "EAP %s requested type %d unknown:\n",
 		       netdev->name, req->type );
@@ -82,10 +87,14 @@ static int eap_rx_request ( struct net_device *netdev,
 /**
  * Handle EAP Success
  *
- * @v netdev		Network device
+ * @v supplicant	EAP supplicant
  * @ret rc		Return status code
  */
-static int eap_rx_success ( struct net_device *netdev ) {
+static int eap_rx_success ( struct eap_supplicant *supplicant ) {
+	struct net_device *netdev = supplicant->netdev;
+
+	/* Mark authentication as complete */
+	supplicant->done = 1;
 
 	/* Mark link as unblocked */
 	DBGC ( netdev, "EAP %s Success\n", netdev->name );
@@ -97,10 +106,14 @@ static int eap_rx_success ( struct net_device *netdev ) {
 /**
  * Handle EAP Failure
  *
- * @v netdev		Network device
+ * @v supplicant	EAP supplicant
  * @ret rc		Return status code
  */
-static int eap_rx_failure ( struct net_device *netdev ) {
+static int eap_rx_failure ( struct eap_supplicant *supplicant ) {
+	struct net_device *netdev = supplicant->netdev;
+
+	/* Mark authentication as complete */
+	supplicant->done = 1;
 
 	/* Record error */
 	DBGC ( netdev, "EAP %s Failure\n", netdev->name );
@@ -110,12 +123,14 @@ static int eap_rx_failure ( struct net_device *netdev ) {
 /**
  * Handle EAP packet
  *
- * @v netdev		Network device
+ * @v supplicant	EAP supplicant
  * @v data		EAP packet
  * @v len		Length of EAP packet
  * @ret rc		Return status code
  */
-int eap_rx ( struct net_device *netdev, const void *data, size_t len ) {
+int eap_rx ( struct eap_supplicant *supplicant, const void *data,
+	     size_t len ) {
+	struct net_device *netdev = supplicant->netdev;
 	const union eap_packet *eap = data;
 
 	/* Sanity check */
@@ -128,11 +143,11 @@ int eap_rx ( struct net_device *netdev, const void *data, size_t len ) {
 	/* Handle according to code */
 	switch ( eap->hdr.code ) {
 	case EAP_CODE_REQUEST:
-		return eap_rx_request ( netdev, &eap->req, len );
+		return eap_rx_request ( supplicant, &eap->req, len );
 	case EAP_CODE_SUCCESS:
-		return eap_rx_success ( netdev );
+		return eap_rx_success ( supplicant );
 	case EAP_CODE_FAILURE:
-		return eap_rx_failure ( netdev );
+		return eap_rx_failure ( supplicant );
 	default:
 		DBGC ( netdev, "EAP %s unsupported code %d\n",
 		       netdev->name, eap->hdr.code );
diff --git a/src/net/eapol.c b/src/net/eapol.c
index 3578f0e37..172037ce1 100644
--- a/src/net/eapol.c
+++ b/src/net/eapol.c
@@ -28,7 +28,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <byteswap.h>
 #include <ipxe/iobuf.h>
 #include <ipxe/if_ether.h>
+#include <ipxe/if_arp.h>
 #include <ipxe/netdevice.h>
+#include <ipxe/vlan.h>
 #include <ipxe/eap.h>
 #include <ipxe/eapol.h>
 
@@ -38,6 +40,13 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
+struct net_driver eapol_driver __net_driver;
+
+/** EAPoL destination MAC address */
+static const uint8_t eapol_mac[ETH_ALEN] = {
+	0x01, 0x80, 0xc2, 0x00, 0x00, 0x03
+};
+
 /**
  * Process EAPoL packet
  *
@@ -51,12 +60,25 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 static int eapol_rx ( struct io_buffer *iobuf, struct net_device *netdev,
 		      const void *ll_dest __unused, const void *ll_source,
 		      unsigned int flags __unused ) {
+	struct eapol_supplicant *supplicant;
 	struct eapol_header *eapol;
 	struct eapol_handler *handler;
 	size_t remaining;
 	size_t len;
 	int rc;
 
+	/* Find matching supplicant */
+	supplicant = netdev_priv ( netdev, &eapol_driver );
+
+	/* Ignore non-EAPoL devices */
+	if ( ! supplicant->eap.netdev ) {
+		DBGC ( netdev, "EAPOL %s is not an EAPoL device\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -ENOTTY;
+		goto drop;
+	}
+
 	/* Sanity checks */
 	if ( iob_len ( iobuf ) < sizeof ( *eapol ) ) {
 		DBGC ( netdev, "EAPOL %s underlength header:\n",
@@ -83,7 +105,7 @@ static int eapol_rx ( struct io_buffer *iobuf, struct net_device *netdev,
 	/* Handle according to type */
 	for_each_table_entry ( handler, EAPOL_HANDLERS ) {
 		if ( handler->type == eapol->type ) {
-			return handler->rx ( iob_disown ( iobuf ) , netdev,
+			return handler->rx ( supplicant, iob_disown ( iobuf ),
 					     ll_source );
 		}
 	}
@@ -107,12 +129,14 @@ struct net_protocol eapol_protocol __net_protocol = {
 /**
  * Process EAPoL-encapsulated EAP packet
  *
- * @v netdev		Network device
+ * @v supplicant	EAPoL supplicant
  * @v ll_source		Link-layer source address
  * @ret rc		Return status code
  */
-static int eapol_eap_rx ( struct io_buffer *iobuf, struct net_device *netdev,
+static int eapol_eap_rx ( struct eapol_supplicant *supplicant,
+			  struct io_buffer *iobuf,
 			  const void *ll_source __unused ) {
+	struct net_device *netdev = supplicant->eap.netdev;
 	struct eapol_header *eapol;
 	int rc;
 
@@ -123,7 +147,8 @@ static int eapol_eap_rx ( struct io_buffer *iobuf, struct net_device *netdev,
 	eapol = iob_pull ( iobuf, sizeof ( *eapol ) );
 
 	/* Process EAP packet */
-	if ( ( rc = eap_rx ( netdev, iobuf->data, iob_len ( iobuf ) ) ) != 0 ) {
+	if ( ( rc = eap_rx ( &supplicant->eap, iobuf->data,
+			     iob_len ( iobuf ) ) ) != 0 ) {
 		DBGC ( netdev, "EAPOL %s v%d EAP failed: %s\n",
 		       netdev->name, eapol->version, strerror ( rc ) );
 		goto drop;
@@ -139,3 +164,93 @@ struct eapol_handler eapol_eap __eapol_handler = {
 	.type = EAPOL_TYPE_EAP,
 	.rx = eapol_eap_rx,
 };
+
+/**
+ * Transmit EAPoL packet
+ *
+ * @v supplicant	EAPoL supplicant
+ * @v type		Packet type
+ * @v data		Packet body
+ * @v len		Length of packet body
+ * @ret rc		Return status code
+ */
+static int eapol_tx ( struct eapol_supplicant *supplicant, unsigned int type,
+		      const void *data, size_t len ) {
+	struct net_device *netdev = supplicant->eap.netdev;
+	struct io_buffer *iobuf;
+	struct eapol_header *eapol;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = alloc_iob ( MAX_LL_HEADER_LEN + sizeof ( *eapol ) + len );
+	if ( ! iobuf )
+		return -ENOMEM;
+	iob_reserve ( iobuf, MAX_LL_HEADER_LEN );
+
+	/* Construct EAPoL header */
+	eapol = iob_put ( iobuf, sizeof ( *eapol ) );
+	eapol->version = EAPOL_VERSION_2001;
+	eapol->type = type;
+	eapol->len = htons ( len );
+
+	/* Append packet body */
+	memcpy ( iob_put ( iobuf, len ), data, len );
+
+	/* Transmit packet */
+	if ( ( rc = net_tx ( iob_disown ( iobuf ), netdev, &eapol_protocol,
+			     &eapol_mac, netdev->ll_addr ) ) != 0 ) {
+		DBGC ( netdev, "EAPOL %s could not transmit type %d: %s\n",
+		       netdev->name, type, strerror ( rc ) );
+		DBGC_HDA ( netdev, 0, data, len );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Transmit EAPoL-encapsulated EAP packet
+ *
+ * @v supplicant	EAPoL supplicant
+ * @v ll_source		Link-layer source address
+ * @ret rc		Return status code
+ */
+static int eapol_eap_tx ( struct eap_supplicant *eap, const void *data,
+			  size_t len ) {
+	struct eapol_supplicant *supplicant =
+		container_of ( eap, struct eapol_supplicant, eap );
+
+	/* Transmit encapsulated packet */
+	return eapol_tx ( supplicant, EAPOL_TYPE_EAP, data, len );
+}
+
+/**
+ * Create EAPoL supplicant
+ *
+ * @v netdev		Network device
+ * @v priv		Private data
+ * @ret rc		Return status code
+ */
+static int eapol_probe ( struct net_device *netdev, void *priv ) {
+	struct eapol_supplicant *supplicant = priv;
+	struct ll_protocol *ll_protocol = netdev->ll_protocol;
+
+	/* Ignore non-EAPoL devices */
+	if ( ll_protocol->ll_proto != htons ( ARPHRD_ETHER ) )
+		return 0;
+	if ( vlan_tag ( netdev ) )
+		return 0;
+
+	/* Initialise structure */
+	supplicant->eap.netdev = netdev;
+	supplicant->eap.tx = eapol_eap_tx;
+
+	return 0;
+}
+
+/** EAPoL driver */
+struct net_driver eapol_driver __net_driver = {
+	.name = "EAPoL",
+	.priv_len = sizeof ( struct eapol_supplicant ),
+	.probe = eapol_probe,
+};

From 8b14652e506d99499cfbeaed0df07d6a83ec029e Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 15 Sep 2023 16:14:59 +0100
Subject: [PATCH 010/237] [eapol] Send EAPoL-Start packets to trigger EAP
 authentication

We have no way to force a link-layer restart in iPXE, and therefore no
way to explicitly trigger a restart of EAP authentication.  If an iPXE
script has performed some action that requires such a restart
(e.g. registering a device such that the port VLAN assignment will be
changed), then the only means currently available to effect the
restart is to reboot the whole system.  If iPXE is taking over a
physical link already used by a preceding bootloader, then even a
reboot may not work.

In the EAP model, the supplicant is a pure responder and never
initiates transmissions.  EAPoL extends this to include an EAPoL-Start
packet type that may be sent by the supplicant to (re)trigger EAP.

Add support for sending EAPoL-Start packets at two-second intervals on
links that are open and have reached physical link-up, but for which
EAP has not yet completed.  This allows "ifclose ; ifopen" to be used
to restart the EAP process.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/eapol.h |  8 +++++
 src/net/eapol.c          | 73 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)

diff --git a/src/include/ipxe/eapol.h b/src/include/ipxe/eapol.h
index f6009a2ff..d4ea39208 100644
--- a/src/include/ipxe/eapol.h
+++ b/src/include/ipxe/eapol.h
@@ -30,6 +30,9 @@ struct eapol_header {
 /** EAPoL-encapsulated EAP packets */
 #define EAPOL_TYPE_EAP 0
 
+/** EAPoL start */
+#define EAPOL_TYPE_START 1
+
 /** EAPoL key */
 #define EAPOL_TYPE_KEY 5
 
@@ -37,8 +40,13 @@ struct eapol_header {
 struct eapol_supplicant {
 	/** EAP supplicant */
 	struct eap_supplicant eap;
+	/** EAPoL-Start retransmission timer */
+	struct retry_timer timer;
 };
 
+/** Delay between EAPoL-Start packets */
+#define EAPOL_START_INTERVAL ( 2 * TICKS_PER_SEC )
+
 /** An EAPoL handler */
 struct eapol_handler {
 	/** Type */
diff --git a/src/net/eapol.c b/src/net/eapol.c
index 172037ce1..1b843e896 100644
--- a/src/net/eapol.c
+++ b/src/net/eapol.c
@@ -31,6 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/if_arp.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/vlan.h>
+#include <ipxe/retry.h>
 #include <ipxe/eap.h>
 #include <ipxe/eapol.h>
 
@@ -47,6 +48,37 @@ static const uint8_t eapol_mac[ETH_ALEN] = {
 	0x01, 0x80, 0xc2, 0x00, 0x00, 0x03
 };
 
+/**
+ * Update EAPoL supplicant state
+ *
+ * @v supplicant	EAPoL supplicant
+ * @v timeout		Timer ticks until next EAPoL-Start (if applicable)
+ */
+static void eapol_update ( struct eapol_supplicant *supplicant,
+			   unsigned long timeout ) {
+	struct net_device *netdev = supplicant->eap.netdev;
+
+	/* Check device and EAP state */
+	if ( netdev_is_open ( netdev ) && netdev_link_ok ( netdev ) ) {
+		if ( supplicant->eap.done ) {
+
+			/* EAP has completed: stop sending EAPoL-Start */
+			stop_timer ( &supplicant->timer );
+
+		} else if ( ! timer_running ( &supplicant->timer ) ) {
+
+			/* EAP has not yet begun: start sending EAPoL-Start */
+			start_timer_fixed ( &supplicant->timer, timeout );
+		}
+
+	} else {
+
+		/* Not ready: clear completion and stop sending EAPoL-Start */
+		supplicant->eap.done = 0;
+		stop_timer ( &supplicant->timer );
+	}
+}
+
 /**
  * Process EAPoL packet
  *
@@ -154,6 +186,9 @@ static int eapol_eap_rx ( struct eapol_supplicant *supplicant,
 		goto drop;
 	}
 
+	/* Update supplicant state */
+	eapol_update ( supplicant, EAPOL_START_INTERVAL );
+
  drop:
 	free_iob ( iobuf );
 	return rc;
@@ -224,6 +259,25 @@ static int eapol_eap_tx ( struct eap_supplicant *eap, const void *data,
 	return eapol_tx ( supplicant, EAPOL_TYPE_EAP, data, len );
 }
 
+/**
+ * (Re)transmit EAPoL-Start packet
+ *
+ * @v timer		EAPoL-Start timer
+ * @v expired		Failure indicator
+ */
+static void eapol_expired ( struct retry_timer *timer, int fail __unused ) {
+	struct eapol_supplicant *supplicant =
+		container_of ( timer, struct eapol_supplicant, timer );
+	struct net_device *netdev = supplicant->eap.netdev;
+
+	/* Schedule next transmission */
+	start_timer_fixed ( timer, EAPOL_START_INTERVAL );
+
+	/* Transmit EAPoL-Start, ignoring errors */
+	DBGC2 ( netdev, "EAPOL %s transmitting Start\n", netdev->name );
+	eapol_tx ( supplicant, EAPOL_TYPE_START, NULL, 0 );
+}
+
 /**
  * Create EAPoL supplicant
  *
@@ -244,13 +298,32 @@ static int eapol_probe ( struct net_device *netdev, void *priv ) {
 	/* Initialise structure */
 	supplicant->eap.netdev = netdev;
 	supplicant->eap.tx = eapol_eap_tx;
+	timer_init ( &supplicant->timer, eapol_expired, &netdev->refcnt );
 
 	return 0;
 }
 
+/**
+ * Handle EAPoL supplicant state change
+ *
+ * @v netdev		Network device
+ * @v priv		Private data
+ */
+static void eapol_notify ( struct net_device *netdev __unused, void *priv ) {
+	struct eapol_supplicant *supplicant = priv;
+
+	/* Ignore non-EAPoL devices */
+	if ( ! supplicant->eap.netdev )
+		return;
+
+	/* Update supplicant state */
+	eapol_update ( supplicant, 0 );
+}
+
 /** EAPoL driver */
 struct net_driver eapol_driver __net_driver = {
 	.name = "EAPoL",
 	.priv_len = sizeof ( struct eapol_supplicant ),
 	.probe = eapol_probe,
+	.notify = eapol_notify,
 };

From ff0f860483e344f1af633f94696ff7bc1854611f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 6 Oct 2023 12:43:02 +0100
Subject: [PATCH 011/237] [libc] Use wall clock time as seed for the
 (non-cryptographic) RNG

We currently use the number of timer ticks since power-on as a seed
for the non-cryptographic RNG implemented by random().  Since iPXE is
often executed directly after power-on, and since the timer tick
resolution is generally low, this can often result in identical seed
values being used on each cold boot attempt.

As of commit 41f786c ("[settings] Add "unixtime" builtin setting to
expose the current time"), the current wall-clock time is always
available within the default build of iPXE.  Use this time instead, to
introduce variability between cold boot attempts on the same host.
(Note that variability between different hosts is obtained by using
the MAC address as an additional seed value.)

This has no effect on the separate DRBG used by cryptographic code.

Suggested-by: Heiko <heik0@xs4all.nl>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/random.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/core/random.c b/src/core/random.c
index 975a03cf5..e3251964b 100644
--- a/src/core/random.c
+++ b/src/core/random.c
@@ -6,8 +6,9 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stddef.h>
 #include <stdlib.h>
-#include <ipxe/timer.h>
+#include <time.h>
 
 static int32_t rnd_seed = 0;
 
@@ -30,8 +31,9 @@ void srandom ( unsigned int seed ) {
 long int random ( void ) {
 	int32_t q;
 
-	if ( ! rnd_seed ) /* Initialize linear congruential generator */
-		srandom ( currticks() );
+	/* Initialize linear congruential generator */
+	if ( ! rnd_seed )
+		srandom ( time ( NULL ) );
 
 	/* simplified version of the LCG given in Bruce Schneier's
 	   "Applied Cryptography" */

From 115707c0edebad65f87525fed583fef73880016d Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 24 Oct 2023 11:43:56 +0100
Subject: [PATCH 012/237] [iphone] Add missing va_start()/va_end() around
 reused argument list
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The ipair_tx() function uses a va_list twice (first to calculate the
formatted string length before allocation, then to construct the
string in the allocated buffer) but is missing the va_start() and
va_end() around the second usage.  This is undefined behaviour that
happens to work on some build platforms.

Fix by adding the missing va_start() and va_end() around the second
usage of the variadic argument list.

Reported-by: Andreas Hammarskjöld <andreas@2PintSoftware.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/iphone.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/drivers/net/iphone.c b/src/drivers/net/iphone.c
index 7d0eb4b64..bbac527bd 100644
--- a/src/drivers/net/iphone.c
+++ b/src/drivers/net/iphone.c
@@ -1304,7 +1304,9 @@ ipair_tx ( struct ipair *ipair, const char *fmt, ... ) {
 	memset ( hdr, 0, sizeof ( *hdr ) );
 	hdr->len = htonl ( len );
 	msg = iob_put ( iobuf, len );
+	va_start ( args, fmt );
 	vsnprintf ( msg, len, fmt, args );
+	va_end ( args );
 	DBGC2 ( ipair, "IPAIR %p transmitting:\n%s\n", ipair, msg );
 
 	/* Transmit message */

From f883203132b3d06ffb354f1b11d1f65a0c70ff42 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 1 Nov 2023 22:03:34 +0000
Subject: [PATCH 013/237] [pci] Force completion of ECAM configuration space
 writes

The PCIe specification requires that "processor and host bridge
implementations must ensure that a method exists for the software to
determine when the write using the ECAM is completed by the completer"
but does not specify any particular method to be used.  Some platforms
might treat writes to the ECAM region as non-posted, others might
require reading back from a dedicated (and implementation-specific)
completion register to determine when the configuration space write
has completed.

Since PCI configuration space writes will never be used for any
performance-critical datapath operations (on any sane hardware), a
simple and platform-independent solution is to always read back from
the written register in order to guarantee that the write must have
completed.  This is safe to do, since the PCIe specification defines a
limited set of configuration register types, none of which have read
side effects.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/bus/ecam.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/drivers/bus/ecam.c b/src/drivers/bus/ecam.c
index 1d57bd2a0..376f34957 100644
--- a/src/drivers/bus/ecam.c
+++ b/src/drivers/bus/ecam.c
@@ -235,7 +235,7 @@ int ecam_write ( struct pci_device *pci, unsigned int location,
 	if ( ( rc = ecam_access ( pci ) ) != 0 )
 		return rc;
 
-	/* Read from address */
+	/* Write to address */
 	index = ( pci->busdevfn - ecam.range.start );
 	addr = ( ecam.regs + ( index * ECAM_SIZE ) + where );
 	switch ( len ) {
@@ -252,6 +252,15 @@ int ecam_write ( struct pci_device *pci, unsigned int location,
 		assert ( 0 );
 	}
 
+	/* Read from address, to guarantee completion of the write
+	 *
+	 * PCIe configuration space registers may not have read side
+	 * effects.  Reading back is therefore always safe to do, and
+	 * guarantees that the write has reached the device.
+	 */
+	mb();
+	ecam_read ( pci, location, &value );
+
 	return 0;
 }
 

From 74ec00a9f336152fccf8df9cc9c5a46ab1a649b4 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 2 Nov 2023 15:05:15 +0000
Subject: [PATCH 014/237] [pci] Handle non-zero starting bus in ECAM
 allocations

The base address provided in the PCI ECAM allocation within the ACPI
MCFG table is the base address for the segment as a whole, not for the
starting bus within that allocation.  On machines that provide ECAM
allocations with a non-zero starting bus number (observed with an AWS
EC2 m7a.large instance), this will result in iPXE accessing the wrong
memory addresses within the ECAM region.

Fix by adding the appropriate starting bus offset to the base address.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/bus/ecam.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/drivers/bus/ecam.c b/src/drivers/bus/ecam.c
index 376f34957..dd3e1e514 100644
--- a/src/drivers/bus/ecam.c
+++ b/src/drivers/bus/ecam.c
@@ -150,6 +150,7 @@ static int ecam_access ( struct pci_device *pci ) {
 
 	/* Map configuration space for this allocation */
 	base = le64_to_cpu ( ecam.alloc.base );
+	base += ( ecam.alloc.start * ECAM_SIZE * PCI_BUSDEVFN ( 0, 1, 0, 0 ) );
 	len = ( ecam.range.count * ECAM_SIZE );
 	ecam.regs = ioremap ( base, len );
 	if ( ! ecam.regs ) {

From 1f3a37e342ff110a451afcdf15c75a90e643208d Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 2 Nov 2023 15:16:19 +0000
Subject: [PATCH 015/237] [pci] Cache ECAM mapping errors

When an error occurs during ECAM configuration space mapping, preserve
the error within the existing cached mapping (instead of invalidating
the cached mapping) in order to avoid flooding the debug log with
repeated identical mapping errors.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/bus/ecam.c  | 6 ++++--
 src/include/ipxe/ecam.h | 2 ++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/drivers/bus/ecam.c b/src/drivers/bus/ecam.c
index dd3e1e514..62baebeed 100644
--- a/src/drivers/bus/ecam.c
+++ b/src/drivers/bus/ecam.c
@@ -127,7 +127,7 @@ static int ecam_access ( struct pci_device *pci ) {
 
 	/* Reuse mapping if possible */
 	if ( ( pci->busdevfn - ecam.range.start ) < ecam.range.count )
-		return 0;
+		return ecam.rc;
 
 	/* Clear any existing mapping */
 	if ( ecam.regs ) {
@@ -145,6 +145,7 @@ static int ecam_access ( struct pci_device *pci ) {
 	if ( ecam.range.start > pci->busdevfn ) {
 		DBGC ( &ecam, "ECAM found no allocation for " PCI_FMT "\n",
 		       PCI_ARGS ( pci ) );
+		rc = -ENOENT;
 		goto err_find;
 	}
 
@@ -165,12 +166,13 @@ static int ecam_access ( struct pci_device *pci ) {
 	DBGC ( &ecam, "ECAM %04x:[%02x-%02x] mapped [%08llx,%08llx) -> %p\n",
 	       le16_to_cpu ( ecam.alloc.segment ), ecam.alloc.start,
 	       ecam.alloc.end, base, ( base + len ), ecam.regs );
+	ecam.rc = 0;
 	return 0;
 
 	iounmap ( ecam.regs );
  err_ioremap:
  err_find:
-	ecam.range.count = 0;
+	ecam.rc = rc;
 	return rc;
 }
 
diff --git a/src/include/ipxe/ecam.h b/src/include/ipxe/ecam.h
index 683d613a0..ff08aee5a 100644
--- a/src/include/ipxe/ecam.h
+++ b/src/include/ipxe/ecam.h
@@ -50,6 +50,8 @@ struct ecam_mapping {
 	struct pci_range range;
 	/** MMIO base address */
 	void *regs;
+	/** Mapping result */
+	int rc;
 };
 
 extern struct pci_api ecam_api;

From 36e1a559a28ea9d62eb1f6cde4df8fda3999525e Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 2 Nov 2023 15:38:08 +0000
Subject: [PATCH 016/237] [pci] Check that ECAM configuration space is within
 reachable memory

Some machines (observed with an AWS EC2 m7a.large instance) will place
the ECAM configuration space window above 4GB, thereby making it
unreachable from non-paged 32-bit code.  This problem is currently
ignored by iPXE, since the address is silently truncated in the call
to ioremap().  (Note that other uses of ioremap() are not affected
since the PCI core will already have checked for unreachable 64-bit
BARs when retrieving the physical address to be mapped.)

Fix by adding an explicit check that the region to be mapped starts
within the reachable memory address space.  (Assume that no machines
will be sufficiently peverse to provide a region that straddles the
4GB boundary.)

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/bus/ecam.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/drivers/bus/ecam.c b/src/drivers/bus/ecam.c
index 62baebeed..5e3debddd 100644
--- a/src/drivers/bus/ecam.c
+++ b/src/drivers/bus/ecam.c
@@ -153,6 +153,14 @@ static int ecam_access ( struct pci_device *pci ) {
 	base = le64_to_cpu ( ecam.alloc.base );
 	base += ( ecam.alloc.start * ECAM_SIZE * PCI_BUSDEVFN ( 0, 1, 0, 0 ) );
 	len = ( ecam.range.count * ECAM_SIZE );
+	if ( base != ( ( unsigned long ) base ) ) {
+		DBGC ( &ecam, "ECAM %04x:[%02x-%02x] could not map "
+		       "[%08llx,%08llx) outside CPU range\n",
+		       le16_to_cpu ( ecam.alloc.segment ), ecam.alloc.start,
+		       ecam.alloc.end, base, ( base + len ) );
+		rc = -ERANGE;
+		goto err_range;
+	}
 	ecam.regs = ioremap ( base, len );
 	if ( ! ecam.regs ) {
 		DBGC ( &ecam, "ECAM %04x:[%02x-%02x] could not map "
@@ -171,6 +179,7 @@ static int ecam_access ( struct pci_device *pci ) {
 
 	iounmap ( ecam.regs );
  err_ioremap:
+ err_range:
  err_find:
 	ecam.rc = rc;
 	return rc;

From 5524bb98328dd1b16037916498b0f91e0200a87c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 2 Nov 2023 16:11:38 +0000
Subject: [PATCH 017/237] [pci] Require discovery of a PCI device when
 determining usable PCI APIs

The PCI cloud API (PCIAPI_CLOUD) currently selects the first PCI API
that successfully discovers a PCI device address range.  The ECAM API
may discover an address range but subsequently be unable to map the
configuration space region, which would result in the selected PCI API
being unusable.

Fix by instead selecting the first PCI API that can be successfully
used to discover a PCI device.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/interface/pcbios/pcicloud.c | 25 +++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/arch/x86/interface/pcbios/pcicloud.c b/src/arch/x86/interface/pcbios/pcicloud.c
index 97d7cac1d..98ba38b31 100644
--- a/src/arch/x86/interface/pcbios/pcicloud.c
+++ b/src/arch/x86/interface/pcbios/pcicloud.c
@@ -165,24 +165,27 @@ static void pcicloud_init ( void ) {
 	static struct pci_api *apis[] = {
 		&ecam_api, &pcibios_api, &pcidirect_api
 	};
-	struct pci_range range;
+	struct pci_device pci;
+	uint32_t busdevfn;
 	unsigned int i;
+	int rc;
 
-	/* Select first API that successfully discovers an address range */
+	/* Select first API that successfully discovers a PCI device */
 	for ( i = 0 ; i < ( sizeof ( apis ) / sizeof ( apis[0] ) ) ; i++ ) {
 		pcicloud = apis[i];
-		pcicloud_discover ( 0, &range );
-		if ( range.count != 0 ) {
-			DBGC ( pcicloud, "PCICLOUD selected %s API\n",
-			       pcicloud->name );
-			break;
+		busdevfn = 0;
+		if ( ( rc = pci_find_next ( &pci, &busdevfn ) ) == 0 ) {
+			DBGC ( pcicloud, "PCICLOUD selected %s API (found "
+			       PCI_FMT ")\n", pcicloud->name,
+			       PCI_ARGS ( &pci ) );
+			return;
 		}
 	}
 
-	/* The PCI direct API can never fail discovery since the range
-	 * is hardcoded.
-	 */
-	assert ( range.count != 0 );
+	/* Fall back to using final attempted API if no devices found */
+	pcicloud = apis[ i - 1 ];
+	DBGC ( pcicloud, "PCICLOUD selected %s API (nothing detected)\n",
+	       pcicloud->name );
 }
 
 /** Cloud VM PCI configuration space access initialisation function */

From 1bd01b761f1f33723f0b07d277863b3284dfe232 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 7 Nov 2023 11:08:33 +0000
Subject: [PATCH 018/237] [eapol] Delay EAPoL-Start while waiting for EAP to
 complete

EAP exchanges may take a long time to reach a final status, especially
when relying upon MAC Authentication Bypass (MAB).  Our current
behaviour of sending EAPoL-Start every few seconds until a final
status is obtained can prevent these exchanges from ever completing.

Fix by redefining the EAP supplicant state to allow EAPoL-Start to be
suppressed: either temporarily (while waiting for a full EAP exchange
to complete, in which case we need to eventually resend EAPoL-Start if
the final Success or Failure packet is lost), or permanently (while
waiting for the potentially very long MAC Authentication Bypass
timeout period).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/eap.h | 41 ++++++++++++++++++++++--
 src/net/eap.c          | 15 ++++++---
 src/net/eapol.c        | 72 +++++++++++++++++++++---------------------
 3 files changed, 84 insertions(+), 44 deletions(-)

diff --git a/src/include/ipxe/eap.h b/src/include/ipxe/eap.h
index e5f606553..818862a94 100644
--- a/src/include/ipxe/eap.h
+++ b/src/include/ipxe/eap.h
@@ -51,7 +51,7 @@ union eap_packet {
 	struct eap_request req;
 };
 
-/** Link block timeout
+/** EAP link block timeout
  *
  * We mark the link as blocked upon receiving a Request-Identity, on
  * the basis that this most likely indicates that the switch will not
@@ -64,12 +64,30 @@ union eap_packet {
  */
 #define EAP_BLOCK_TIMEOUT ( 45 * TICKS_PER_SEC )
 
+/** EAP protocol wait timeout
+ *
+ * In the EAP model, the supplicant is a pure responder.  The model
+ * also defines no acknowledgement response for the final Success or
+ * Failure "requests".  This leaves open the possibility that the
+ * final Success or Failure packet is lost, with the supplicant having
+ * no way to determine the final authentication status.
+ *
+ * Sideband mechanisms such as EAPoL-Start may be used to restart the
+ * entire EAP process, as a (crude) workaround for this protocol flaw.
+ * When expecting to receive a further EAP request (e.g. an
+ * authentication challenge), we may wait for some length of time
+ * before triggering this restart.  Choose a duration that is shorter
+ * than the link block timeout, so that there is no period during
+ * which we erroneously leave the link marked as not blocked.
+ */
+#define EAP_WAIT_TIMEOUT ( EAP_BLOCK_TIMEOUT * 7 / 8 )
+
 /** An EAP supplicant */
 struct eap_supplicant {
 	/** Network device */
 	struct net_device *netdev;
-	/** Authentication outcome is final */
-	int done;
+	/** Flags */
+	unsigned int flags;
 	/**
 	 * Transmit EAP response
 	 *
@@ -82,6 +100,23 @@ struct eap_supplicant {
 		       const void *data, size_t len );
 };
 
+/** EAP authentication is in progress
+ *
+ * This indicates that we have received an EAP Request-Identity, but
+ * have not yet received a final EAP Success or EAP Failure.
+ */
+#define EAP_FL_ONGOING 0x0001
+
+/** EAP supplicant is passive
+ *
+ * This indicates that the supplicant should not transmit any futher
+ * unsolicited packets (e.g. EAPoL-Start for a supplicant running over
+ * EAPoL).  This could be because authentication has already
+ * completed, or because we are relying upon MAC Authentication Bypass
+ * (MAB) which may have a very long timeout.
+ */
+#define EAP_FL_PASSIVE 0x0002
+
 extern int eap_rx ( struct eap_supplicant *supplicant,
 		    const void *data, size_t len );
 
diff --git a/src/net/eap.c b/src/net/eap.c
index beaeb61d2..2c68b75d4 100644
--- a/src/net/eap.c
+++ b/src/net/eap.c
@@ -47,6 +47,14 @@ static int eap_rx_request_identity ( struct eap_supplicant *supplicant ) {
 	       netdev->name );
 	netdev_link_block ( netdev, EAP_BLOCK_TIMEOUT );
 
+	/* Mark EAP as in progress */
+	supplicant->flags |= EAP_FL_ONGOING;
+
+	/* We have no identity to offer, so wait until the switch
+	 * times out and switches to MAC Authentication Bypass (MAB).
+	 */
+	supplicant->flags |= EAP_FL_PASSIVE;
+
 	return 0;
 }
 
@@ -69,9 +77,6 @@ static int eap_rx_request ( struct eap_supplicant *supplicant,
 		return -EINVAL;
 	}
 
-	/* Mark authentication as incomplete */
-	supplicant->done = 0;
-
 	/* Handle according to type */
 	switch ( req->type ) {
 	case EAP_TYPE_IDENTITY:
@@ -94,7 +99,7 @@ static int eap_rx_success ( struct eap_supplicant *supplicant ) {
 	struct net_device *netdev = supplicant->netdev;
 
 	/* Mark authentication as complete */
-	supplicant->done = 1;
+	supplicant->flags = EAP_FL_PASSIVE;
 
 	/* Mark link as unblocked */
 	DBGC ( netdev, "EAP %s Success\n", netdev->name );
@@ -113,7 +118,7 @@ static int eap_rx_failure ( struct eap_supplicant *supplicant ) {
 	struct net_device *netdev = supplicant->netdev;
 
 	/* Mark authentication as complete */
-	supplicant->done = 1;
+	supplicant->flags = EAP_FL_PASSIVE;
 
 	/* Record error */
 	DBGC ( netdev, "EAP %s Failure\n", netdev->name );
diff --git a/src/net/eapol.c b/src/net/eapol.c
index 1b843e896..ce7be55d5 100644
--- a/src/net/eapol.c
+++ b/src/net/eapol.c
@@ -48,37 +48,6 @@ static const uint8_t eapol_mac[ETH_ALEN] = {
 	0x01, 0x80, 0xc2, 0x00, 0x00, 0x03
 };
 
-/**
- * Update EAPoL supplicant state
- *
- * @v supplicant	EAPoL supplicant
- * @v timeout		Timer ticks until next EAPoL-Start (if applicable)
- */
-static void eapol_update ( struct eapol_supplicant *supplicant,
-			   unsigned long timeout ) {
-	struct net_device *netdev = supplicant->eap.netdev;
-
-	/* Check device and EAP state */
-	if ( netdev_is_open ( netdev ) && netdev_link_ok ( netdev ) ) {
-		if ( supplicant->eap.done ) {
-
-			/* EAP has completed: stop sending EAPoL-Start */
-			stop_timer ( &supplicant->timer );
-
-		} else if ( ! timer_running ( &supplicant->timer ) ) {
-
-			/* EAP has not yet begun: start sending EAPoL-Start */
-			start_timer_fixed ( &supplicant->timer, timeout );
-		}
-
-	} else {
-
-		/* Not ready: clear completion and stop sending EAPoL-Start */
-		supplicant->eap.done = 0;
-		stop_timer ( &supplicant->timer );
-	}
-}
-
 /**
  * Process EAPoL packet
  *
@@ -186,8 +155,19 @@ static int eapol_eap_rx ( struct eapol_supplicant *supplicant,
 		goto drop;
 	}
 
-	/* Update supplicant state */
-	eapol_update ( supplicant, EAPOL_START_INTERVAL );
+	/* Update EAPoL-Start transmission timer */
+	if ( supplicant->eap.flags & EAP_FL_PASSIVE ) {
+		/* Stop sending EAPoL-Start */
+		if ( timer_running ( &supplicant->timer ) ) {
+			DBGC ( netdev, "EAPOL %s becoming passive\n",
+			       netdev->name );
+		}
+		stop_timer ( &supplicant->timer );
+	} else if ( supplicant->eap.flags & EAP_FL_ONGOING ) {
+		/* Delay EAPoL-Start until after next expected packet */
+		DBGC ( netdev, "EAPOL %s deferring Start\n", netdev->name );
+		start_timer_fixed ( &supplicant->timer, EAP_WAIT_TIMEOUT );
+	}
 
  drop:
 	free_iob ( iobuf );
@@ -309,15 +289,35 @@ static int eapol_probe ( struct net_device *netdev, void *priv ) {
  * @v netdev		Network device
  * @v priv		Private data
  */
-static void eapol_notify ( struct net_device *netdev __unused, void *priv ) {
+static void eapol_notify ( struct net_device *netdev, void *priv ) {
 	struct eapol_supplicant *supplicant = priv;
 
 	/* Ignore non-EAPoL devices */
 	if ( ! supplicant->eap.netdev )
 		return;
 
-	/* Update supplicant state */
-	eapol_update ( supplicant, 0 );
+	/* Terminate and reset EAP when link goes down */
+	if ( ! ( netdev_is_open ( netdev ) && netdev_link_ok ( netdev ) ) ) {
+		if ( timer_running ( &supplicant->timer ) ) {
+			DBGC ( netdev, "EAPOL %s shutting down\n",
+			       netdev->name );
+		}
+		supplicant->eap.flags = 0;
+		stop_timer ( &supplicant->timer );
+		return;
+	}
+
+	/* Do nothing if EAP is already in progress */
+	if ( timer_running ( &supplicant->timer ) )
+		return;
+
+	/* Do nothing if EAP has already finished transmitting */
+	if ( supplicant->eap.flags & EAP_FL_PASSIVE )
+		return;
+
+	/* Otherwise, start sending EAPoL-Start */
+	start_timer_nodelay ( &supplicant->timer );
+	DBGC ( netdev, "EAPOL %s starting up\n", netdev->name );
 }
 
 /** EAPoL driver */

From 595b1796f6dc980cf27ca3883cde3baa23655528 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 7 Nov 2023 13:50:15 +0000
Subject: [PATCH 019/237] [eapol] Limit number of EAPoL-Start packets
 transmitted per attempt

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/eapol.h | 5 +++++
 src/net/eapol.c          | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/src/include/ipxe/eapol.h b/src/include/ipxe/eapol.h
index d4ea39208..dcf392946 100644
--- a/src/include/ipxe/eapol.h
+++ b/src/include/ipxe/eapol.h
@@ -42,11 +42,16 @@ struct eapol_supplicant {
 	struct eap_supplicant eap;
 	/** EAPoL-Start retransmission timer */
 	struct retry_timer timer;
+	/** EAPoL-Start transmission count */
+	unsigned int count;
 };
 
 /** Delay between EAPoL-Start packets */
 #define EAPOL_START_INTERVAL ( 2 * TICKS_PER_SEC )
 
+/** Maximum number of EAPoL-Start packets to transmit */
+#define EAPOL_START_COUNT 3
+
 /** An EAPoL handler */
 struct eapol_handler {
 	/** Type */
diff --git a/src/net/eapol.c b/src/net/eapol.c
index ce7be55d5..8b09ca231 100644
--- a/src/net/eapol.c
+++ b/src/net/eapol.c
@@ -167,6 +167,7 @@ static int eapol_eap_rx ( struct eapol_supplicant *supplicant,
 		/* Delay EAPoL-Start until after next expected packet */
 		DBGC ( netdev, "EAPOL %s deferring Start\n", netdev->name );
 		start_timer_fixed ( &supplicant->timer, EAP_WAIT_TIMEOUT );
+		supplicant->count = 0;
 	}
 
  drop:
@@ -250,6 +251,12 @@ static void eapol_expired ( struct retry_timer *timer, int fail __unused ) {
 		container_of ( timer, struct eapol_supplicant, timer );
 	struct net_device *netdev = supplicant->eap.netdev;
 
+	/* Stop transmitting after maximum number of attempts */
+	if ( supplicant->count++ >= EAPOL_START_COUNT ) {
+		DBGC ( netdev, "EAPOL %s giving up\n", netdev->name );
+		return;
+	}
+
 	/* Schedule next transmission */
 	start_timer_fixed ( timer, EAPOL_START_INTERVAL );
 
@@ -317,6 +324,7 @@ static void eapol_notify ( struct net_device *netdev, void *priv ) {
 
 	/* Otherwise, start sending EAPoL-Start */
 	start_timer_nodelay ( &supplicant->timer );
+	supplicant->count = 0;
 	DBGC ( netdev, "EAPOL %s starting up\n", netdev->name );
 }
 

From d8f9c221ede6d67a251134989a30bb850f8709e6 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 7 Nov 2023 15:54:59 +0000
Subject: [PATCH 020/237] [cloud] Add ability to overwrite existing AMI images

AMI names must be unique within a region.  Add a --overwrite option
that allows an existing AMI of the same name to be deregistered (and
its underlying snapshot deleted).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 contrib/cloud/aws-import | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/contrib/cloud/aws-import b/contrib/cloud/aws-import
index b9a350f63..ace005870 100755
--- a/contrib/cloud/aws-import
+++ b/contrib/cloud/aws-import
@@ -46,11 +46,19 @@ def create_snapshot(region, description, image):
     return snapshot_id
 
 
-def import_image(region, name, architecture, image, public):
+def import_image(region, name, architecture, image, public, overwrite):
     """Import an AMI image"""
     client = boto3.client('ec2', region_name=region)
     resource = boto3.resource('ec2', region_name=region)
     description = '%s (%s)' % (name, architecture)
+    images = client.describe_images(Filters=[{'Name': 'name',
+                                              'Values': [description]}])
+    if overwrite and images['Images']:
+        images = images['Images'][0]
+        image_id = images['ImageId']
+        snapshot_id = images['BlockDeviceMappings'][0]['Ebs']['SnapshotId']
+        resource.Image(image_id).deregister()
+        resource.Snapshot(snapshot_id).delete()
     snapshot_id = create_snapshot(region=region, description=description,
                                   image=image)
     client.get_waiter('snapshot_completed').wait(SnapshotIds=[snapshot_id])
@@ -88,6 +96,8 @@ parser.add_argument('--name', '-n',
                     help="Image name")
 parser.add_argument('--public', '-p', action='store_true',
                     help="Make image public")
+parser.add_argument('--overwrite', action='store_true',
+                    help="Overwrite any existing image with same name")
 parser.add_argument('--region', '-r', action='append',
                     help="AWS region(s)")
 parser.add_argument('--wiki', '-w', metavar='FILE',
@@ -115,7 +125,8 @@ with ThreadPoolExecutor(max_workers=len(imports)) as executor:
                                name=args.name,
                                architecture=architectures[image],
                                image=image,
-                               public=args.public): (region, image)
+                               public=args.public,
+                               overwrite=args.overwrite): (region, image)
                for region, image in imports}
     results = {futures[future]: future.result()
                for future in as_completed(futures)}

From 77b07ea4fdc259d7253c6f9df2beda6e6c7a9d85 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 7 Nov 2023 18:05:45 +0000
Subject: [PATCH 021/237] [cloud] Add utility script to read iPXE output from
 INT13CON partition

Some AWS instance types still do not support serial console output or
screenshots.  For these instance types, the only viable way to extract
debugging information is to use the INT13 console (which is already
enabled via CONFIG=cloud for all AWS images).

Obtaining the INT13 console output can be very cumbersome, since there
is no direct way to read from an AWS volume.  The simplest current
approach is to stop the instance under test, detach its root volume,
and reattach the volume to a Linux instance in the same region.

Add a utility script aws-int13con to retrieve the INT13 console output
by creating a temporary snapshot, reading the first block from the
snapshot, and extracting the INT13 console partition content.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 contrib/cloud/aws-int13con | 68 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100755 contrib/cloud/aws-int13con

diff --git a/contrib/cloud/aws-int13con b/contrib/cloud/aws-int13con
new file mode 100755
index 000000000..b79b40657
--- /dev/null
+++ b/contrib/cloud/aws-int13con
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import argparse
+
+import boto3
+
+BLOCKSIZE = 512 * 1024
+
+IPXELOG_OFFSET = 16 * 1024
+
+IPXELOG_MAGIC = b'iPXE LOG'
+
+
+def create_snapshot(region, instance_id):
+    """Create root volume snapshot"""
+    client = boto3.client('ec2', region_name=region)
+    resource = boto3.resource('ec2', region_name=region)
+    instance = resource.Instance(instance_id)
+    volumes = list(instance.volumes.all())
+    snapshot = volumes[0].create_snapshot()
+    snapshot.wait_until_completed()
+    return snapshot.id
+
+
+def get_snapshot_block(region, snapshot_id, index):
+    """Get block content from snapshot"""
+    client = boto3.client('ebs', region_name=region)
+    blocks = client.list_snapshot_blocks(SnapshotId=snapshot_id,
+                                         StartingBlockIndex=index)
+    token = blocks['Blocks'][0]['BlockToken']
+    block = client.get_snapshot_block(SnapshotId=snapshot_id,
+                                      BlockIndex=index,
+                                      BlockToken=token)
+    return block['BlockData'].read()
+
+
+def get_block0_content(region, instance_id):
+    """Get content of root volume block zero from instance"""
+    client = boto3.client('ec2', region_name=region)
+    resource = boto3.resource('ec2', region_name=region)
+    snapshot_id = create_snapshot(region, instance_id)
+    block = get_snapshot_block(region, snapshot_id, 0)
+    resource.Snapshot(snapshot_id).delete()
+    return block
+
+
+def get_int13con_output(region, instance_id):
+    """Get INT13 console output"""
+    block = get_block0_content(region, instance_id)
+    logpart = block[IPXELOG_OFFSET:]
+    magic = logpart[:len(IPXELOG_MAGIC)]
+    if magic != IPXELOG_MAGIC:
+        raise ValueError("Invalid log magic signature")
+    log = logpart[len(IPXELOG_MAGIC):].split(b'\0')[0]
+    return log.decode()
+
+
+# Parse command-line arguments
+parser = argparse.ArgumentParser(description="Get AWS INT13 console output")
+parser.add_argument('--region', '-r', help="AWS region")
+parser.add_argument('id', help="Instance ID")
+args = parser.parse_args()
+
+# Get console output from INT13CON partition
+output = get_int13con_output(args.region, args.id)
+
+# Print console output
+print(output)

From 8c8ead25305bbe9521b488144257c131abbbcd23 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 22 Nov 2023 23:30:56 +0000
Subject: [PATCH 022/237] [efi] Update to current EDK2 headers

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/Base.h                   | 10 +++++
 .../ipxe/efi/IndustryStandard/PeImage.h       | 22 +++++++---
 src/include/ipxe/efi/Library/BaseLib.h        |  5 +++
 src/include/ipxe/efi/Pi/PiStatusCode.h        | 42 ++++++++++---------
 src/include/ipxe/efi/Protocol/Rng.h           | 10 +++++
 src/include/ipxe/efi/Uefi/UefiBaseType.h      |  2 +
 6 files changed, 65 insertions(+), 26 deletions(-)

diff --git a/src/include/ipxe/efi/Base.h b/src/include/ipxe/efi/Base.h
index e76013c19..46c31a3b1 100644
--- a/src/include/ipxe/efi/Base.h
+++ b/src/include/ipxe/efi/Base.h
@@ -1231,6 +1231,11 @@ typedef UINTN RETURN_STATUS;
 ///
 #define RETURN_COMPROMISED_DATA  ENCODE_ERROR (33)
 
+///
+/// There is an address conflict address allocation.
+///
+#define RETURN_IP_ADDRESS_CONFLICT  ENCODE_ERROR (34)
+
 ///
 /// A HTTP error occurred during the network operation.
 ///
@@ -1270,6 +1275,11 @@ typedef UINTN RETURN_STATUS;
 ///
 #define RETURN_WARN_FILE_SYSTEM  ENCODE_WARNING (6)
 
+///
+/// The operation will be processed across a system reset.
+///
+#define RETURN_WARN_RESET_REQUIRED  ENCODE_WARNING (7)
+
 /**
   Returns a 16-bit signature built from 2 ASCII characters.
 
diff --git a/src/include/ipxe/efi/IndustryStandard/PeImage.h b/src/include/ipxe/efi/IndustryStandard/PeImage.h
index 401e961c9..c1f1a09cb 100644
--- a/src/include/ipxe/efi/IndustryStandard/PeImage.h
+++ b/src/include/ipxe/efi/IndustryStandard/PeImage.h
@@ -4,7 +4,7 @@
   EFI_IMAGE_NT_HEADERS64 is for PE32+.
 
   This file is coded to the Visual Studio, Microsoft Portable Executable and
-  Common Object File Format Specification, Revision 8.3 - February 6, 2013.
+  Common Object File Format Specification, Revision 9.3 - December 29, 2015.
   This file also includes some definitions in PI Specification, Revision 1.0.
 
 Copyright (c) 2006 - 2018, Intel Corporation. All rights reserved.<BR>
@@ -271,6 +271,21 @@ typedef struct {
 #define EFI_IMAGE_SUBSYSTEM_OS2_CUI      5
 #define EFI_IMAGE_SUBSYSTEM_POSIX_CUI    7
 
+//
+// DLL Characteristics
+//
+#define IMAGE_DLLCHARACTERISTICS_HIGH_ENTROPY_VA        0x0020
+#define IMAGE_DLLCHARACTERISTICS_DYNAMIC_BASE           0x0040
+#define IMAGE_DLLCHARACTERISTICS_FORCE_INTEGRITY        0x0080
+#define IMAGE_DLLCHARACTERISTICS_NX_COMPAT              0x0100
+#define IMAGE_DLLCHARACTERISTICS_NO_ISOLATION           0x0200
+#define IMAGE_DLLCHARACTERISTICS_NO_SEH                 0x0400
+#define IMAGE_DLLCHARACTERISTICS_NO_BIND                0x0800
+#define IMAGE_DLLCHARACTERISTICS_APPCONTAINER           0x1000
+#define IMAGE_DLLCHARACTERISTICS_WDM_DRIVER             0x2000
+#define IMAGE_DLLCHARACTERISTICS_GUARD_CF               0x4000
+#define IMAGE_DLLCHARACTERISTICS_TERMINAL_SERVER_AWARE  0x8000
+
 ///
 /// Length of ShortName.
 ///
@@ -680,9 +695,6 @@ typedef struct {
   //
 } EFI_IMAGE_DEBUG_CODEVIEW_MTOC_ENTRY;
 
-// avoid conflict with windows header files
-#ifndef RUNTIME_FUNCTION_INDIRECT
-
 //
 // .pdata entries for X64
 //
@@ -692,8 +704,6 @@ typedef struct {
   UINT32    UnwindInfoAddress;
 } RUNTIME_FUNCTION;
 
-#endif
-
 typedef struct {
   UINT8    Version             : 3;
   UINT8    Flags               : 5;
diff --git a/src/include/ipxe/efi/Library/BaseLib.h b/src/include/ipxe/efi/Library/BaseLib.h
index e17f3da26..6d2365955 100644
--- a/src/include/ipxe/efi/Library/BaseLib.h
+++ b/src/include/ipxe/efi/Library/BaseLib.h
@@ -183,6 +183,11 @@ RiscVSetSupervisorAddressTranslationRegister (
   IN UINT64
   );
 
+UINT64
+RiscVGetSupervisorAddressTranslationRegister (
+  VOID
+  );
+
 UINT64
 RiscVReadTimer (
   VOID
diff --git a/src/include/ipxe/efi/Pi/PiStatusCode.h b/src/include/ipxe/efi/Pi/PiStatusCode.h
index 4375f7046..427e5061d 100644
--- a/src/include/ipxe/efi/Pi/PiStatusCode.h
+++ b/src/include/ipxe/efi/Pi/PiStatusCode.h
@@ -365,6 +365,7 @@ typedef struct {
 #define EFI_PERIPHERAL_LCD_DEVICE       (EFI_PERIPHERAL | 0x000B0000)
 #define EFI_PERIPHERAL_NETWORK          (EFI_PERIPHERAL | 0x000C0000)
 #define EFI_PERIPHERAL_DOCKING          (EFI_PERIPHERAL | 0x000D0000)
+#define EFI_PERIPHERAL_TPM              (EFI_PERIPHERAL | 0x000E0000)
 ///@}
 
 ///
@@ -967,26 +968,27 @@ typedef struct {
 /// These are shared by all subclasses.
 ///
 ///@{
-#define EFI_SW_EC_NON_SPECIFIC            0x00000000
-#define EFI_SW_EC_LOAD_ERROR              0x00000001
-#define EFI_SW_EC_INVALID_PARAMETER       0x00000002
-#define EFI_SW_EC_UNSUPPORTED             0x00000003
-#define EFI_SW_EC_INVALID_BUFFER          0x00000004
-#define EFI_SW_EC_OUT_OF_RESOURCES        0x00000005
-#define EFI_SW_EC_ABORTED                 0x00000006
-#define EFI_SW_EC_ILLEGAL_SOFTWARE_STATE  0x00000007
-#define EFI_SW_EC_ILLEGAL_HARDWARE_STATE  0x00000008
-#define EFI_SW_EC_START_ERROR             0x00000009
-#define EFI_SW_EC_BAD_DATE_TIME           0x0000000A
-#define EFI_SW_EC_CFG_INVALID             0x0000000B
-#define EFI_SW_EC_CFG_CLR_REQUEST         0x0000000C
-#define EFI_SW_EC_CFG_DEFAULT             0x0000000D
-#define EFI_SW_EC_PWD_INVALID             0x0000000E
-#define EFI_SW_EC_PWD_CLR_REQUEST         0x0000000F
-#define EFI_SW_EC_PWD_CLEARED             0x00000010
-#define EFI_SW_EC_EVENT_LOG_FULL          0x00000011
-#define EFI_SW_EC_WRITE_PROTECTED         0x00000012
-#define EFI_SW_EC_FV_CORRUPTED            0x00000013
+#define EFI_SW_EC_NON_SPECIFIC             0x00000000
+#define EFI_SW_EC_LOAD_ERROR               0x00000001
+#define EFI_SW_EC_INVALID_PARAMETER        0x00000002
+#define EFI_SW_EC_UNSUPPORTED              0x00000003
+#define EFI_SW_EC_INVALID_BUFFER           0x00000004
+#define EFI_SW_EC_OUT_OF_RESOURCES         0x00000005
+#define EFI_SW_EC_ABORTED                  0x00000006
+#define EFI_SW_EC_ILLEGAL_SOFTWARE_STATE   0x00000007
+#define EFI_SW_EC_ILLEGAL_HARDWARE_STATE   0x00000008
+#define EFI_SW_EC_START_ERROR              0x00000009
+#define EFI_SW_EC_BAD_DATE_TIME            0x0000000A
+#define EFI_SW_EC_CFG_INVALID              0x0000000B
+#define EFI_SW_EC_CFG_CLR_REQUEST          0x0000000C
+#define EFI_SW_EC_CFG_DEFAULT              0x0000000D
+#define EFI_SW_EC_PWD_INVALID              0x0000000E
+#define EFI_SW_EC_PWD_CLR_REQUEST          0x0000000F
+#define EFI_SW_EC_PWD_CLEARED              0x00000010
+#define EFI_SW_EC_EVENT_LOG_FULL           0x00000011
+#define EFI_SW_EC_WRITE_PROTECTED          0x00000012
+#define EFI_SW_EC_FV_CORRUPTED             0x00000013
+#define EFI_SW_EC_INCONSISTENT_MEMORY_MAP  0x00000014
 ///@}
 
 //
diff --git a/src/include/ipxe/efi/Protocol/Rng.h b/src/include/ipxe/efi/Protocol/Rng.h
index 87c5c0edd..92d648bee 100644
--- a/src/include/ipxe/efi/Protocol/Rng.h
+++ b/src/include/ipxe/efi/Protocol/Rng.h
@@ -69,6 +69,15 @@ typedef EFI_GUID EFI_RNG_ALGORITHM;
   { \
     0xe43176d7, 0xb6e8, 0x4827, {0xb7, 0x84, 0x7f, 0xfd, 0xc4, 0xb6, 0x85, 0x61 } \
   }
+///
+/// The Arm Architecture states the RNDR that the DRBG algorithm should be compliant
+/// with NIST SP800-90A, while not mandating a particular algorithm, so as to be
+/// inclusive of different geographies.
+///
+#define EFI_RNG_ALGORITHM_ARM_RNDR \
+  { \
+    0x43d2fde3, 0x9d4e, 0x4d79,  {0x02, 0x96, 0xa8, 0x9b, 0xca, 0x78, 0x08, 0x41} \
+  }
 
 /**
   Returns information about the random number generation implementation.
@@ -148,5 +157,6 @@ extern EFI_GUID  gEfiRngAlgorithmSp80090Ctr256Guid;
 extern EFI_GUID  gEfiRngAlgorithmX9313DesGuid;
 extern EFI_GUID  gEfiRngAlgorithmX931AesGuid;
 extern EFI_GUID  gEfiRngAlgorithmRaw;
+extern EFI_GUID  gEfiRngAlgorithmArmRndr;
 
 #endif
diff --git a/src/include/ipxe/efi/Uefi/UefiBaseType.h b/src/include/ipxe/efi/Uefi/UefiBaseType.h
index 049275995..bf3aa9bb2 100644
--- a/src/include/ipxe/efi/Uefi/UefiBaseType.h
+++ b/src/include/ipxe/efi/Uefi/UefiBaseType.h
@@ -143,6 +143,7 @@ typedef union {
 #define EFI_END_OF_FILE           RETURN_END_OF_FILE
 #define EFI_INVALID_LANGUAGE      RETURN_INVALID_LANGUAGE
 #define EFI_COMPROMISED_DATA      RETURN_COMPROMISED_DATA
+#define EFI_IP_ADDRESS_CONFLICT   RETURN_IP_ADDRESS_CONFLICT
 #define EFI_HTTP_ERROR            RETURN_HTTP_ERROR
 
 #define EFI_WARN_UNKNOWN_GLYPH     RETURN_WARN_UNKNOWN_GLYPH
@@ -151,6 +152,7 @@ typedef union {
 #define EFI_WARN_BUFFER_TOO_SMALL  RETURN_WARN_BUFFER_TOO_SMALL
 #define EFI_WARN_STALE_DATA        RETURN_WARN_STALE_DATA
 #define EFI_WARN_FILE_SYSTEM       RETURN_WARN_FILE_SYSTEM
+#define EFI_WARN_RESET_REQUIRED    RETURN_WARN_RESET_REQUIRED
 ///@}
 
 ///

From 678a60f61d76b6fce2d9e3b323db1892f69800d3 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 22 Nov 2023 23:14:38 +0000
Subject: [PATCH 023/237] [efi] Treat writable sections as data sections

Hybrid bzImage and UEFI binaries (such as wimboot) may include 16-bit
executable code that is opaque data from the perspective of a UEFI PE
binary, as described in wimboot commit fe456ca ("[efi] Use separate
.text and .data PE sections").

The ELF section will be marked as containing both executable code and
writable data.  Choose to treat such a section as a data section
rather than a code section, since that matches the expected semantics
for ELF files that we expect to process.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 5b3e785f5..4ed016c0f 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -591,17 +591,7 @@ static struct pe_section * process_section ( struct elf_file *elf,
 
 	/* Fill in section characteristics and update RVA limits */
 	if ( ( shdr->sh_type == SHT_PROGBITS ) &&
-	     ( shdr->sh_flags & SHF_EXECINSTR ) ) {
-		/* .text-type section */
-		new->hdr.Characteristics =
-			( EFI_IMAGE_SCN_CNT_CODE |
-			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
-			  EFI_IMAGE_SCN_MEM_EXECUTE |
-			  EFI_IMAGE_SCN_MEM_READ );
-		applicable_start = &code_start;
-		applicable_end = &code_end;
-	} else if ( ( shdr->sh_type == SHT_PROGBITS ) &&
-		    ( shdr->sh_flags & SHF_WRITE ) ) {
+	     ( shdr->sh_flags & SHF_WRITE ) ) {
 		/* .data-type section */
 		new->hdr.Characteristics =
 			( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
@@ -610,6 +600,16 @@ static struct pe_section * process_section ( struct elf_file *elf,
 			  EFI_IMAGE_SCN_MEM_WRITE );
 		applicable_start = &data_start;
 		applicable_end = &data_mid;
+	} else if ( ( shdr->sh_type == SHT_PROGBITS ) &&
+		    ( shdr->sh_flags & SHF_EXECINSTR ) ) {
+		/* .text-type section */
+		new->hdr.Characteristics =
+			( EFI_IMAGE_SCN_CNT_CODE |
+			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
+			  EFI_IMAGE_SCN_MEM_EXECUTE |
+			  EFI_IMAGE_SCN_MEM_READ );
+		applicable_start = &code_start;
+		applicable_end = &code_end;
 	} else if ( shdr->sh_type == SHT_PROGBITS ) {
 		/* .rodata-type section */
 		new->hdr.Characteristics =

From a9e89787d08df46daee85a871ba256daf84c6f34 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 22 Nov 2023 23:36:35 +0000
Subject: [PATCH 024/237] [efi] Set NXCOMPAT bit in PE header

Indicate that the binary is compatible with W^X protections by setting
the NXCOMPAT bit in the DllCharacteristics field of the PE header.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 4ed016c0f..f4d591d1e 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -225,6 +225,8 @@ static struct pe_header efi_pe_header = {
 			.FileAlignment = EFI_FILE_ALIGN,
 			.SizeOfImage = EFI_IMAGE_ALIGN,
 			.SizeOfHeaders = sizeof ( efi_pe_header ),
+			.DllCharacteristics =
+				IMAGE_DLLCHARACTERISTICS_NX_COMPAT,
 			.NumberOfRvaAndSizes = NUMBER_OF_DIRECTORY_ENTRIES,
 		},
 	},

From 3d8a614657e68fb6cb7241397bc14cb5b9a7c0b8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 22 Nov 2023 23:05:39 +0000
Subject: [PATCH 025/237] [efi] Mark PE images as large address aware

The images generated by elf2efi can be loaded anywhere in the address
space, and are not limited to the low 2GB.

Indicate this by setting the "large address aware" flag within the PE
header, for compatibility with EFI images generated by the EDK2 build
process.  (The EDK2 PE loader does not ever check this flag, and it is
unlikely that any other EFI PE loader ever does so, but we may as well
report it accurately.)

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index f4d591d1e..b13ff43eb 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -215,6 +215,7 @@ static struct pe_header efi_pe_header = {
 					     DataDirectory[0] ) ) ),
 			.Characteristics = ( EFI_IMAGE_FILE_DLL |
 					     EFI_IMAGE_FILE_MACHINE |
+					     EFI_IMAGE_FILE_LARGE_ADDRESS_AWARE|
 					     EFI_IMAGE_FILE_EXECUTABLE_IMAGE ),
 		},
 		.OptionalHeader = {

From b30a0987e239baf4689adbf79035b7f737eef873 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 22 Nov 2023 14:57:05 +0000
Subject: [PATCH 026/237] [efi] Use load memory address as file offset for
 hybrid binaries

Hybrid bzImage and UEFI binaries (such as wimboot) may be loaded as a
single contiguous blob without reference to the PE headers, and the
placement of sections within the PE file must therefore be known at
link time.

Use the load memory address (extracted from the ELF program headers)
to determine the physical placement of the section within the PE file
when generating a hybrid binary.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 84 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 77 insertions(+), 7 deletions(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index b13ff43eb..682580f44 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -50,6 +50,7 @@
 #define EFI_IMAGE_FILE_MACHINE		EFI_IMAGE_FILE_32BIT_MACHINE
 #define ELFCLASS   ELFCLASS32
 #define Elf_Ehdr   Elf32_Ehdr
+#define Elf_Phdr   Elf32_Phdr
 #define Elf_Shdr   Elf32_Shdr
 #define Elf_Sym    Elf32_Sym
 #define Elf_Addr   Elf32_Addr
@@ -65,6 +66,7 @@
 #define EFI_IMAGE_FILE_MACHINE		0
 #define ELFCLASS   ELFCLASS64
 #define Elf_Ehdr   Elf64_Ehdr
+#define Elf_Phdr   Elf64_Phdr
 #define Elf_Shdr   Elf64_Shdr
 #define Elf_Sym    Elf64_Sym
 #define Elf_Addr   Elf64_Addr
@@ -168,6 +170,9 @@
  */
 #define EFI_IMAGE_ALIGN 0x1000
 
+/** Set PointerToRawData automatically */
+#define PTRD_AUTO 0xffffffffUL
+
 /** Number of data directory entries */
 #define NUMBER_OF_DIRECTORY_ENTRIES 8
 
@@ -429,6 +434,14 @@ static void read_elf_file ( const char *name, struct elf_file *elf ) {
 	}
 	elf->ehdr = ehdr;
 
+	/* Check program headers */
+	if ( ( elf->len < ehdr->e_phoff ) ||
+	     ( ( elf->len - ehdr->e_phoff ) < ( ehdr->e_phnum *
+						ehdr->e_phentsize ) ) ) {
+		eprintf ( "ELF program headers outside file in %s\n", name );
+		exit ( 1 );
+	}
+
 	/* Check section headers */
 	for ( i = 0 ; i < ehdr->e_shnum ; i++ ) {
 		offset = ( ehdr->e_shoff + ( i * ehdr->e_shentsize ) );
@@ -498,6 +511,39 @@ static const char * elf_string ( struct elf_file *elf, unsigned int section,
 	return string;
 }
 
+/**
+ * Get section load memory address
+ *
+ * @v elf		ELF file
+ * @v shdr		ELF section header
+ * @v name		ELF section name
+ * @ret lma		Load memory address
+ */
+static unsigned long elf_lma ( struct elf_file *elf, const Elf_Shdr *shdr,
+			       const char *name ) {
+	const Elf_Ehdr *ehdr = elf->ehdr;
+	const Elf_Phdr *phdr;
+	size_t offset;
+	unsigned int i;
+
+	/* Find containing segment */
+	for ( i = 0 ; i < ehdr->e_phnum ; i++ ) {
+		offset = ( ehdr->e_phoff + ( i * ehdr->e_phentsize ) );
+		phdr = ( elf->data + offset );
+		if ( ( phdr->p_type == PT_LOAD ) &&
+		     ( phdr->p_vaddr <= shdr->sh_addr ) &&
+		     ( ( shdr->sh_addr - phdr->p_vaddr + shdr->sh_size ) <=
+		       phdr->p_memsz ) ) {
+			/* Found matching segment */
+			return ( phdr->p_paddr +
+				 ( shdr->sh_addr - phdr->p_vaddr ) );
+		}
+	}
+
+	eprintf ( "No containing segment for section %s\n", name );
+	exit ( 1 );
+}
+
 /**
  * Set machine architecture
  *
@@ -540,11 +586,13 @@ static void set_machine ( struct elf_file *elf, struct pe_header *pe_header ) {
  * @v elf		ELF file
  * @v shdr		ELF section header
  * @v pe_header		PE file header
+ * @v opts		Options
  * @ret new		New PE section
  */
 static struct pe_section * process_section ( struct elf_file *elf,
 					     const Elf_Shdr *shdr,
-					     struct pe_header *pe_header ) {
+					     struct pe_header *pe_header,
+					     struct options *opts ) {
 	struct pe_section *new;
 	const char *name;
 	size_t name_len;
@@ -591,6 +639,13 @@ static struct pe_section * process_section ( struct elf_file *elf,
 	new->hdr.Misc.VirtualSize = section_memsz;
 	new->hdr.VirtualAddress = shdr->sh_addr;
 	new->hdr.SizeOfRawData = section_filesz;
+	if ( shdr->sh_type == SHT_PROGBITS ) {
+		if ( opts->hybrid ) {
+			new->hdr.PointerToRawData = elf_lma ( elf, shdr, name );
+		} else {
+			new->hdr.PointerToRawData = PTRD_AUTO;
+		}
+	}
 
 	/* Fill in section characteristics and update RVA limits */
 	if ( ( shdr->sh_type == SHT_PROGBITS ) &&
@@ -835,6 +890,7 @@ create_reloc_section ( struct pe_header *pe_header,
 	reloc->hdr.Misc.VirtualSize = section_memsz;
 	reloc->hdr.VirtualAddress = pe_header->nt.OptionalHeader.SizeOfImage;
 	reloc->hdr.SizeOfRawData = section_filesz;
+	reloc->hdr.PointerToRawData = PTRD_AUTO;
 	reloc->hdr.Characteristics = ( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
 				       EFI_IMAGE_SCN_MEM_DISCARDABLE |
 				       EFI_IMAGE_SCN_MEM_NOT_PAGED |
@@ -901,6 +957,7 @@ create_debug_section ( struct pe_header *pe_header, const char *filename ) {
 	debug->hdr.Misc.VirtualSize = section_memsz;
 	debug->hdr.VirtualAddress = pe_header->nt.OptionalHeader.SizeOfImage;
 	debug->hdr.SizeOfRawData = section_filesz;
+	debug->hdr.PointerToRawData = PTRD_AUTO;
 	debug->hdr.Characteristics = ( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
 				       EFI_IMAGE_SCN_MEM_DISCARDABLE |
 				       EFI_IMAGE_SCN_MEM_NOT_PAGED |
@@ -944,19 +1001,25 @@ static void write_pe_file ( struct pe_header *pe_header,
 			    FILE *pe ) {
 	struct pe_section *section;
 	unsigned long fpos = 0;
+	unsigned long fposmax = 0;
 	unsigned int count = 0;
 
 	/* Align length of headers */
-	fpos = pe_header->nt.OptionalHeader.SizeOfHeaders =
+	fpos = fposmax = pe_header->nt.OptionalHeader.SizeOfHeaders =
 		efi_file_align ( pe_header->nt.OptionalHeader.SizeOfHeaders );
 
 	/* Assign raw data pointers */
 	for ( section = pe_sections ; section ; section = section->next ) {
-		if ( section->hdr.SizeOfRawData ) {
-			section->hdr.PointerToRawData = fpos;
-			fpos += section->hdr.SizeOfRawData;
-			fpos = efi_file_align ( fpos );
+		if ( section->hdr.PointerToRawData == PTRD_AUTO ) {
+			fpos = fposmax;
+		} else {
+			fpos = section->hdr.PointerToRawData;
 		}
+		section->hdr.PointerToRawData = fpos;
+		fpos += section->hdr.SizeOfRawData;
+		fpos = efi_file_align ( fpos );
+		if ( fpos > fposmax )
+			fposmax = fpos;
 		if ( section->fixup )
 			section->fixup ( section );
 	}
@@ -985,6 +1048,12 @@ static void write_pe_file ( struct pe_header *pe_header,
 
 	/* Write sections */
 	for ( section = pe_sections ; section ; section = section->next ) {
+		if ( section->hdr.PointerToRawData & ( EFI_FILE_ALIGN - 1 ) ) {
+			eprintf ( "Section %.8s file offset %x is "
+				  "misaligned\n", section->hdr.Name,
+				  section->hdr.PointerToRawData );
+			exit ( 1 );
+		}
 		if ( fseek ( pe, section->hdr.PointerToRawData,
 			     SEEK_SET ) != 0 ) {
 			eprintf ( "Could not seek to %x: %s\n",
@@ -1044,7 +1113,8 @@ static void elf2pe ( const char *elf_name, const char *pe_name,
 
 			/* Create output section */
 			*(next_pe_section) = process_section ( &elf, shdr,
-							       &pe_header );
+							       &pe_header,
+							       opts );
 			next_pe_section = &(*next_pe_section)->next;
 
 		} else if ( shdr->sh_type == SHT_REL ) {

From cc858acd3205b18934ffff19ac55f8987e9c6135 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 22 Nov 2023 15:38:22 +0000
Subject: [PATCH 027/237] [efi] Write out PE header only after writing sections

Hybrid bzImage and UEFI binaries (such as wimboot) include a bzImage
header within a section starting at offset zero, with the PE header
effectively occupying unused space within this section.

Allow for this by treating a section placed at offset zero as hidden,
and by deferring the writing of the PE header until after the output
sections have been written.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 50 ++++++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 22 deletions(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 682580f44..a8074915a 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -642,6 +642,8 @@ static struct pe_section * process_section ( struct elf_file *elf,
 	if ( shdr->sh_type == SHT_PROGBITS ) {
 		if ( opts->hybrid ) {
 			new->hdr.PointerToRawData = elf_lma ( elf, shdr, name );
+			if ( new->hdr.PointerToRawData == 0 )
+				new->hidden = 1;
 		} else {
 			new->hdr.PointerToRawData = PTRD_AUTO;
 		}
@@ -1024,28 +1026,6 @@ static void write_pe_file ( struct pe_header *pe_header,
 			section->fixup ( section );
 	}
 
-	/* Write file header */
-	if ( fwrite ( pe_header,
-		      ( offsetof ( typeof ( *pe_header ), nt.OptionalHeader ) +
-			pe_header->nt.FileHeader.SizeOfOptionalHeader ),
-		      1, pe ) != 1 ) {
-		perror ( "Could not write PE header" );
-		exit ( 1 );
-	}
-
-	/* Write section headers */
-	for ( section = pe_sections ; section ; section = section->next ) {
-		if ( section->hidden )
-			continue;
-		if ( fwrite ( &section->hdr, sizeof ( section->hdr ),
-			      1, pe ) != 1 ) {
-			perror ( "Could not write section header" );
-			exit ( 1 );
-		}
-		count++;
-	}
-	assert ( count == pe_header->nt.FileHeader.NumberOfSections );
-
 	/* Write sections */
 	for ( section = pe_sections ; section ; section = section->next ) {
 		if ( section->hdr.PointerToRawData & ( EFI_FILE_ALIGN - 1 ) ) {
@@ -1069,6 +1049,32 @@ static void write_pe_file ( struct pe_header *pe_header,
 			exit ( 1 );
 		}
 	}
+
+	/* Write file header */
+	if ( fseek ( pe, 0, SEEK_SET ) != 0 ) {
+		eprintf ( "Could not rewind: %s\n", strerror ( errno ) );
+		exit ( 1 );
+	}
+	if ( fwrite ( pe_header,
+		      ( offsetof ( typeof ( *pe_header ), nt.OptionalHeader ) +
+			pe_header->nt.FileHeader.SizeOfOptionalHeader ),
+		      1, pe ) != 1 ) {
+		perror ( "Could not write PE header" );
+		exit ( 1 );
+	}
+
+	/* Write section headers */
+	for ( section = pe_sections ; section ; section = section->next ) {
+		if ( section->hidden )
+			continue;
+		if ( fwrite ( &section->hdr, sizeof ( section->hdr ),
+			      1, pe ) != 1 ) {
+			perror ( "Could not write section header" );
+			exit ( 1 );
+		}
+		count++;
+	}
+	assert ( count == pe_header->nt.FileHeader.NumberOfSections );
 }
 
 /**

From b37d89db90566cc1dcab9fe5713bde142fbc8f63 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 23 Nov 2023 14:54:12 +0000
Subject: [PATCH 028/237] [efi] Fix recorded overall size of headers in NT
 optional header

Commit 1e4c378 ("[efi] Shrink size of data directory in PE header")
reduced the number of entries used in the data directory and reduced
the recorded size of the NT "optional" header, but did not also adjust
the recorded overall size of the PE headers, resulting in unused space
between the PE headers and the first section.

Fix by reducing the initial recorded size of the PE headers by the
size of the omitted data directory entries.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index a8074915a..76d02f249 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -230,7 +230,12 @@ static struct pe_header efi_pe_header = {
 			.SectionAlignment = EFI_IMAGE_ALIGN,
 			.FileAlignment = EFI_FILE_ALIGN,
 			.SizeOfImage = EFI_IMAGE_ALIGN,
-			.SizeOfHeaders = sizeof ( efi_pe_header ),
+			.SizeOfHeaders =
+				( sizeof ( efi_pe_header ) -
+				  ( ( EFI_IMAGE_NUMBER_OF_DIRECTORY_ENTRIES -
+				      NUMBER_OF_DIRECTORY_ENTRIES ) *
+				    sizeof ( efi_pe_header.nt.OptionalHeader.
+					     DataDirectory[0] ) ) ),
 			.DllCharacteristics =
 				IMAGE_DLLCHARACTERISTICS_NX_COMPAT,
 			.NumberOfRvaAndSizes = NUMBER_OF_DIRECTORY_ENTRIES,

From 6714b20ea20af5046cdb36b58ab777b55a3c003c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 23 Nov 2023 14:17:36 +0000
Subject: [PATCH 029/237] [efi] Place PE debug information in a hidden section

The PE debug information generated by elf2efi is used only to hold the
image filename, and the debug information is located via the relevant
data directory entry rather than via the section table.

Make the .debug section a hidden section in order to save one entry in
the PE section list.  Choose to place the debug information in the
unused space at the end of the PE headers, since it no longer needs to
satisfy the general section alignment constraints.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 46 +++++++++++++++-------------------------------
 1 file changed, 15 insertions(+), 31 deletions(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 76d02f249..c98c4d4a9 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -185,7 +185,6 @@ struct elf_file {
 struct pe_section {
 	struct pe_section *next;
 	EFI_IMAGE_SECTION_HEADER hdr;
-	void ( * fixup ) ( struct pe_section *section );
 	int hidden;
 	uint8_t contents[0];
 };
@@ -919,20 +918,6 @@ create_reloc_section ( struct pe_header *pe_header,
 	return reloc;
 }
 
-/**
- * Fix up debug section
- *
- * @v debug		Debug section
- */
-static void fixup_debug_section ( struct pe_section *debug ) {
-	EFI_IMAGE_DEBUG_DIRECTORY_ENTRY *contents;
-
-	/* Fix up FileOffset */
-	contents = ( ( void * ) debug->contents );
-	contents->FileOffset += ( debug->hdr.PointerToRawData -
-				  debug->hdr.VirtualAddress );
-}
-
 /**
  * Create debug section
  *
@@ -952,24 +937,27 @@ create_debug_section ( struct pe_header *pe_header, const char *filename ) {
 	} *contents;
 
 	/* Allocate PE section */
-	section_memsz = sizeof ( *contents );
-	section_filesz = efi_file_align ( section_memsz );
+	section_filesz = section_memsz = sizeof ( *contents );
 	debug = xmalloc ( sizeof ( *debug ) + section_filesz );
 	memset ( debug, 0, sizeof ( *debug ) + section_filesz );
 	contents = ( void * ) debug->contents;
 
+	/* Place at end of headers */
+	pe_header->nt.OptionalHeader.SizeOfHeaders += sizeof ( *contents );
+	pe_header->nt.OptionalHeader.SizeOfHeaders =
+		efi_file_align ( pe_header->nt.OptionalHeader.SizeOfHeaders );
+	pe_header->nt.OptionalHeader.SizeOfHeaders -= sizeof ( *contents );
+
 	/* Fill in section header details */
 	strncpy ( ( char * ) debug->hdr.Name, ".debug",
 		  sizeof ( debug->hdr.Name ) );
 	debug->hdr.Misc.VirtualSize = section_memsz;
-	debug->hdr.VirtualAddress = pe_header->nt.OptionalHeader.SizeOfImage;
+	debug->hdr.VirtualAddress =
+		pe_header->nt.OptionalHeader.SizeOfHeaders;
 	debug->hdr.SizeOfRawData = section_filesz;
-	debug->hdr.PointerToRawData = PTRD_AUTO;
-	debug->hdr.Characteristics = ( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
-				       EFI_IMAGE_SCN_MEM_DISCARDABLE |
-				       EFI_IMAGE_SCN_MEM_NOT_PAGED |
-				       EFI_IMAGE_SCN_MEM_READ );
-	debug->fixup = fixup_debug_section;
+	debug->hdr.PointerToRawData =
+		pe_header->nt.OptionalHeader.SizeOfHeaders;
+	debug->hidden = 1;
 
 	/* Create section contents */
 	contents->debug.TimeDateStamp = 0x10d1a884;
@@ -984,10 +972,7 @@ create_debug_section ( struct pe_header *pe_header, const char *filename ) {
 		   filename );
 
 	/* Update file header details */
-	pe_header->nt.FileHeader.NumberOfSections++;
-	pe_header->nt.OptionalHeader.SizeOfHeaders += sizeof ( debug->hdr );
-	pe_header->nt.OptionalHeader.SizeOfImage +=
-		efi_image_align ( section_memsz );
+	pe_header->nt.OptionalHeader.SizeOfHeaders += sizeof ( *contents );
 	debugdir = &(pe_header->nt.OptionalHeader.DataDirectory
 		     [EFI_IMAGE_DIRECTORY_ENTRY_DEBUG]);
 	debugdir->VirtualAddress = debug->hdr.VirtualAddress;
@@ -1027,13 +1012,12 @@ static void write_pe_file ( struct pe_header *pe_header,
 		fpos = efi_file_align ( fpos );
 		if ( fpos > fposmax )
 			fposmax = fpos;
-		if ( section->fixup )
-			section->fixup ( section );
 	}
 
 	/* Write sections */
 	for ( section = pe_sections ; section ; section = section->next ) {
-		if ( section->hdr.PointerToRawData & ( EFI_FILE_ALIGN - 1 ) ) {
+		if ( ( section->hdr.PointerToRawData & ( EFI_FILE_ALIGN - 1 ) )
+		     && ( ! section->hidden ) ) {
 			eprintf ( "Section %.8s file offset %x is "
 				  "misaligned\n", section->hdr.Name,
 				  section->hdr.PointerToRawData );

From 18582a05fc65fc8835220c27528ef8902f5da60e Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 24 Nov 2023 12:16:49 +0000
Subject: [PATCH 030/237] [efi] Treat 16-bit sections as hidden in hybrid
 binaries

Hybrid bzImage and UEFI binaries (such as wimboot) may include 16-bit
sections such as .bss16 that do not need to consume an entry in the PE
section list.  Treat any such sections as hidden.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index c98c4d4a9..171e2b58c 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -653,6 +653,12 @@ static struct pe_section * process_section ( struct elf_file *elf,
 		}
 	}
 
+	/* Treat 16-bit sections as hidden in hybrid binaries */
+	if ( opts->hybrid && ( strlen ( name ) > 2 ) &&
+	     ( strcmp ( &name[ strlen ( name ) - 2 ], "16" ) == 0 ) ) {
+		new->hidden = 1;
+	}
+
 	/* Fill in section characteristics and update RVA limits */
 	if ( ( shdr->sh_type == SHT_PROGBITS ) &&
 	     ( shdr->sh_flags & SHF_WRITE ) ) {

From 03ff1bb99a28230397fb583853ab2160ff227e77 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 24 Nov 2023 15:55:41 +0000
Subject: [PATCH 031/237] [efi] Do not assume canonical PE section ordering

The BaseOfCode (and, in PE32, BaseOfData) fields imply an assumption
that binaries are laid out as code followed by initialised data
followed by uninitialised data.  This assumption may not be valid for
complex binaries such as wimboot.

Remove this implicit assumption, and use arguably justifiable values
for the assorted summary start and size fields within the PE headers.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 93 ++++++++++++++++++----------------------------
 1 file changed, 37 insertions(+), 56 deletions(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 171e2b58c..72d50d386 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -602,31 +602,29 @@ static struct pe_section * process_section ( struct elf_file *elf,
 	size_t name_len;
 	size_t section_memsz;
 	size_t section_filesz;
-	unsigned long code_start;
-	unsigned long code_end;
-	unsigned long data_start;
-	unsigned long data_mid;
-	unsigned long data_end;
-	unsigned long start;
-	unsigned long end;
-	unsigned long *applicable_start;
-	unsigned long *applicable_end;
+	uint32_t start;
+	uint32_t end;
+	uint32_t *code_start;
+	uint32_t *data_start;
+	uint32_t *code_size;
+	uint32_t *data_size;
+	uint32_t *bss_size;
+	uint32_t *applicable_start;
+	uint32_t *applicable_size;
 
 	/* Get section name */
 	name = elf_string ( elf, elf->ehdr->e_shstrndx, shdr->sh_name );
 
-	/* Extract current RVA limits from file header */
-	code_start = pe_header->nt.OptionalHeader.BaseOfCode;
-	code_end = ( code_start + pe_header->nt.OptionalHeader.SizeOfCode );
+	/* Identify start and size limit fields from file header */
+	code_start = &pe_header->nt.OptionalHeader.BaseOfCode;
+	code_size = &pe_header->nt.OptionalHeader.SizeOfCode;
 #if defined(EFI_TARGET32)
-	data_start = pe_header->nt.OptionalHeader.BaseOfData;
+	data_start = &pe_header->nt.OptionalHeader.BaseOfData;
 #elif defined(EFI_TARGET64)
-	data_start = code_end;
+	data_start = NULL;
 #endif
-	data_mid = ( data_start +
-		     pe_header->nt.OptionalHeader.SizeOfInitializedData );
-	data_end = ( data_mid +
-		     pe_header->nt.OptionalHeader.SizeOfUninitializedData );
+	data_size = &pe_header->nt.OptionalHeader.SizeOfInitializedData;
+	bss_size = &pe_header->nt.OptionalHeader.SizeOfUninitializedData;
 
 	/* Allocate PE section */
 	section_memsz = shdr->sh_size;
@@ -659,7 +657,7 @@ static struct pe_section * process_section ( struct elf_file *elf,
 		new->hidden = 1;
 	}
 
-	/* Fill in section characteristics and update RVA limits */
+	/* Fill in section characteristics and identify applicable limits */
 	if ( ( shdr->sh_type == SHT_PROGBITS ) &&
 	     ( shdr->sh_flags & SHF_WRITE ) ) {
 		/* .data-type section */
@@ -668,8 +666,8 @@ static struct pe_section * process_section ( struct elf_file *elf,
 			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
 			  EFI_IMAGE_SCN_MEM_READ |
 			  EFI_IMAGE_SCN_MEM_WRITE );
-		applicable_start = &data_start;
-		applicable_end = &data_mid;
+		applicable_start = data_start;
+		applicable_size = data_size;
 	} else if ( ( shdr->sh_type == SHT_PROGBITS ) &&
 		    ( shdr->sh_flags & SHF_EXECINSTR ) ) {
 		/* .text-type section */
@@ -678,16 +676,16 @@ static struct pe_section * process_section ( struct elf_file *elf,
 			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
 			  EFI_IMAGE_SCN_MEM_EXECUTE |
 			  EFI_IMAGE_SCN_MEM_READ );
-		applicable_start = &code_start;
-		applicable_end = &code_end;
+		applicable_start = code_start;
+		applicable_size = code_size;
 	} else if ( shdr->sh_type == SHT_PROGBITS ) {
 		/* .rodata-type section */
 		new->hdr.Characteristics =
 			( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
 			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
 			  EFI_IMAGE_SCN_MEM_READ );
-		applicable_start = &data_start;
-		applicable_end = &data_mid;
+		applicable_start = data_start;
+		applicable_size = data_size;
 	} else if ( shdr->sh_type == SHT_NOBITS ) {
 		/* .bss-type section */
 		new->hdr.Characteristics =
@@ -695,8 +693,8 @@ static struct pe_section * process_section ( struct elf_file *elf,
 			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
 			  EFI_IMAGE_SCN_MEM_READ |
 			  EFI_IMAGE_SCN_MEM_WRITE );
-		applicable_start = &data_mid;
-		applicable_end = &data_end;
+		applicable_start = data_start;
+		applicable_size = bss_size;
 	} else {
 		eprintf ( "Unrecognised characteristics for section %s\n",
 			  name );
@@ -709,41 +707,24 @@ static struct pe_section * process_section ( struct elf_file *elf,
 			 shdr->sh_size );
 	}
 
-	/* Update RVA limits */
+	/* Update file header fields */
 	start = new->hdr.VirtualAddress;
-	end = ( start + new->hdr.Misc.VirtualSize );
-	if ( ! new->hidden ) {
-		if ( ( ! *applicable_start ) || ( *applicable_start >= start ) )
-			*applicable_start = start;
-		if ( *applicable_end < end )
-			*applicable_end = end;
-	}
-	if ( data_start < code_end )
-		data_start = code_end;
-	if ( data_mid < data_start )
-		data_mid = data_start;
-	if ( data_end < data_mid )
-		data_end = data_mid;
-
-	/* Write RVA limits back to file header */
-	pe_header->nt.OptionalHeader.BaseOfCode = code_start;
-	pe_header->nt.OptionalHeader.SizeOfCode = ( code_end - code_start );
-#if defined(EFI_TARGET32)
-	pe_header->nt.OptionalHeader.BaseOfData = data_start;
-#endif
-	pe_header->nt.OptionalHeader.SizeOfInitializedData =
-		( data_mid - data_start );
-	pe_header->nt.OptionalHeader.SizeOfUninitializedData =
-		( data_end - data_mid );
-
-	/* Update remaining file header fields */
 	if ( ! new->hidden ) {
 		pe_header->nt.FileHeader.NumberOfSections++;
 		pe_header->nt.OptionalHeader.SizeOfHeaders +=
 			sizeof ( new->hdr );
+		if ( applicable_start && ( ( *applicable_start == 0 ) ||
+					   ( start < *applicable_start ) ) ) {
+			*applicable_start = start;
+		}
+		if ( applicable_size ) {
+			*applicable_size += section_memsz;
+		}
+	}
+	end = efi_image_align ( start + section_memsz );
+	if ( end > pe_header->nt.OptionalHeader.SizeOfImage ) {
+		pe_header->nt.OptionalHeader.SizeOfImage = end;
 	}
-	pe_header->nt.OptionalHeader.SizeOfImage =
-		efi_image_align ( data_end );
 
 	return new;
 }

From b829b1750de041f7d4fd0f4f86fbf968bdccda6a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 24 Nov 2023 12:26:43 +0000
Subject: [PATCH 032/237] [efi] Maximise image base address

iPXE images are linked with a starting virtual address of zero.  Other
images (such as wimboot) may use a non-zero starting virtual address.

There is no direct equivalent of the PE ImageBase address field within
ELF object files.  Choose to use the highest possible address that
accommodates all sections and the PE header itself, since this will
minimise the memory allocated to hold the loaded image.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 72d50d386..e97601390 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -729,6 +729,47 @@ static struct pe_section * process_section ( struct elf_file *elf,
 	return new;
 }
 
+/**
+ * Update image base address
+ *
+ * @v pe_header		PE file header
+ * @v pe_sections	List of PE sections
+ * @v pe_reltab		PE relocation table
+ */
+static void update_image_base ( struct pe_header *pe_header,
+				struct pe_section *pe_sections,
+				struct pe_relocs *pe_reltab ) {
+	struct pe_section *section;
+	struct pe_relocs *pe_rel;
+	unsigned long base = -1UL;
+
+	/* Set ImageBase to the highest possible value, leaving space
+	 * for the PE header itself.
+	 */
+	for ( section = pe_sections ; section ; section = section->next ) {
+		if ( ! section->hidden ) {
+			if ( base > section->hdr.VirtualAddress )
+				base = section->hdr.VirtualAddress;
+		}
+	}
+	base -= EFI_IMAGE_ALIGN;
+	pe_header->nt.OptionalHeader.ImageBase = base;
+
+	/* Adjust RVAs to match ImageBase */
+	pe_header->nt.OptionalHeader.AddressOfEntryPoint -= base;
+	pe_header->nt.OptionalHeader.BaseOfCode -= base;
+#if defined(EFI_TARGET32)
+	pe_header->nt.OptionalHeader.BaseOfData -= base;
+#endif
+	pe_header->nt.OptionalHeader.SizeOfImage -= base;
+	for ( section = pe_sections ; section ; section = section->next ) {
+		section->hdr.VirtualAddress -= base;
+	}
+	for ( pe_rel = pe_reltab ; pe_rel ; pe_rel = pe_rel->next ) {
+		pe_rel->start_rva -= base;
+	}
+}
+
 /**
  * Process relocation record
  *
@@ -1113,6 +1154,9 @@ static void elf2pe ( const char *elf_name, const char *pe_name,
 		}
 	}
 
+	/* Update image base address */
+	update_image_base ( &pe_header, pe_sections, pe_reltab );
+
 	/* Create the .reloc section */
 	*(next_pe_section) = create_reloc_section ( &pe_header, pe_reltab );
 	next_pe_section = &(*next_pe_section)->next;

From c3dd3168c916f624af1b843515c1305387060611 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 24 Nov 2023 22:26:50 +0000
Subject: [PATCH 033/237] [efi] Fix dependency list construction in EDK2 header
 import script

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/import.pl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/include/ipxe/efi/import.pl b/src/include/ipxe/efi/import.pl
index 34aed9a28..0a7669f43 100755
--- a/src/include/ipxe/efi/import.pl
+++ b/src/include/ipxe/efi/import.pl
@@ -68,7 +68,7 @@ sub try_import_file {
 	chomp;
 	# Update include lines, and record included files
 	if ( s/^(\s*\#include\s+)[<\"](\S+)[>\"]/$1<ipxe\/efi\/$2>/ ) {
-	  push @dependencies, $1;
+	  push @dependencies, $2;
 	}
 	# Check for BSD licence statement
 	if ( /^\s*SPDX-License-Identifier: BSD-2-Clause-Patent$/ ) {

From a147245f1a2f92a85a75226ea921acb22322ab4d Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 27 Nov 2023 12:08:19 +0000
Subject: [PATCH 034/237] [efi] Extend PE header size to cover space up to
 first section

Hybrid bzImage and UEFI binaries (such as wimboot) may place sections
at explicit offsets within the PE file, as described in commit b30a098
("[efi] Use load memory address as file offset for hybrid binaries").
This can leave a gap after the PE headers that is not covered by any
section.  It is not entirely clear whether or not such gaps are
permitted in binaries submitted for Secure Boot signing.

To minimise potential problems, extend the PE header size to cover any
space before the first explicitly placed section.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index e97601390..a3aff8f18 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -1020,13 +1020,34 @@ static void write_pe_file ( struct pe_header *pe_header,
 			    struct pe_section *pe_sections,
 			    FILE *pe ) {
 	struct pe_section *section;
-	unsigned long fpos = 0;
-	unsigned long fposmax = 0;
+	unsigned long hdrmax;
+	unsigned long fpos;
+	unsigned long fposmax;
 	unsigned int count = 0;
 
+	/* Extend header length to reach first explicitly placed section */
+	hdrmax = -1UL;
+	for ( section = pe_sections ; section ; section = section->next ) {
+		if ( ( section->hdr.PointerToRawData != PTRD_AUTO ) &&
+		     ( section->hdr.SizeOfRawData > 0 ) &&
+		     ( ! section->hidden ) &&
+		     ( hdrmax > section->hdr.PointerToRawData ) ) {
+			hdrmax = section->hdr.PointerToRawData;
+		}
+	}
+	if ( ( hdrmax != -1UL ) &&
+	     ( pe_header->nt.OptionalHeader.SizeOfHeaders < hdrmax ) ) {
+		pe_header->nt.OptionalHeader.SizeOfHeaders = hdrmax;
+	}
+
 	/* Align length of headers */
 	fpos = fposmax = pe_header->nt.OptionalHeader.SizeOfHeaders =
 		efi_file_align ( pe_header->nt.OptionalHeader.SizeOfHeaders );
+	if ( fpos > hdrmax ) {
+		eprintf ( "Cannot fit %lx bytes of headers before section at "
+			  "file offset %lx\n", fpos, hdrmax );
+		exit ( 1 );
+	}
 
 	/* Assign raw data pointers */
 	for ( section = pe_sections ; section ; section = section->next ) {

From 98dd25a3bb2d3aafa71f088cbabf89418a783132 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 29 Nov 2023 12:49:06 +0000
Subject: [PATCH 035/237] [efi] Avoid modifying PE/COFF debug filename

The function efi_pecoff_debug_name() (called by efi_handle_name()) is
used to extract a filename from the debug data directory entry located
within a PE/COFF image.  The name is copied into a temporary static
buffer to allow for modifications, but the code currently erroneously
modifies the original name within the loaded PE/COFF image.

Fix by performing the modification on the copy in the temporary
buffer, as originally intended.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_debug.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/interface/efi/efi_debug.c b/src/interface/efi/efi_debug.c
index 1372776fe..ad79f0d37 100644
--- a/src/interface/efi/efi_debug.c
+++ b/src/interface/efi/efi_debug.c
@@ -665,10 +665,10 @@ efi_pecoff_debug_name ( EFI_LOADED_IMAGE_PROTOCOL *loaded ) {
 	snprintf ( buf, sizeof ( buf ), "%s", name );
 
 	/* Strip file suffix, if present */
-	if ( ( tmp = strrchr ( name, '.' ) ) != NULL )
+	if ( ( tmp = strrchr ( buf, '.' ) ) != NULL )
 		*tmp = '\0';
 
-	return name;
+	return buf;
 }
 
 /**

From f22879ca994aedd2667bb0c73ebce505bf5f8cef Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 5 Dec 2023 15:49:17 +0000
Subject: [PATCH 036/237] [efi] Allow compiling elf2efi with clang

The clang compiler does not (and apparently will not ever) allow for
variable-length arrays within structs.

Work around this limitation by using a fixed-length array to hold the
PDB filename in the debug section.

This mirrors wimboot commit f52c3ff ("[efi] Allow compiling elf2efi
with clang").

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index a3aff8f18..acde75c08 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -961,7 +961,7 @@ create_debug_section ( struct pe_header *pe_header, const char *filename ) {
 	struct {
 		EFI_IMAGE_DEBUG_DIRECTORY_ENTRY debug;
 		EFI_IMAGE_DEBUG_CODEVIEW_RSDS_ENTRY rsds;
-		char name[ strlen ( filename ) + 1 ];
+		char name[32];
 	} *contents;
 
 	/* Allocate PE section */

From 337880deaa7ea5d86e5c7de63faa1c976d6114cf Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 19 Dec 2023 14:39:36 +0000
Subject: [PATCH 037/237] [build] Use SOURCE_DATE_EPOCH for FAT serial number
 if it exists

Reported-by: Bernhard M. Wiedemann <bwiedemann@suse.de>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/genfsimg | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/util/genfsimg b/src/util/genfsimg
index 0c0692793..a981a62d8 100755
--- a/src/util/genfsimg
+++ b/src/util/genfsimg
@@ -269,6 +269,10 @@ if [ -n "${FATIMG}" ] ; then
 	FATSIZE=$(( FATCYLS * 504 ))
 	FATARGS="-s 63 -h 16 -t ${FATCYLS}"
     fi
+    if [ -n "${SOURCE_DATE_EPOCH:-}" ] ; then
+	FATSERIAL=$(( SOURCE_DATE_EPOCH % 100000000 ))
+	FATARGS="${FATARGS} -N ${FATSERIAL}"
+    fi
     truncate -s "${FATSIZE}K" "${FATIMG}"
     mformat -v iPXE -i "${FATIMG}" ${FATARGS} ::
     mcopy -i "${FATIMG}" -s "${FATDIR}"/* ::

From 0958e01463f7ad6a532939e77712ed31b4a12dbc Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 19 Dec 2023 16:29:42 +0000
Subject: [PATCH 038/237] [efi] Add relocation types generated by clang

Add additional PC-relative relocation types that may be encountered
when converting binaries compiled with clang.

This mirrors the relevant elf2efi portions of wimboot commit 7910830
("[build] Support building with the clang compiler").

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index acde75c08..3bf6cbf9e 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -828,6 +828,8 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_ARM, R_ARM_V4BX ):
 		case ELF_MREL ( EM_X86_64, R_X86_64_PC32 ) :
 		case ELF_MREL ( EM_X86_64, R_X86_64_PLT32 ) :
+		case ELF_MREL ( EM_X86_64, R_X86_64_GOTPCRELX ) :
+		case ELF_MREL ( EM_X86_64, R_X86_64_REX_GOTPCRELX ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_CALL26 ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_JUMP26 ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_ADR_PREL_LO21 ) :
@@ -837,6 +839,7 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST16_ABS_LO12_NC ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST32_ABS_LO12_NC ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST64_ABS_LO12_NC ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST128_ABS_LO12_NC ) :
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_B26):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCALA_HI20 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCALA_LO12 ):

From 3fc1b407d204e950a41c567e84f4c4f079cc052e Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 19 Dec 2023 16:56:34 +0000
Subject: [PATCH 039/237] [efi] Fix Coverity warning about unintended sign
 extension

The result of multiplying a uint16_t by another uint16_t will be a
signed int.  Comparing this against a size_t will perform an unwanted
sign extension.

Fix by explicitly casting e_phnum to an unsigned int, thereby matching
the data type used for the loop index variable (and avoiding the
unwanted sign extension).

This mirrors wimboot commit 15f6162 ("[efi] Fix Coverity warning about
unintended sign extension").

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 3bf6cbf9e..29e4f24ab 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -440,8 +440,8 @@ static void read_elf_file ( const char *name, struct elf_file *elf ) {
 
 	/* Check program headers */
 	if ( ( elf->len < ehdr->e_phoff ) ||
-	     ( ( elf->len - ehdr->e_phoff ) < ( ehdr->e_phnum *
-						ehdr->e_phentsize ) ) ) {
+	     ( ( elf->len - ehdr->e_phoff ) <
+	       ( ( ( unsigned int ) ehdr->e_phnum ) * ehdr->e_phentsize ) ) ) {
 		eprintf ( "ELF program headers outside file in %s\n", name );
 		exit ( 1 );
 	}

From 9e92c39894610c825a4cd6ccb40229fdeaa7223f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 21 Dec 2023 12:29:03 +0000
Subject: [PATCH 040/237] [efi] Add potentially missing relocation types

Add definitions for relocation types that may be missing on older
versions of the host system's elf.h.

This mirrors wimboot commit 47f6298 ("[efi] Add potentially missing
relocation types").

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 29e4f24ab..88713b663 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -122,6 +122,9 @@
 #ifndef R_AARCH64_LDST64_ABS_LO12_NC
 #define R_AARCH64_LDST64_ABS_LO12_NC 286
 #endif
+#ifndef R_AARCH64_LDST128_ABS_LO12_NC
+#define R_AARCH64_LDST128_ABS_LO12_NC 299
+#endif
 #ifndef R_ARM_CALL
 #define R_ARM_CALL 28
 #endif
@@ -152,6 +155,12 @@
 #ifndef R_LARCH_GOT_PC_LO12
 #define R_LARCH_GOT_PC_LO12 76
 #endif
+#ifndef R_X86_64_GOTPCRELX
+#define R_X86_64_GOTPCRELX 41
+#endif
+#ifndef R_X86_64_REX_GOTPCRELX
+#define R_X86_64_REX_GOTPCRELX 42
+#endif
 
 /**
  * Alignment of raw data of sections in the image file

From 119c415ee47aaef2717104fea493377aa9a65874 Mon Sep 17 00:00:00 2001
From: Christian Helmuth <christian.helmuth@genode-labs.com>
Date: Thu, 21 Dec 2023 13:51:26 +0100
Subject: [PATCH 041/237] [intel] Add PCI ID for I219-LM (23)

Successfully tested on FUJITSU LIFEBOOK U7413.

Signed-off-by: Christian Helmuth <christian.helmuth@genode-labs.com>
---
 src/drivers/net/intel.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/drivers/net/intel.c b/src/drivers/net/intel.c
index 46527bdbb..7879714f6 100644
--- a/src/drivers/net/intel.c
+++ b/src/drivers/net/intel.c
@@ -1043,6 +1043,7 @@ static struct pci_device_id intel_nics[] = {
 	PCI_ROM ( 0x8086, 0x0d4f, "i219v-10", "I219-V (10)", INTEL_I219 ),
 	PCI_ROM ( 0x8086, 0x0d53, "i219lm-12", "I219-LM (12)", INTEL_I219 ),
 	PCI_ROM ( 0x8086, 0x0d55, "i219v-12", "I219-V (12)", INTEL_I219 ),
+	PCI_ROM ( 0x8086, 0x0dc5, "i219lm-23", "I219-LM (23)", INTEL_I219 ),
 	PCI_ROM ( 0x8086, 0x1000, "82542-f", "82542 (Fiber)", 0 ),
 	PCI_ROM ( 0x8086, 0x1001, "82543gc-f", "82543GC (Fiber)", 0 ),
 	PCI_ROM ( 0x8086, 0x1004, "82543gc", "82543GC (Copper)", 0 ),

From fa62213231a882eb6bbcefa7ad1106bdb9aaeae2 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Dec 2023 19:38:12 +0000
Subject: [PATCH 042/237] [smbios] Support scanning for the 64-bit SMBIOS3
 entry point

Support scanning for the 64-bit SMBIOS3 entry point in addition to the
32-bit SMBIOS2 entry point.

Prefer use of the 32-bit entry point if present, since this is
guaranteed to be within accessible memory.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/interface/pcbios/bios_smbios.c | 55 ++++++++++++++-
 src/include/ipxe/smbios.h                   |  2 +
 src/interface/smbios/smbios.c               | 78 +++++++++++++++++----
 3 files changed, 120 insertions(+), 15 deletions(-)

diff --git a/src/arch/x86/interface/pcbios/bios_smbios.c b/src/arch/x86/interface/pcbios/bios_smbios.c
index a8c0fc325..366679d36 100644
--- a/src/arch/x86/interface/pcbios/bios_smbios.c
+++ b/src/arch/x86/interface/pcbios/bios_smbios.c
@@ -44,11 +44,11 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v smbios		SMBIOS entry point descriptor structure to fill in
  * @ret rc		Return status code
  */
-static int bios_find_smbios ( struct smbios *smbios ) {
+static int bios_find_smbios2 ( struct smbios *smbios ) {
 	struct smbios_entry entry;
 	int rc;
 
-	/* Scan through BIOS segment to find SMBIOS entry point */
+	/* Scan through BIOS segment to find SMBIOS 32-bit entry point */
 	if ( ( rc = find_smbios_entry ( real_to_user ( BIOS_SEG, 0 ), 0x10000,
 					&entry ) ) != 0 )
 		return rc;
@@ -62,4 +62,55 @@ static int bios_find_smbios ( struct smbios *smbios ) {
 	return 0;
 }
 
+/**
+ * Find SMBIOS
+ *
+ * @v smbios		SMBIOS entry point descriptor structure to fill in
+ * @ret rc		Return status code
+ */
+static int bios_find_smbios3 ( struct smbios *smbios ) {
+	struct smbios3_entry entry;
+	int rc;
+
+	/* Scan through BIOS segment to find SMBIOS 64-bit entry point */
+	if ( ( rc = find_smbios3_entry ( real_to_user ( BIOS_SEG, 0 ), 0x10000,
+					 &entry ) ) != 0 )
+		return rc;
+
+	/* Check that address is accessible */
+	if ( entry.smbios_address > ~( ( physaddr_t ) 0 ) ) {
+		DBG ( "SMBIOS3 at %08llx is inaccessible\n",
+		      ( ( unsigned long long ) entry.smbios_address ) );
+		return -ENOTSUP;
+	}
+
+	/* Fill in entry point descriptor structure */
+	smbios->address = phys_to_user ( entry.smbios_address );
+	smbios->len = entry.smbios_len;
+	smbios->count = 0;
+	smbios->version = SMBIOS_VERSION ( entry.major, entry.minor );
+
+	return 0;
+}
+
+/**
+ * Find SMBIOS
+ *
+ * @v smbios		SMBIOS entry point descriptor structure to fill in
+ * @ret rc		Return status code
+ */
+static int bios_find_smbios ( struct smbios *smbios ) {
+	int rc;
+
+	/* Use 32-bit table if present */
+	if ( ( rc = bios_find_smbios2 ( smbios ) ) == 0 )
+		return 0;
+
+	/* Otherwise, use 64-bit table if present and accessible */
+	if ( ( rc = bios_find_smbios3 ( smbios ) ) == 0 )
+		return 0;
+
+	return rc;
+}
+
 PROVIDE_SMBIOS ( pcbios, find_smbios, bios_find_smbios );
diff --git a/src/include/ipxe/smbios.h b/src/include/ipxe/smbios.h
index 42278fb24..077a67a85 100644
--- a/src/include/ipxe/smbios.h
+++ b/src/include/ipxe/smbios.h
@@ -227,6 +227,8 @@ struct smbios {
 extern int find_smbios ( struct smbios *smbios );
 extern int find_smbios_entry ( userptr_t start, size_t len,
 			       struct smbios_entry *entry );
+extern int find_smbios3_entry ( userptr_t start, size_t len,
+				struct smbios3_entry *entry );
 extern int find_smbios_structure ( unsigned int type, unsigned int instance,
 				   struct smbios_structure *structure );
 extern int read_smbios_structure ( struct smbios_structure *structure,
diff --git a/src/interface/smbios/smbios.c b/src/interface/smbios/smbios.c
index 12a080da2..fdd14499f 100644
--- a/src/interface/smbios/smbios.c
+++ b/src/interface/smbios/smbios.c
@@ -42,7 +42,27 @@ static struct smbios smbios = {
 };
 
 /**
- * Scan for SMBIOS entry point structure
+ * Calculate SMBIOS entry point structure checksum
+ *
+ * @v start		Start address of region
+ * @v offset		Offset of SMBIOS entry point structure
+ * @v len		Length of entry point structure
+ * @ret sum		Byte checksum
+ */
+static uint8_t smbios_checksum ( userptr_t start, size_t offset, size_t len ) {
+	size_t end = ( offset + len );
+	uint8_t sum;
+	uint8_t byte;
+
+	for ( sum = 0 ; offset < end ; offset++ ) {
+		copy_from_user ( &byte, start, offset, sizeof ( byte ) );
+		sum += byte;
+	}
+	return sum;
+}
+
+/**
+ * Scan for SMBIOS 32-bit entry point structure
  *
  * @v start		Start address of region to scan
  * @v len		Length of region to scan
@@ -51,28 +71,20 @@ static struct smbios smbios = {
  */
 int find_smbios_entry ( userptr_t start, size_t len,
 			struct smbios_entry *entry ) {
-	uint8_t buf[256]; /* 256 is maximum length possible */
 	static size_t offset = 0; /* Avoid repeated attempts to locate SMBIOS */
-	size_t entry_len;
-	unsigned int i;
 	uint8_t sum;
 
 	/* Try to find SMBIOS */
-	for ( ; offset < len ; offset += 0x10 ) {
+	for ( ; ( offset + sizeof ( *entry ) ) <= len ; offset += 0x10 ) {
 
 		/* Read start of header and verify signature */
 		copy_from_user ( entry, start, offset, sizeof ( *entry ) );
 		if ( entry->signature != SMBIOS_SIGNATURE )
 			continue;
 
-		/* Read whole header and verify checksum */
-		entry_len = entry->len;
-		assert ( entry_len <= sizeof ( buf ) );
-		copy_from_user ( buf, start, offset, entry_len );
-		for ( i = 0, sum = 0 ; i < entry_len ; i++ ) {
-			sum += buf[i];
-		}
-		if ( sum != 0 ) {
+		/* Verify checksum */
+		if ( ( sum = smbios_checksum ( start, offset,
+					       entry->len ) ) != 0 ) {
 			DBG ( "SMBIOS at %08lx has bad checksum %02x\n",
 			      user_to_phys ( start, offset ), sum );
 			continue;
@@ -89,6 +101,46 @@ int find_smbios_entry ( userptr_t start, size_t len,
 	return -ENODEV;
 }
 
+/**
+ * Scan for SMBIOS 64-bit entry point structure
+ *
+ * @v start		Start address of region to scan
+ * @v len		Length of region to scan
+ * @v entry		SMBIOS entry point structure to fill in
+ * @ret rc		Return status code
+ */
+int find_smbios3_entry ( userptr_t start, size_t len,
+			 struct smbios3_entry *entry ) {
+	static size_t offset = 0; /* Avoid repeated attempts to locate SMBIOS */
+	uint8_t sum;
+
+	/* Try to find SMBIOS */
+	for ( ; ( offset + sizeof ( *entry ) ) <= len ; offset += 0x10 ) {
+
+		/* Read start of header and verify signature */
+		copy_from_user ( entry, start, offset, sizeof ( *entry ) );
+		if ( entry->signature != SMBIOS3_SIGNATURE )
+			continue;
+
+		/* Verify checksum */
+		if ( ( sum = smbios_checksum ( start, offset,
+					       entry->len ) ) != 0 ) {
+			DBG ( "SMBIOS3 at %08lx has bad checksum %02x\n",
+			      user_to_phys ( start, offset ), sum );
+			continue;
+		}
+
+		/* Fill result structure */
+		DBG ( "Found SMBIOS3 v%d.%d entry point at %08lx\n",
+		      entry->major, entry->minor,
+		      user_to_phys ( start, offset ) );
+		return 0;
+	}
+
+	DBG ( "No SMBIOS3 found\n" );
+	return -ENODEV;
+}
+
 /**
  * Find SMBIOS strings terminator
  *

From 4ed7a5718f29e59c19d6bef51dc3430861c9da87 Mon Sep 17 00:00:00 2001
From: Mark Rogalski <rogalski@us.ibm.com>
Date: Mon, 8 Jan 2024 10:03:12 -0600
Subject: [PATCH 043/237] [build] Reduce scope of wildcard .gitignore rules

Ensure that .gitignore rules do not cover any files that do exist
within the repository.

Modified-by: Michael Brown <mcb30@ipxe.org>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/.gitignore              | 2 +-
 src/bin/.gitignore          | 1 +
 src/config/local/.gitignore | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/.gitignore b/src/.gitignore
index cc8e33e28..4e4f00c81 100644
--- a/src/.gitignore
+++ b/src/.gitignore
@@ -1,4 +1,4 @@
 .toolcheck
 .echocheck
 TAGS*
-bin*
+bin-*
diff --git a/src/bin/.gitignore b/src/bin/.gitignore
index 72e8ffc0d..d6b7ef32c 100644
--- a/src/bin/.gitignore
+++ b/src/bin/.gitignore
@@ -1 +1,2 @@
 *
+!.gitignore
diff --git a/src/config/local/.gitignore b/src/config/local/.gitignore
index 72e8ffc0d..d6b7ef32c 100644
--- a/src/config/local/.gitignore
+++ b/src/config/local/.gitignore
@@ -1 +1,2 @@
 *
+!.gitignore

From 0abb3e85e518c4d9366d4555093c0aff0c060858 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 10 Jan 2024 15:28:59 +0000
Subject: [PATCH 044/237] [eap] Ignore any received EAP responses

EAP responses (including our own) may be broadcast by switches but are
not of interest to us and can be safely ignored if received.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/eap.h | 3 +++
 src/net/eap.c          | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/src/include/ipxe/eap.h b/src/include/ipxe/eap.h
index 818862a94..4b689cc24 100644
--- a/src/include/ipxe/eap.h
+++ b/src/include/ipxe/eap.h
@@ -26,6 +26,9 @@ struct eap_header {
 /** EAP request */
 #define EAP_CODE_REQUEST 1
 
+/** EAP response */
+#define EAP_CODE_RESPONSE 2
+
 /** EAP request */
 struct eap_request {
 	/** Header */
diff --git a/src/net/eap.c b/src/net/eap.c
index 2c68b75d4..8ba87e292 100644
--- a/src/net/eap.c
+++ b/src/net/eap.c
@@ -149,6 +149,9 @@ int eap_rx ( struct eap_supplicant *supplicant, const void *data,
 	switch ( eap->hdr.code ) {
 	case EAP_CODE_REQUEST:
 		return eap_rx_request ( supplicant, &eap->req, len );
+	case EAP_CODE_RESPONSE:
+		DBGC2 ( netdev, "EAP %s ignoring response\n", netdev->name );
+		return 0;
 	case EAP_CODE_SUCCESS:
 		return eap_rx_success ( supplicant );
 	case EAP_CODE_FAILURE:

From c6226f104e22db7d19b4c983e962d84b5665c04b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 10 Jan 2024 15:30:36 +0000
Subject: [PATCH 045/237] [eap] Add support for sending an EAP identity

Allow the ${netX/username} setting to be used to specify an EAP
identity to be returned in response to a Request-Identity, and provide
a mechanism for responding with a NAK to indicate which authentication
types we support.

If no identity is specified then fall back to the current behaviour of
not sending any Request-Identity response, so that switches will time
out and switch to MAC Authentication Bypass (MAB) if applicable.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/eap.h |  45 ++++++++++--
 src/net/eap.c          | 159 +++++++++++++++++++++++++++++++++++------
 2 files changed, 178 insertions(+), 26 deletions(-)

diff --git a/src/include/ipxe/eap.h b/src/include/ipxe/eap.h
index 4b689cc24..bbae517da 100644
--- a/src/include/ipxe/eap.h
+++ b/src/include/ipxe/eap.h
@@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <stdint.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/timer.h>
+#include <ipxe/tables.h>
 
 /** EAP header */
 struct eap_header {
@@ -29,17 +30,25 @@ struct eap_header {
 /** EAP response */
 #define EAP_CODE_RESPONSE 2
 
-/** EAP request */
-struct eap_request {
+/** EAP request/response message */
+struct eap_message {
 	/** Header */
 	struct eap_header hdr;
 	/** Type */
 	uint8_t type;
+	/** Type data */
+	uint8_t data[0];
 } __attribute__ (( packed ));
 
+/** EAP "no available types" marker */
+#define EAP_TYPE_NONE 0
+
 /** EAP identity */
 #define EAP_TYPE_IDENTITY 1
 
+/** EAP NAK */
+#define EAP_TYPE_NAK 3
+
 /** EAP success */
 #define EAP_CODE_SUCCESS 3
 
@@ -50,8 +59,8 @@ struct eap_request {
 union eap_packet {
 	/** Header */
 	struct eap_header hdr;
-	/** Request */
-	struct eap_request req;
+	/** Request/response message */
+	struct eap_message msg;
 };
 
 /** EAP link block timeout
@@ -90,7 +99,11 @@ struct eap_supplicant {
 	/** Network device */
 	struct net_device *netdev;
 	/** Flags */
-	unsigned int flags;
+	uint16_t flags;
+	/** ID for current request/response */
+	uint8_t id;
+	/** Type for current request/response */
+	uint8_t type;
 	/**
 	 * Transmit EAP response
 	 *
@@ -120,6 +133,28 @@ struct eap_supplicant {
  */
 #define EAP_FL_PASSIVE 0x0002
 
+/** An EAP method */
+struct eap_method {
+	/** Type */
+	uint8_t type;
+	/**
+	 * Handle EAP request
+	 *
+	 * @v supplicant	EAP supplicant
+	 * @v req		Request type data
+	 * @v req_len		Length of request type data
+	 * @ret rc		Return status code
+	 */
+	int ( * rx ) ( struct eap_supplicant *supplicant,
+		       const void *req, size_t req_len );
+};
+
+/** EAP method table */
+#define EAP_METHODS __table ( struct eap_method, "eap_methods" )
+
+/** Declare an EAP method */
+#define __eap_method __table_entry ( EAP_METHODS, 01 )
+
 extern int eap_rx ( struct eap_supplicant *supplicant,
 		    const void *data, size_t len );
 
diff --git a/src/net/eap.c b/src/net/eap.c
index 8ba87e292..fe01f136e 100644
--- a/src/net/eap.c
+++ b/src/net/eap.c
@@ -23,7 +23,10 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stdlib.h>
 #include <errno.h>
+#include <string.h>
+#include <byteswap.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/eap.h>
 
@@ -34,59 +37,173 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 
 /**
- * Handle EAP Request-Identity
+ * Transmit EAP response
+ *
+ * @v supplicant	EAP supplicant
+ * @v rsp		Response type data
+ * @v rsp_len		Length of response type data
+ * @ret rc		Return status code
+ */
+static int eap_tx_response ( struct eap_supplicant *supplicant,
+			     const void *rsp, size_t rsp_len ) {
+	struct net_device *netdev = supplicant->netdev;
+	struct eap_message *msg;
+	size_t len;
+	int rc;
+
+	/* Allocate and populate response */
+	len = ( sizeof ( *msg ) + rsp_len );
+	msg = malloc ( len );
+	if ( ! msg ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	msg->hdr.code = EAP_CODE_RESPONSE;
+	msg->hdr.id = supplicant->id;
+	msg->hdr.len = htons ( len );
+	msg->type = supplicant->type;
+	memcpy ( msg->data, rsp, rsp_len );
+
+	/* Transmit response */
+	if ( ( rc = supplicant->tx ( supplicant, msg, len ) ) != 0 ) {
+		DBGC ( netdev, "EAP %s could not transmit: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_tx;
+	}
+
+ err_tx:
+	free ( msg );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Transmit EAP NAK
  *
  * @v supplicant	EAP supplicant
  * @ret rc		Return status code
  */
-static int eap_rx_request_identity ( struct eap_supplicant *supplicant ) {
+static int eap_tx_nak ( struct eap_supplicant *supplicant ) {
+	unsigned int max = table_num_entries ( EAP_METHODS );
+	uint8_t methods[ max + 1 /* potential EAP_TYPE_NONE */ ];
+	unsigned int count = 0;
+	struct eap_method *method;
+
+	/* Populate methods list */
+	for_each_table_entry ( method, EAP_METHODS ) {
+		if ( method->type > EAP_TYPE_NAK )
+			methods[count++] = method->type;
+	}
+	if ( ! count )
+		methods[count++] = EAP_TYPE_NONE;
+	assert ( count <= max );
+
+	/* Transmit response */
+	supplicant->type = EAP_TYPE_NAK;
+	return eap_tx_response ( supplicant, methods, count );
+}
+
+/**
+ * Handle EAP Request-Identity
+ *
+ * @v supplicant	EAP supplicant
+ * @v req		Request type data
+ * @v req_len		Length of request type data
+ * @ret rc		Return status code
+ */
+static int eap_rx_identity ( struct eap_supplicant *supplicant,
+			     const void *req, size_t req_len ) {
 	struct net_device *netdev = supplicant->netdev;
+	void *rsp;
+	int rsp_len;
+	int rc;
 
 	/* Treat Request-Identity as blocking the link */
 	DBGC ( netdev, "EAP %s Request-Identity blocking link\n",
 	       netdev->name );
+	DBGC_HDA ( netdev, 0, req, req_len );
 	netdev_link_block ( netdev, EAP_BLOCK_TIMEOUT );
 
 	/* Mark EAP as in progress */
 	supplicant->flags |= EAP_FL_ONGOING;
 
-	/* We have no identity to offer, so wait until the switch
-	 * times out and switches to MAC Authentication Bypass (MAB).
-	 */
-	supplicant->flags |= EAP_FL_PASSIVE;
+	/* Construct response, if applicable */
+	rsp_len = fetch_raw_setting_copy ( netdev_settings ( netdev ),
+					   &username_setting, &rsp );
+	if ( rsp_len < 0 ) {
+		/* We have no identity to offer, so wait until the
+		 * switch times out and switches to MAC Authentication
+		 * Bypass (MAB).
+		 */
+		DBGC2 ( netdev, "EAP %s has no identity\n", netdev->name );
+		supplicant->flags |= EAP_FL_PASSIVE;
+		rc = 0;
+		goto no_response;
+	}
 
-	return 0;
+	/* Transmit response */
+	if ( ( rc = eap_tx_response ( supplicant, rsp, rsp_len ) ) != 0 )
+		goto err_tx;
+
+ err_tx:
+	free ( rsp );
+ no_response:
+	return rc;
 }
 
+/** EAP Request-Identity method */
+struct eap_method eap_identity_method __eap_method = {
+	.type = EAP_TYPE_IDENTITY,
+	.rx = eap_rx_identity,
+};
+
 /**
  * Handle EAP Request
  *
  * @v supplicant	EAP supplicant
- * @v req		EAP request
+ * @v msg		EAP request
  * @v len		Length of EAP request
  * @ret rc		Return status code
  */
 static int eap_rx_request ( struct eap_supplicant *supplicant,
-			    const struct eap_request *req, size_t len ) {
+			    const struct eap_message *msg, size_t len ) {
 	struct net_device *netdev = supplicant->netdev;
+	struct eap_method *method;
+	const void *req;
+	size_t req_len;
 
-	/* Sanity check */
-	if ( len < sizeof ( *req ) ) {
+	/* Sanity checks */
+	if ( len < sizeof ( *msg ) ) {
 		DBGC ( netdev, "EAP %s underlength request:\n", netdev->name );
-		DBGC_HDA ( netdev, 0, req, len );
+		DBGC_HDA ( netdev, 0, msg, len );
 		return -EINVAL;
 	}
+	if ( len < ntohs ( msg->hdr.len ) ) {
+		DBGC ( netdev, "EAP %s truncated request:\n", netdev->name );
+		DBGC_HDA ( netdev, 0, msg, len );
+		return -EINVAL;
+	}
+	req = msg->data;
+	req_len = ( ntohs ( msg->hdr.len ) - sizeof ( *msg ) );
+
+	/* Record request details */
+	supplicant->id = msg->hdr.id;
+	supplicant->type = msg->type;
 
 	/* Handle according to type */
-	switch ( req->type ) {
-	case EAP_TYPE_IDENTITY:
-		return eap_rx_request_identity ( supplicant );
-	default:
-		DBGC ( netdev, "EAP %s requested type %d unknown:\n",
-		       netdev->name, req->type );
-		DBGC_HDA ( netdev, 0, req, len );
-		return -ENOTSUP;
+	for_each_table_entry ( method, EAP_METHODS ) {
+		if ( msg->type == method->type )
+			return method->rx ( supplicant, req, req_len );
 	}
+	DBGC ( netdev, "EAP %s requested type %d unknown:\n",
+	       netdev->name, msg->type );
+	DBGC_HDA ( netdev, 0, msg, len );
+
+	/* Send NAK if applicable */
+	if ( msg->type > EAP_TYPE_NAK )
+		return eap_tx_nak ( supplicant );
+
+	return -ENOTSUP;
 }
 
 /**
@@ -148,7 +265,7 @@ int eap_rx ( struct eap_supplicant *supplicant, const void *data,
 	/* Handle according to code */
 	switch ( eap->hdr.code ) {
 	case EAP_CODE_REQUEST:
-		return eap_rx_request ( supplicant, &eap->req, len );
+		return eap_rx_request ( supplicant, &eap->msg, len );
 	case EAP_CODE_RESPONSE:
 		DBGC2 ( netdev, "EAP %s ignoring response\n", netdev->name );
 		return 0;

From 08fcb0e8fba4ef1dc770253bfbb330fa0c02e096 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 10 Jan 2024 15:23:07 +0000
Subject: [PATCH 046/237] [eap] Add support for the MD5-Challenge
 authentication type

RFC 3748 states that support for MD5-Challenge is mandatory for EAP
implementations.  The MD5 and CHAP code is already included in the
default build since it is required by iSCSI, and so this does not
substantially increase the binary size.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/eap.h | 11 ++++++
 src/net/eap.c          | 80 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+)

diff --git a/src/include/ipxe/eap.h b/src/include/ipxe/eap.h
index bbae517da..cf1c7c00d 100644
--- a/src/include/ipxe/eap.h
+++ b/src/include/ipxe/eap.h
@@ -49,6 +49,17 @@ struct eap_message {
 /** EAP NAK */
 #define EAP_TYPE_NAK 3
 
+/** EAP MD5 challenge request/response */
+#define EAP_TYPE_MD5 4
+
+/** EAP MD5 challenge request/response type data */
+struct eap_md5 {
+	/** Value length */
+	uint8_t len;
+	/** Value */
+	uint8_t value[0];
+} __attribute__ (( packed ));
+
 /** EAP success */
 #define EAP_CODE_SUCCESS 3
 
diff --git a/src/net/eap.c b/src/net/eap.c
index fe01f136e..79b060aa6 100644
--- a/src/net/eap.c
+++ b/src/net/eap.c
@@ -28,6 +28,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <string.h>
 #include <byteswap.h>
 #include <ipxe/netdevice.h>
+#include <ipxe/md5.h>
+#include <ipxe/chap.h>
 #include <ipxe/eap.h>
 
 /** @file
@@ -157,6 +159,84 @@ struct eap_method eap_identity_method __eap_method = {
 	.rx = eap_rx_identity,
 };
 
+/**
+ * Handle EAP MD5-Challenge
+ *
+ * @v req		Request type data
+ * @v req_len		Length of request type data
+ * @ret rc		Return status code
+ */
+static int eap_rx_md5 ( struct eap_supplicant *supplicant,
+			const void *req, size_t req_len ) {
+	struct net_device *netdev = supplicant->netdev;
+	const struct eap_md5 *md5req = req;
+	struct {
+		uint8_t len;
+		uint8_t value[MD5_DIGEST_SIZE];
+	} __attribute__ (( packed )) md5rsp;
+	struct chap_response chap;
+	void *secret;
+	int secret_len;
+	int rc;
+
+	/* Sanity checks */
+	if ( req_len < sizeof ( *md5req ) ) {
+		DBGC ( netdev, "EAP %s underlength MD5-Challenge:\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+	if ( ( req_len - sizeof ( *md5req ) ) < md5req->len ) {
+		DBGC ( netdev, "EAP %s truncated MD5-Challenge:\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+
+	/* Construct response */
+	if ( ( rc = chap_init ( &chap, &md5_algorithm ) ) != 0 ) {
+		DBGC ( netdev, "EAP %s could not initialise CHAP: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_chap;
+	}
+	chap_set_identifier ( &chap, supplicant->id );
+	secret_len = fetch_raw_setting_copy ( netdev_settings ( netdev ),
+					      &password_setting, &secret );
+	if ( secret_len < 0 ) {
+		rc = secret_len;
+		DBGC ( netdev, "EAP %s has no secret: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_secret;
+	}
+	chap_update ( &chap, secret, secret_len );
+	chap_update ( &chap, md5req->value, md5req->len );
+	chap_respond ( &chap );
+	assert ( chap.response_len == sizeof ( md5rsp.value ) );
+	md5rsp.len = sizeof ( md5rsp.value );
+	memcpy ( md5rsp.value, chap.response, sizeof ( md5rsp.value ) );
+
+	/* Transmit response */
+	if ( ( rc = eap_tx_response ( supplicant, &md5rsp,
+				      sizeof ( md5rsp ) ) ) != 0 )
+		goto err_tx;
+
+ err_tx:
+	free ( secret );
+ err_secret:
+	chap_finish ( &chap );
+ err_chap:
+ err_sanity:
+	return rc;
+}
+
+/** EAP MD5-Challenge method */
+struct eap_method eap_md5_method __eap_method = {
+	.type = EAP_TYPE_MD5,
+	.rx = eap_rx_md5,
+};
+
 /**
  * Handle EAP Request
  *

From e66552eeede19a91b1a52468a550b58fd031777b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 14 Jan 2024 11:51:16 +0000
Subject: [PATCH 047/237] [build] Remove unnecessary ".text" directives

The .text directive is entirely redundant when followed by a .section
directive giving an explicit section name and attributes.

Remove these unnecessary directives to simplify the code.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/libgcc/lldivmod.S            | 1 -
 src/arch/arm32/libgcc/llshift.S             | 1 -
 src/arch/x86/core/patch_cf.S                | 1 -
 src/arch/x86/drivers/net/undiisr.S          | 1 -
 src/arch/x86/interface/pcbios/e820mangler.S | 1 -
 src/arch/x86/prefix/bootpart.S              | 1 -
 src/arch/x86/prefix/dskprefix.S             | 1 -
 src/arch/x86/prefix/exeprefix.S             | 1 -
 src/arch/x86/prefix/hdprefix.S              | 1 -
 src/arch/x86/prefix/lkrnprefix.S            | 1 -
 src/arch/x86/prefix/mbr.S                   | 1 -
 src/arch/x86/prefix/mromprefix.S            | 1 -
 src/arch/x86/prefix/nbiprefix.S             | 1 -
 src/arch/x86/prefix/nullprefix.S            | 1 -
 src/arch/x86/prefix/pxeprefix.S             | 1 -
 src/arch/x86/prefix/rawprefix.S             | 1 -
 src/arch/x86/prefix/romprefix.S             | 1 -
 src/arch/x86/prefix/undiloader.S            | 1 -
 src/arch/x86/prefix/unlzma.S                | 1 -
 src/arch/x86/prefix/usbdisk.S               | 1 -
 src/arch/x86/transitions/libkir.S           | 1 -
 21 files changed, 21 deletions(-)

diff --git a/src/arch/arm32/libgcc/lldivmod.S b/src/arch/arm32/libgcc/lldivmod.S
index 746fa8fde..c9c224506 100644
--- a/src/arch/arm32/libgcc/lldivmod.S
+++ b/src/arch/arm32/libgcc/lldivmod.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", %progbits
-	.text
 	.thumb
 
 /**
diff --git a/src/arch/arm32/libgcc/llshift.S b/src/arch/arm32/libgcc/llshift.S
index c1b51e778..592e28e65 100644
--- a/src/arch/arm32/libgcc/llshift.S
+++ b/src/arch/arm32/libgcc/llshift.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", %progbits
-	.text
 	.arm
 
 /**
diff --git a/src/arch/x86/core/patch_cf.S b/src/arch/x86/core/patch_cf.S
index 63730c3fd..ebf628769 100644
--- a/src/arch/x86/core/patch_cf.S
+++ b/src/arch/x86/core/patch_cf.S
@@ -23,7 +23,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.code16
 
diff --git a/src/arch/x86/drivers/net/undiisr.S b/src/arch/x86/drivers/net/undiisr.S
index a1098b839..aa8991d77 100644
--- a/src/arch/x86/drivers/net/undiisr.S
+++ b/src/arch/x86/drivers/net/undiisr.S
@@ -11,7 +11,6 @@ FILE_LICENCE ( GPL2_OR_LATER )
 #define PIC2_ICR 0xa0
 	
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.code16
 
diff --git a/src/arch/x86/interface/pcbios/e820mangler.S b/src/arch/x86/interface/pcbios/e820mangler.S
index 46e1cab4d..40f84c702 100644
--- a/src/arch/x86/interface/pcbios/e820mangler.S
+++ b/src/arch/x86/interface/pcbios/e820mangler.S
@@ -24,7 +24,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.code16
 
diff --git a/src/arch/x86/prefix/bootpart.S b/src/arch/x86/prefix/bootpart.S
index 575cb1c07..cf75ff793 100644
--- a/src/arch/x86/prefix/bootpart.S
+++ b/src/arch/x86/prefix/bootpart.S
@@ -6,7 +6,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define STACK_SIZE	0x2000
 	
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.section ".prefix", "awx", @progbits
 	.code16
diff --git a/src/arch/x86/prefix/dskprefix.S b/src/arch/x86/prefix/dskprefix.S
index bc1948879..f9d301956 100644
--- a/src/arch/x86/prefix/dskprefix.S
+++ b/src/arch/x86/prefix/dskprefix.S
@@ -27,7 +27,6 @@ FILE_LICENCE ( GPL2_ONLY )
 	.section ".note.GNU-stack", "", @progbits
 	.org	0
 	.arch i386
-	.text
 	.section ".prefix", "ax", @progbits
 	.code16
 	.globl	_dsk_start
diff --git a/src/arch/x86/prefix/exeprefix.S b/src/arch/x86/prefix/exeprefix.S
index 5b2605e8d..86333489a 100644
--- a/src/arch/x86/prefix/exeprefix.S
+++ b/src/arch/x86/prefix/exeprefix.S
@@ -37,7 +37,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define PSP_CMDLINE_START 0x81
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.org 0
 	.code16
diff --git a/src/arch/x86/prefix/hdprefix.S b/src/arch/x86/prefix/hdprefix.S
index fbf8d2e47..1bbf72dd2 100644
--- a/src/arch/x86/prefix/hdprefix.S
+++ b/src/arch/x86/prefix/hdprefix.S
@@ -3,7 +3,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.section ".prefix", "awx", @progbits
 	.code16
diff --git a/src/arch/x86/prefix/lkrnprefix.S b/src/arch/x86/prefix/lkrnprefix.S
index 2c17f79df..9f1a2f09b 100644
--- a/src/arch/x86/prefix/lkrnprefix.S
+++ b/src/arch/x86/prefix/lkrnprefix.S
@@ -5,7 +5,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define BZI_LOAD_HIGH_ADDR 0x100000
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.code16
 	.section ".prefix", "ax", @progbits
diff --git a/src/arch/x86/prefix/mbr.S b/src/arch/x86/prefix/mbr.S
index 928bb338b..b37eed714 100644
--- a/src/arch/x86/prefix/mbr.S
+++ b/src/arch/x86/prefix/mbr.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.section ".prefix", "awx", @progbits
 	.code16
diff --git a/src/arch/x86/prefix/mromprefix.S b/src/arch/x86/prefix/mromprefix.S
index 5f3496b28..543fd006a 100644
--- a/src/arch/x86/prefix/mromprefix.S
+++ b/src/arch/x86/prefix/mromprefix.S
@@ -42,7 +42,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include "pciromprefix.S"
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.code16
 
diff --git a/src/arch/x86/prefix/nbiprefix.S b/src/arch/x86/prefix/nbiprefix.S
index cae1009b3..0a74ca718 100644
--- a/src/arch/x86/prefix/nbiprefix.S
+++ b/src/arch/x86/prefix/nbiprefix.S
@@ -3,7 +3,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.code16
 	.section ".prefix", "ax", @progbits
diff --git a/src/arch/x86/prefix/nullprefix.S b/src/arch/x86/prefix/nullprefix.S
index 1568188d0..bbc697fd2 100644
--- a/src/arch/x86/prefix/nullprefix.S
+++ b/src/arch/x86/prefix/nullprefix.S
@@ -2,7 +2,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
 	.org	0
-	.text
 	.arch i386
 
 	.section ".prefix", "ax", @progbits
diff --git a/src/arch/x86/prefix/pxeprefix.S b/src/arch/x86/prefix/pxeprefix.S
index 494fbc138..c62a50117 100644
--- a/src/arch/x86/prefix/pxeprefix.S
+++ b/src/arch/x86/prefix/pxeprefix.S
@@ -12,7 +12,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define PXE_HACK_EB54			0x0001
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.org 0
 	.code16
diff --git a/src/arch/x86/prefix/rawprefix.S b/src/arch/x86/prefix/rawprefix.S
index 4a3d35042..d97b3b512 100644
--- a/src/arch/x86/prefix/rawprefix.S
+++ b/src/arch/x86/prefix/rawprefix.S
@@ -9,7 +9,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.org 0
 	.code16
diff --git a/src/arch/x86/prefix/romprefix.S b/src/arch/x86/prefix/romprefix.S
index 79fed2a35..09837ceef 100644
--- a/src/arch/x86/prefix/romprefix.S
+++ b/src/arch/x86/prefix/romprefix.S
@@ -55,7 +55,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #endif
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.code16
 	.arch i386
 	.section ".prefix", "ax", @progbits
diff --git a/src/arch/x86/prefix/undiloader.S b/src/arch/x86/prefix/undiloader.S
index e544d5048..33573230b 100644
--- a/src/arch/x86/prefix/undiloader.S
+++ b/src/arch/x86/prefix/undiloader.S
@@ -3,7 +3,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.code16
 	.arch i386
 	.section ".prefix", "ax", @progbits
diff --git a/src/arch/x86/prefix/unlzma.S b/src/arch/x86/prefix/unlzma.S
index f4bd81bd2..4059090b9 100644
--- a/src/arch/x86/prefix/unlzma.S
+++ b/src/arch/x86/prefix/unlzma.S
@@ -44,7 +44,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i486
 	.section ".prefix.lib", "ax", @progbits
 
diff --git a/src/arch/x86/prefix/usbdisk.S b/src/arch/x86/prefix/usbdisk.S
index 461a08379..b8fc5e950 100644
--- a/src/arch/x86/prefix/usbdisk.S
+++ b/src/arch/x86/prefix/usbdisk.S
@@ -3,7 +3,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <config/console.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.section ".prefix", "awx", @progbits
 	.code16
diff --git a/src/arch/x86/transitions/libkir.S b/src/arch/x86/transitions/libkir.S
index af090b266..5909654c2 100644
--- a/src/arch/x86/transitions/libkir.S
+++ b/src/arch/x86/transitions/libkir.S
@@ -32,7 +32,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define BOCHSBP xchgw %bx, %bx
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.arch i386
 	.section ".text16", "awx", @progbits
 	.code16

From 6ca597eee9f95b846a3c2dc1231e63cfc02272c1 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 14 Jan 2024 12:12:18 +0000
Subject: [PATCH 048/237] [build] Fix building with newer binutils

Newer versions of the GNU assembler (observed with binutils 2.41) will
complain about the ".arch i386" in files assembled with "as --64",
with the message "Error: 64bit mode not supported on 'i386'".

In files such as stack.S that contain no instructions to be assembled,
the ".arch i386" is redundant and may be removed entirely.

In the remaining files, fix by moving ".arch i386" below the relevant
".code16" or ".code32" directive, so that the assembler is no longer
expecting 64-bit instructions to be used by the time that the ".arch
i386" directive is encountered.

Reported-by: Ali Mustakim <alim@forwardcomputers.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/i386/core/setjmp.S                 | 2 +-
 src/arch/i386/tests/gdbstub_test.S          | 1 +
 src/arch/x86/core/patch_cf.S                | 2 +-
 src/arch/x86/core/stack.S                   | 1 -
 src/arch/x86/core/stack16.S                 | 1 -
 src/arch/x86/drivers/net/undiisr.S          | 2 +-
 src/arch/x86/interface/pcbios/e820mangler.S | 2 +-
 src/arch/x86/interface/pxe/pxe_entry.S      | 1 +
 src/arch/x86/prefix/bootpart.S              | 2 +-
 src/arch/x86/prefix/dskprefix.S             | 2 +-
 src/arch/x86/prefix/exeprefix.S             | 2 +-
 src/arch/x86/prefix/hdprefix.S              | 2 +-
 src/arch/x86/prefix/libprefix.S             | 1 +
 src/arch/x86/prefix/lkrnprefix.S            | 2 +-
 src/arch/x86/prefix/mbr.S                   | 2 +-
 src/arch/x86/prefix/mromprefix.S            | 2 +-
 src/arch/x86/prefix/nbiprefix.S             | 2 +-
 src/arch/x86/prefix/nullprefix.S            | 2 +-
 src/arch/x86/prefix/pxeprefix.S             | 2 +-
 src/arch/x86/prefix/rawprefix.S             | 2 +-
 src/arch/x86/prefix/unlzma.S                | 1 +
 src/arch/x86/prefix/usbdisk.S               | 2 +-
 src/arch/x86/transitions/liba20.S           | 1 +
 src/arch/x86/transitions/libkir.S           | 2 +-
 24 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/src/arch/i386/core/setjmp.S b/src/arch/i386/core/setjmp.S
index e0bbb7ef8..cbb5e7138 100644
--- a/src/arch/i386/core/setjmp.S
+++ b/src/arch/i386/core/setjmp.S
@@ -2,8 +2,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
 	.text
-	.arch i386
 	.code32
+	.arch i386
 
 	/* Must match jmp_buf structure layout */
 	.struct	0
diff --git a/src/arch/i386/tests/gdbstub_test.S b/src/arch/i386/tests/gdbstub_test.S
index e0c9e6c9a..e44c13c2d 100644
--- a/src/arch/i386/tests/gdbstub_test.S
+++ b/src/arch/i386/tests/gdbstub_test.S
@@ -1,4 +1,5 @@
 	.section ".note.GNU-stack", "", @progbits
+	.code32
 	.arch i386
 
 	.section ".data", "aw", @progbits
diff --git a/src/arch/x86/core/patch_cf.S b/src/arch/x86/core/patch_cf.S
index ebf628769..62f19e45f 100644
--- a/src/arch/x86/core/patch_cf.S
+++ b/src/arch/x86/core/patch_cf.S
@@ -23,8 +23,8 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 	.code16
+	.arch i386
 
 /****************************************************************************
  * Set/clear CF on the stack as appropriate, assumes stack is as it should
diff --git a/src/arch/x86/core/stack.S b/src/arch/x86/core/stack.S
index 493453473..1bcaf18f8 100644
--- a/src/arch/x86/core/stack.S
+++ b/src/arch/x86/core/stack.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 
 #ifdef __x86_64__
 #define STACK_SIZE 8192
diff --git a/src/arch/x86/core/stack16.S b/src/arch/x86/core/stack16.S
index d3949a557..622887eab 100644
--- a/src/arch/x86/core/stack16.S
+++ b/src/arch/x86/core/stack16.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 
 /****************************************************************************
  * Internal stack
diff --git a/src/arch/x86/drivers/net/undiisr.S b/src/arch/x86/drivers/net/undiisr.S
index aa8991d77..8ba5c5354 100644
--- a/src/arch/x86/drivers/net/undiisr.S
+++ b/src/arch/x86/drivers/net/undiisr.S
@@ -11,8 +11,8 @@ FILE_LICENCE ( GPL2_OR_LATER )
 #define PIC2_ICR 0xa0
 	
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 	.code16
+	.arch i386
 
 	.section ".text16", "ax", @progbits
 	.globl undiisr
diff --git a/src/arch/x86/interface/pcbios/e820mangler.S b/src/arch/x86/interface/pcbios/e820mangler.S
index 40f84c702..ef5dc2754 100644
--- a/src/arch/x86/interface/pcbios/e820mangler.S
+++ b/src/arch/x86/interface/pcbios/e820mangler.S
@@ -24,8 +24,8 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 	.code16
+	.arch i386
 
 #define SMAP 0x534d4150
 
diff --git a/src/arch/x86/interface/pxe/pxe_entry.S b/src/arch/x86/interface/pxe/pxe_entry.S
index 354dd1b35..3899e1bcc 100644
--- a/src/arch/x86/interface/pxe/pxe_entry.S
+++ b/src/arch/x86/interface/pxe/pxe_entry.S
@@ -27,6 +27,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 
 /****************************************************************************
diff --git a/src/arch/x86/prefix/bootpart.S b/src/arch/x86/prefix/bootpart.S
index cf75ff793..7b9920fdb 100644
--- a/src/arch/x86/prefix/bootpart.S
+++ b/src/arch/x86/prefix/bootpart.S
@@ -6,9 +6,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define STACK_SIZE	0x2000
 	
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 	.section ".prefix", "awx", @progbits
-	.code16
 
 /*
  * Find active partition
diff --git a/src/arch/x86/prefix/dskprefix.S b/src/arch/x86/prefix/dskprefix.S
index f9d301956..e8e55ef1d 100644
--- a/src/arch/x86/prefix/dskprefix.S
+++ b/src/arch/x86/prefix/dskprefix.S
@@ -26,9 +26,9 @@ FILE_LICENCE ( GPL2_ONLY )
 
 	.section ".note.GNU-stack", "", @progbits
 	.org	0
+	.code16
 	.arch i386
 	.section ".prefix", "ax", @progbits
-	.code16
 	.globl	_dsk_start
 _dsk_start:
 
diff --git a/src/arch/x86/prefix/exeprefix.S b/src/arch/x86/prefix/exeprefix.S
index 86333489a..98ed6c5fb 100644
--- a/src/arch/x86/prefix/exeprefix.S
+++ b/src/arch/x86/prefix/exeprefix.S
@@ -37,9 +37,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define PSP_CMDLINE_START 0x81
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 	.org 0
-	.code16
 	.section ".prefix", "awx", @progbits
 
 signature:
diff --git a/src/arch/x86/prefix/hdprefix.S b/src/arch/x86/prefix/hdprefix.S
index 1bbf72dd2..3133dec6c 100644
--- a/src/arch/x86/prefix/hdprefix.S
+++ b/src/arch/x86/prefix/hdprefix.S
@@ -3,9 +3,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 	.section ".prefix", "awx", @progbits
-	.code16
 	.org 0
 	.globl	_hd_start
 _hd_start:
diff --git a/src/arch/x86/prefix/libprefix.S b/src/arch/x86/prefix/libprefix.S
index 380e471dd..b08a5782f 100644
--- a/src/arch/x86/prefix/libprefix.S
+++ b/src/arch/x86/prefix/libprefix.S
@@ -27,6 +27,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 
 /* Image compression enabled */
diff --git a/src/arch/x86/prefix/lkrnprefix.S b/src/arch/x86/prefix/lkrnprefix.S
index 9f1a2f09b..c8a04c9d7 100644
--- a/src/arch/x86/prefix/lkrnprefix.S
+++ b/src/arch/x86/prefix/lkrnprefix.S
@@ -5,8 +5,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define BZI_LOAD_HIGH_ADDR 0x100000
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 	.code16
+	.arch i386
 	.section ".prefix", "ax", @progbits
 	.globl	_lkrn_start
 _lkrn_start:
diff --git a/src/arch/x86/prefix/mbr.S b/src/arch/x86/prefix/mbr.S
index b37eed714..5e0ed5ddb 100644
--- a/src/arch/x86/prefix/mbr.S
+++ b/src/arch/x86/prefix/mbr.S
@@ -1,9 +1,9 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 	.section ".prefix", "awx", @progbits
-	.code16
 	.org 0
 
 	.globl mbr
diff --git a/src/arch/x86/prefix/mromprefix.S b/src/arch/x86/prefix/mromprefix.S
index 543fd006a..d05278e64 100644
--- a/src/arch/x86/prefix/mromprefix.S
+++ b/src/arch/x86/prefix/mromprefix.S
@@ -42,8 +42,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include "pciromprefix.S"
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 	.code16
+	.arch i386
 
 /* Obtain access to payload by exposing the expansion ROM BAR at the
  * address currently used by a suitably large memory BAR on the same
diff --git a/src/arch/x86/prefix/nbiprefix.S b/src/arch/x86/prefix/nbiprefix.S
index 0a74ca718..bbacd4b7b 100644
--- a/src/arch/x86/prefix/nbiprefix.S
+++ b/src/arch/x86/prefix/nbiprefix.S
@@ -3,8 +3,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 	.code16
+	.arch i386
 	.section ".prefix", "ax", @progbits
 	.org 0
 
diff --git a/src/arch/x86/prefix/nullprefix.S b/src/arch/x86/prefix/nullprefix.S
index bbc697fd2..426f1f2ce 100644
--- a/src/arch/x86/prefix/nullprefix.S
+++ b/src/arch/x86/prefix/nullprefix.S
@@ -2,10 +2,10 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
 	.org	0
+	.code16
 	.arch i386
 
 	.section ".prefix", "ax", @progbits
-	.code16
 _prefix:
 
 	.section ".text16", "ax", @progbits
diff --git a/src/arch/x86/prefix/pxeprefix.S b/src/arch/x86/prefix/pxeprefix.S
index c62a50117..5181ef618 100644
--- a/src/arch/x86/prefix/pxeprefix.S
+++ b/src/arch/x86/prefix/pxeprefix.S
@@ -12,9 +12,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define PXE_HACK_EB54			0x0001
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 	.org 0
-	.code16
 
 #include <librm.h>
 #include <undi.h>
diff --git a/src/arch/x86/prefix/rawprefix.S b/src/arch/x86/prefix/rawprefix.S
index d97b3b512..962c97187 100644
--- a/src/arch/x86/prefix/rawprefix.S
+++ b/src/arch/x86/prefix/rawprefix.S
@@ -9,9 +9,9 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 	.org 0
-	.code16
 
 #include <librm.h>
 
diff --git a/src/arch/x86/prefix/unlzma.S b/src/arch/x86/prefix/unlzma.S
index 4059090b9..e4d1e190d 100644
--- a/src/arch/x86/prefix/unlzma.S
+++ b/src/arch/x86/prefix/unlzma.S
@@ -44,6 +44,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 
 	.section ".note.GNU-stack", "", @progbits
+	.code32
 	.arch i486
 	.section ".prefix.lib", "ax", @progbits
 
diff --git a/src/arch/x86/prefix/usbdisk.S b/src/arch/x86/prefix/usbdisk.S
index b8fc5e950..11ab6a46a 100644
--- a/src/arch/x86/prefix/usbdisk.S
+++ b/src/arch/x86/prefix/usbdisk.S
@@ -3,9 +3,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <config/console.h>
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 	.section ".prefix", "awx", @progbits
-	.code16
 	.org 0
 
 #include "mbr.S"
diff --git a/src/arch/x86/transitions/liba20.S b/src/arch/x86/transitions/liba20.S
index 6c1bac672..971cff226 100644
--- a/src/arch/x86/transitions/liba20.S
+++ b/src/arch/x86/transitions/liba20.S
@@ -25,6 +25,7 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 
 /****************************************************************************
diff --git a/src/arch/x86/transitions/libkir.S b/src/arch/x86/transitions/libkir.S
index 5909654c2..2c4dc948b 100644
--- a/src/arch/x86/transitions/libkir.S
+++ b/src/arch/x86/transitions/libkir.S
@@ -32,9 +32,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define BOCHSBP xchgw %bx, %bx
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 	.section ".text16", "awx", @progbits
-	.code16
 	
 /****************************************************************************
  * init_libkir (real-mode or 16:xx protected-mode far call)

From 6d29415c89d607b988381bc367c9c521694fa728 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 16 Jan 2024 12:23:02 +0000
Subject: [PATCH 049/237] [libc] Make static_assert() available via assert.h

Expose static_assert() via assert.h and migrate link-time assertions
to build-time assertions where possible.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/infiniband/arbel.c  | 4 ++--
 src/drivers/infiniband/hermon.c | 4 ++--
 src/image/script.c              | 3 +--
 src/include/assert.h            | 9 +++++++++
 src/net/aoe.c                   | 2 +-
 5 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c
index 293c1b647..24c0b53b0 100644
--- a/src/drivers/infiniband/arbel.c
+++ b/src/drivers/infiniband/arbel.c
@@ -545,8 +545,8 @@ static int arbel_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
 	union arbelprm_mad mad_ifc;
 	int rc;
 
-	linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
-			mad_size_mismatch );
+	/* Sanity check */
+	static_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ) );
 
 	/* Copy in request packet */
 	memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c
index c09baf7ae..6fc7d8bd4 100644
--- a/src/drivers/infiniband/hermon.c
+++ b/src/drivers/infiniband/hermon.c
@@ -779,8 +779,8 @@ static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
 	union hermonprm_mad mad_ifc;
 	int rc;
 
-	linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
-			mad_size_mismatch );
+	/* Sanity check */
+	static_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ) );
 
 	/* Copy in request packet */
 	memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
diff --git a/src/image/script.c b/src/image/script.c
index 49b356403..9e8566bc5 100644
--- a/src/image/script.c
+++ b/src/image/script.c
@@ -219,8 +219,7 @@ static int script_exec ( struct image *image ) {
 static int script_probe ( struct image *image ) {
 	static const char ipxe_magic[] = "#!ipxe";
 	static const char gpxe_magic[] = "#!gpxe";
-	linker_assert ( sizeof ( ipxe_magic ) == sizeof ( gpxe_magic ),
-			magic_size_mismatch );
+	static_assert ( sizeof ( ipxe_magic ) == sizeof ( gpxe_magic ) );
 	char test[ sizeof ( ipxe_magic ) - 1 /* NUL */
 		   + 1 /* terminating space */];
 
diff --git a/src/include/assert.h b/src/include/assert.h
index dd71fa713..6d0531801 100644
--- a/src/include/assert.h
+++ b/src/include/assert.h
@@ -55,6 +55,15 @@ assert_printf ( const char *fmt, ... ) asm ( "printf" );
 		} 							     \
 	} while ( 0 )
 
+/**
+ * Assert a condition at build time
+ *
+ * If the compiler cannot prove that the condition is true, the build
+ * will fail with an error message.
+ */
+#undef static_assert
+#define static_assert(x) _Static_assert( x, #x )
+
 /**
  * Assert a condition at link-time.
  *
diff --git a/src/net/aoe.c b/src/net/aoe.c
index e785e8979..dba4f51b5 100644
--- a/src/net/aoe.c
+++ b/src/net/aoe.c
@@ -374,7 +374,7 @@ static void aoecmd_ata_cmd ( struct aoe_command *aoecmd,
 	struct aoeata *aoeata = &aoehdr->payload[0].ata;
 
 	/* Sanity check */
-	linker_assert ( AOE_FL_DEV_HEAD	== ATA_DEV_SLAVE, __fix_ata_h__ );
+	static_assert ( AOE_FL_DEV_HEAD == ATA_DEV_SLAVE );
 	assert ( len == ( sizeof ( *aoehdr ) + sizeof ( *aoeata ) +
 			  command->data_out_len ) );
 

From 4b7d9a6af08cb704ce77eadba2a7bb1b06c1554c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 16 Jan 2024 13:24:29 +0000
Subject: [PATCH 050/237] [libc] Replace linker_assert() with build_assert()

We currently implement build-time assertions via a mechanism that
generates a call to an undefined external function that will cause the
link to fail unless the compiler can prove that the asserted condition
is true (and thereby eliminate the undefined function call).

This assertion mechanism can be used for conditions that are not
amenable to the use of static_assert(), since static_assert() will not
allow for proofs via dead code elimination.

Add __attribute__((error(...))) to the undefined external function, so
that the error is raised at compile time rather than at link time.
This allows us to provide a more meaningful error message (which will
include the file name and line number, as with any other compile-time
error), and avoids the need for the caller to specify a unique symbol
name for the external function.

Change the name from linker_assert() to build_assert(), since the
assertion now takes place at compile time rather than at link time.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/gcm.c                 |  8 +++---
 src/crypto/md4.c                 | 10 ++++----
 src/crypto/md5.c                 | 10 ++++----
 src/crypto/sha1.c                | 12 ++++-----
 src/crypto/sha256.c              | 18 +++++++-------
 src/crypto/sha512.c              | 18 +++++++-------
 src/drivers/infiniband/linda.c   |  4 +--
 src/drivers/infiniband/qib7322.c |  4 +--
 src/include/assert.h             | 21 +++++++++-------
 src/include/ipxe/asn1.h          |  7 +++---
 src/include/ipxe/entropy.h       | 42 +++++++++++++-------------------
 src/include/ipxe/gcm.h           | 11 +++------
 12 files changed, 78 insertions(+), 87 deletions(-)

diff --git a/src/crypto/gcm.c b/src/crypto/gcm.c
index 9d8bae824..c21aad14c 100644
--- a/src/crypto/gcm.c
+++ b/src/crypto/gcm.c
@@ -472,10 +472,10 @@ void gcm_setiv ( struct gcm_context *context, const void *iv, size_t ivlen ) {
 	union gcm_block *check = ( ( void * ) context );
 
 	/* Sanity checks */
-	linker_assert ( &context->hash == check, gcm_bad_layout );
-	linker_assert ( &context->len == check + 1, gcm_bad_layout );
-	linker_assert ( &context->ctr == check + 2, gcm_bad_layout );
-	linker_assert ( &context->key == check + 3, gcm_bad_layout );
+	build_assert ( &context->hash == check );
+	build_assert ( &context->len == check + 1 );
+	build_assert ( &context->ctr == check + 2 );
+	build_assert ( &context->key == check + 3 );
 
 	/* Reset non-key state */
 	memset ( context, 0, offsetof ( typeof ( *context ), key ) );
diff --git a/src/crypto/md4.c b/src/crypto/md4.c
index ca5dcc21b..dcd86a428 100644
--- a/src/crypto/md4.c
+++ b/src/crypto/md4.c
@@ -155,11 +155,11 @@ static void md4_digest ( struct md4_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
-	linker_assert ( &u.ddd.dd.digest.h[0] == a, md4_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[1] == b, md4_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[2] == c, md4_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[3] == d, md4_bad_layout );
-	linker_assert ( &u.ddd.dd.data.dword[0] == w, md4_bad_layout );
+	build_assert ( &u.ddd.dd.digest.h[0] == a );
+	build_assert ( &u.ddd.dd.digest.h[1] == b );
+	build_assert ( &u.ddd.dd.digest.h[2] == c );
+	build_assert ( &u.ddd.dd.digest.h[3] == d );
+	build_assert ( &u.ddd.dd.data.dword[0] == w );
 
 	DBGC ( context, "MD4 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddd.dd.digest,
diff --git a/src/crypto/md5.c b/src/crypto/md5.c
index bee382e95..5c62513e2 100644
--- a/src/crypto/md5.c
+++ b/src/crypto/md5.c
@@ -178,11 +178,11 @@ static void md5_digest ( struct md5_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
-	linker_assert ( &u.ddd.dd.digest.h[0] == a, md5_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[1] == b, md5_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[2] == c, md5_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[3] == d, md5_bad_layout );
-	linker_assert ( &u.ddd.dd.data.dword[0] == w, md5_bad_layout );
+	build_assert ( &u.ddd.dd.digest.h[0] == a );
+	build_assert ( &u.ddd.dd.digest.h[1] == b );
+	build_assert ( &u.ddd.dd.digest.h[2] == c );
+	build_assert ( &u.ddd.dd.digest.h[3] == d );
+	build_assert ( &u.ddd.dd.data.dword[0] == w );
 
 	DBGC ( context, "MD5 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddd.dd.digest,
diff --git a/src/crypto/sha1.c b/src/crypto/sha1.c
index 94fce0029..8eecc75b3 100644
--- a/src/crypto/sha1.c
+++ b/src/crypto/sha1.c
@@ -145,12 +145,12 @@ static void sha1_digest ( struct sha1_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
-	linker_assert ( &u.ddd.dd.digest.h[0] == a, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[1] == b, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[2] == c, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[3] == d, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[4] == e, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.data.dword[0] == w, sha1_bad_layout );
+	build_assert ( &u.ddd.dd.digest.h[0] == a );
+	build_assert ( &u.ddd.dd.digest.h[1] == b );
+	build_assert ( &u.ddd.dd.digest.h[2] == c );
+	build_assert ( &u.ddd.dd.digest.h[3] == d );
+	build_assert ( &u.ddd.dd.digest.h[4] == e );
+	build_assert ( &u.ddd.dd.data.dword[0] == w );
 
 	DBGC ( context, "SHA1 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddd.dd.digest,
diff --git a/src/crypto/sha256.c b/src/crypto/sha256.c
index 6bd727719..c30300eb4 100644
--- a/src/crypto/sha256.c
+++ b/src/crypto/sha256.c
@@ -140,15 +140,15 @@ static void sha256_digest ( struct sha256_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
-	linker_assert ( &u.ddd.dd.digest.h[0] == a, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[1] == b, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[2] == c, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[3] == d, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[4] == e, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[5] == f, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[6] == g, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[7] == h, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.data.dword[0] == w, sha256_bad_layout );
+	build_assert ( &u.ddd.dd.digest.h[0] == a );
+	build_assert ( &u.ddd.dd.digest.h[1] == b );
+	build_assert ( &u.ddd.dd.digest.h[2] == c );
+	build_assert ( &u.ddd.dd.digest.h[3] == d );
+	build_assert ( &u.ddd.dd.digest.h[4] == e );
+	build_assert ( &u.ddd.dd.digest.h[5] == f );
+	build_assert ( &u.ddd.dd.digest.h[6] == g );
+	build_assert ( &u.ddd.dd.digest.h[7] == h );
+	build_assert ( &u.ddd.dd.data.dword[0] == w );
 
 	DBGC ( context, "SHA256 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddd.dd.digest,
diff --git a/src/crypto/sha512.c b/src/crypto/sha512.c
index e84895010..d7d44b284 100644
--- a/src/crypto/sha512.c
+++ b/src/crypto/sha512.c
@@ -156,15 +156,15 @@ static void sha512_digest ( struct sha512_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddq.dd.data ) ) == 0 );
-	linker_assert ( &u.ddq.dd.digest.h[0] == a, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[1] == b, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[2] == c, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[3] == d, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[4] == e, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[5] == f, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[6] == g, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[7] == h, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.data.qword[0] == w, sha512_bad_layout );
+	build_assert ( &u.ddq.dd.digest.h[0] == a );
+	build_assert ( &u.ddq.dd.digest.h[1] == b );
+	build_assert ( &u.ddq.dd.digest.h[2] == c );
+	build_assert ( &u.ddq.dd.digest.h[3] == d );
+	build_assert ( &u.ddq.dd.digest.h[4] == e );
+	build_assert ( &u.ddq.dd.digest.h[5] == f );
+	build_assert ( &u.ddq.dd.digest.h[6] == g );
+	build_assert ( &u.ddq.dd.digest.h[7] == h );
+	build_assert ( &u.ddq.dd.data.qword[0] == w );
 
 	DBGC ( context, "SHA512 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddq.dd.digest,
diff --git a/src/drivers/infiniband/linda.c b/src/drivers/infiniband/linda.c
index 8c5912660..0c8a043a1 100644
--- a/src/drivers/infiniband/linda.c
+++ b/src/drivers/infiniband/linda.c
@@ -721,7 +721,7 @@ static int linda_init_recv ( struct linda *linda ) {
 		eager_array_size_other = LINDA_EAGER_ARRAY_SIZE_17CTX_OTHER;
 		break;
 	default:
-		linker_assert ( 0, invalid_LINDA_NUM_CONTEXTS );
+		build_assert ( 0 );
 		return -EINVAL;
 	}
 
@@ -1108,7 +1108,7 @@ static int linda_post_recv ( struct ib_device *ibdev,
 	case 16384: bufsize = LINDA_EAGER_BUFFER_16K; break;
 	case 32768: bufsize = LINDA_EAGER_BUFFER_32K; break;
 	case 65536: bufsize = LINDA_EAGER_BUFFER_64K; break;
-	default:    linker_assert ( 0, invalid_rx_payload_size );
+	default:    build_assert ( 0 );
 		    bufsize = LINDA_EAGER_BUFFER_NONE;
 	}
 
diff --git a/src/drivers/infiniband/qib7322.c b/src/drivers/infiniband/qib7322.c
index da055b744..a011dafc1 100644
--- a/src/drivers/infiniband/qib7322.c
+++ b/src/drivers/infiniband/qib7322.c
@@ -893,7 +893,7 @@ static int qib7322_init_recv ( struct qib7322 *qib7322 ) {
 		eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_18CTX_USER;
 		break;
 	default:
-		linker_assert ( 0, invalid_QIB7322_NUM_CONTEXTS );
+		build_assert ( 0 );
 		return -EINVAL;
 	}
 
@@ -1351,7 +1351,7 @@ static int qib7322_post_recv ( struct ib_device *ibdev,
 	case 16384: bufsize = QIB7322_EAGER_BUFFER_16K; break;
 	case 32768: bufsize = QIB7322_EAGER_BUFFER_32K; break;
 	case 65536: bufsize = QIB7322_EAGER_BUFFER_64K; break;
-	default:    linker_assert ( 0, invalid_rx_payload_size );
+	default:    build_assert ( 0 );
 		    bufsize = QIB7322_EAGER_BUFFER_NONE;
 	}
 
diff --git a/src/include/assert.h b/src/include/assert.h
index 6d0531801..b3a9b1fe0 100644
--- a/src/include/assert.h
+++ b/src/include/assert.h
@@ -65,19 +65,22 @@ assert_printf ( const char *fmt, ... ) asm ( "printf" );
 #define static_assert(x) _Static_assert( x, #x )
 
 /**
- * Assert a condition at link-time.
+ * Assert a condition at build time (after dead code elimination)
  *
- * If the condition is not true, the link will fail with an unresolved
- * symbol (error_symbol).
+ * If the compiler cannot prove that the condition is true, the build
+ * will fail with an error message.
  *
  * This macro is iPXE-specific.  Do not use this macro in code
  * intended to be portable.
- *
  */
-#define linker_assert( condition, error_symbol )	\
-        if ( ! (condition) ) {				\
-                extern void error_symbol ( void );	\
-                error_symbol();				\
-        }
+#define build_assert( condition )					     \
+	do { 								     \
+		if ( ! (condition) ) {					     \
+			extern void __attribute__ (( error (		     \
+				"build_assert(" #condition ") failed"	     \
+			) )) _C2 ( build_assert_, __LINE__ ) ( void );	     \
+			_C2 ( build_assert_, __LINE__ ) ();		     \
+		}							     \
+	} while ( 0 )
 
 #endif /* _ASSERT_H */
diff --git a/src/include/ipxe/asn1.h b/src/include/ipxe/asn1.h
index 77429f3a0..452fcef0b 100644
--- a/src/include/ipxe/asn1.h
+++ b/src/include/ipxe/asn1.h
@@ -390,10 +390,9 @@ asn1_built ( struct asn1_builder *builder ) {
 	} *u = container_of ( builder, typeof ( *u ), builder );
 
 	/* Sanity check */
-	linker_assert ( ( ( const void * ) &u->builder.data ) ==
-			&u->cursor.data, asn1_builder_cursor_data_mismatch );
-	linker_assert ( &u->builder.len == &u->cursor.len,
-			asn1_builder_cursor_len_mismatch );
+	build_assert ( ( ( const void * ) &u->builder.data ) ==
+		       &u->cursor.data );
+	build_assert ( &u->builder.len == &u->cursor.len );
 
 	return &u->cursor;
 }
diff --git a/src/include/ipxe/entropy.h b/src/include/ipxe/entropy.h
index 240feace0..82bb11826 100644
--- a/src/include/ipxe/entropy.h
+++ b/src/include/ipxe/entropy.h
@@ -237,8 +237,7 @@ get_entropy_input ( unsigned int min_entropy_bits, void *data, size_t min_len,
 	int rc;
 
 	/* Sanity check */
-	linker_assert ( ( min_entropy_bits <= ( 8 * max_len ) ),
-			entropy_buffer_too_small );
+	build_assert ( min_entropy_bits <= ( 8 * max_len ) );
 
 	/* Round up minimum entropy to an integral number of bytes */
 	min_entropy_bits = ( ( min_entropy_bits + 7 ) & ~7 );
@@ -247,11 +246,11 @@ get_entropy_input ( unsigned int min_entropy_bits, void *data, size_t min_len,
 	 * meet or exceed the security strength indicated by the
 	 * min_entropy parameter.
 	 */
-	linker_assert ( ( ( 8 * ENTROPY_HASH_DF_OUTLEN_BYTES ) >=
-			  min_entropy_bits ), hash_df_algorithm_too_weak );
+	build_assert ( ( 8 * ENTROPY_HASH_DF_OUTLEN_BYTES ) >=
+		       min_entropy_bits );
 
 	/* 1.  If ( min_length > max_length ), then return ( FAILURE, Null ) */
-	linker_assert ( ( min_len <= max_len ), min_len_greater_than_max_len );
+	build_assert ( min_len <= max_len );
 
 	/* 2.  n = 2 * min_entropy */
 	n = ( 2 * min_entropy_bits );
@@ -269,9 +268,8 @@ get_entropy_input ( unsigned int min_entropy_bits, void *data, size_t min_len,
 	 * (The implementation of these steps is inside the function
 	 * get_entropy_input_tmp().)
 	 */
-	linker_assert ( __builtin_constant_p ( tmp_len ),
-			tmp_len_not_constant );
-	linker_assert ( ( n == ( 8 * tmp_len ) ), tmp_len_mismatch );
+	build_assert ( __builtin_constant_p ( tmp_len ) );
+	build_assert ( n == ( 8 * tmp_len ) );
 	if ( ( rc = get_entropy_input_tmp ( MIN_ENTROPY ( min_entropy_bits ),
 					    tmp, tmp_len ) ) != 0 ) {
 		return rc;
@@ -283,17 +281,17 @@ get_entropy_input ( unsigned int min_entropy_bits, void *data, size_t min_len,
 	 */
 	if ( tmp_len < min_len ) {
 		/* (Data is already in-place.) */
-		linker_assert ( ( data == tmp ), data_not_inplace );
+		build_assert ( data == tmp );
 		memset ( ( data + tmp_len ), 0, ( min_len - tmp_len ) );
 		return min_len;
 	} else if ( tmp_len > max_len ) {
-		linker_assert ( ( tmp == tmp_buf ), data_inplace );
+		build_assert ( tmp == tmp_buf );
 		hash_df ( &entropy_hash_df_algorithm, tmp, tmp_len,
 			  data, max_len );
 		return max_len;
 	} else {
 		/* (Data is already in-place.) */
-		linker_assert ( ( data == tmp ), data_not_inplace );
+		build_assert ( data == tmp );
 		return tmp_len;
 	}
 }
@@ -328,15 +326,14 @@ entropy_repetition_count_cutoff ( min_entropy_t min_entropy_per_sample ) {
 	cutoff = max_repetitions;
 	if ( cutoff < max_repetitions )
 		cutoff++;
-	linker_assert ( ( cutoff >= max_repetitions ), rounding_error );
+	build_assert ( cutoff >= max_repetitions );
 
 	/* Floating-point operations are not allowed in iPXE since we
 	 * never set up a suitable environment.  Abort the build
 	 * unless the calculated number of repetitions is a
 	 * compile-time constant.
 	 */
-	linker_assert ( __builtin_constant_p ( cutoff ),
-			repetition_count_cutoff_not_constant );
+	build_assert ( __builtin_constant_p ( cutoff ) );
 
 	return cutoff;
 }
@@ -443,12 +440,10 @@ entropy_adaptive_proportion_cutoff ( min_entropy_t min_entropy_per_sample ) {
 	cutoff = entropy_adaptive_proportion_cutoff_lookup ( n, h );
 
 	/* Fail unless cutoff value is a compile-time constant */
-	linker_assert ( __builtin_constant_p ( cutoff ),
-			adaptive_proportion_cutoff_not_constant );
+	build_assert ( __builtin_constant_p ( cutoff ) );
 
 	/* Fail if cutoff value is N/A */
-	linker_assert ( ( cutoff != APC_NA ),
-			adaptive_proportion_cutoff_not_applicable );
+	build_assert ( cutoff != APC_NA );
 
 	return cutoff;
 }
@@ -475,8 +470,7 @@ entropy_startup_test_count ( unsigned int repetition_count_cutoff,
 	num_samples = repetition_count_cutoff;
 	if ( num_samples < adaptive_proportion_cutoff )
 		num_samples = adaptive_proportion_cutoff;
-	linker_assert ( __builtin_constant_p ( num_samples ),
-			startup_test_count_not_constant );
+	build_assert ( __builtin_constant_p ( num_samples ) );
 
 	return num_samples;
 }
@@ -499,11 +493,9 @@ entropy_init ( struct entropy_source *source,
 	unsigned int startup_test_count;
 
 	/* Sanity check */
-	linker_assert ( min_entropy_per_sample > MIN_ENTROPY ( 0 ),
-			min_entropy_per_sample_is_zero );
-	linker_assert ( ( min_entropy_per_sample <=
-			  MIN_ENTROPY ( 8 * sizeof ( noise_sample_t ) ) ),
-			min_entropy_per_sample_is_impossibly_high );
+	build_assert ( min_entropy_per_sample > MIN_ENTROPY ( 0 ) );
+	build_assert ( min_entropy_per_sample <=
+		       MIN_ENTROPY ( 8 * sizeof ( noise_sample_t ) ) );
 
 	/* Calculate test cutoff values */
 	repetition_count_cutoff =
diff --git a/src/include/ipxe/gcm.h b/src/include/ipxe/gcm.h
index 90ef0b522..9653a0a1a 100644
--- a/src/include/ipxe/gcm.h
+++ b/src/include/ipxe/gcm.h
@@ -88,13 +88,10 @@ struct _gcm_name ## _context {						\
 static int _gcm_name ## _setkey ( void *ctx, const void *key,		\
 				  size_t keylen ) {			\
 	struct _gcm_name ## _context *context = ctx;			\
-	linker_assert ( _blocksize == sizeof ( context->gcm.key ),	\
-			_gcm_name ## _unsupported_blocksize );		\
-	linker_assert ( ( ( void * ) &context->gcm ) == ctx,		\
-			_gcm_name ## _context_layout_error );		\
-	linker_assert ( ( ( void * ) &context->raw ) ==			\
-			( ( void * ) context->gcm.raw_ctx ),		\
-			_gcm_name ## _context_layout_error );		\
+	build_assert ( _blocksize == sizeof ( context->gcm.key ) );	\
+	build_assert ( ( ( void * ) &context->gcm ) == ctx );		\
+	build_assert ( ( ( void * ) &context->raw ) ==			\
+		       ( ( void * ) context->gcm.raw_ctx ) );		\
 	return gcm_setkey ( &context->gcm, key, keylen, &_raw_cipher );	\
 }									\
 static void _gcm_name ## _setiv ( void *ctx, const void *iv,		\

From 26d3ef062b33e43e076b7ecef20c4ec3f9441860 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 16 Jan 2024 16:09:16 +0000
Subject: [PATCH 051/237] [crypto] Allow multiplicand and multiplier to differ
 in size

Big integer multiplication is currently used only as part of modular
exponentiation, where both multiplicand and multiplier will be the
same size.

Relax this requirement to allow for the use of big integer
multiplication in other contexts.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/core/arm32_bigint.c     | 26 +++++++++--------
 src/arch/arm32/include/bits/bigint.h   |  4 ++-
 src/arch/arm64/core/arm64_bigint.c     | 26 +++++++++--------
 src/arch/arm64/include/bits/bigint.h   |  4 ++-
 src/arch/loong64/core/loong64_bigint.c | 26 +++++++++--------
 src/arch/loong64/include/bits/bigint.h |  4 ++-
 src/arch/x86/core/x86_bigint.c         | 26 +++++++++--------
 src/arch/x86/include/bits/bigint.h     |  4 ++-
 src/include/ipxe/bigint.h              | 12 ++++----
 src/tests/bigint_test.c                | 39 ++++++++++++++++----------
 10 files changed, 103 insertions(+), 68 deletions(-)

diff --git a/src/arch/arm32/core/arm32_bigint.c b/src/arch/arm32/core/arm32_bigint.c
index 839bead18..29fb40a7c 100644
--- a/src/arch/arm32/core/arm32_bigint.c
+++ b/src/arch/arm32/core/arm32_bigint.c
@@ -36,19 +36,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Multiply big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
  * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements
  */
 void bigint_multiply_raw ( const uint32_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const uint32_t *multiplier0,
-			   uint32_t *result0, unsigned int size ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
+			   unsigned int multiplier_size,
+			   uint32_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 	unsigned int i;
 	unsigned int j;
 	uint32_t multiplicand_element;
@@ -62,9 +66,9 @@ void bigint_multiply_raw ( const uint32_t *multiplicand0,
 	memset ( result, 0, sizeof ( *result ) );
 
 	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < size ; i++ ) {
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < size ; j++ ) {
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			multiplier_element = multiplier->element[j];
 			result_elements = &result->element[ i + j ];
 			/* Perform a single multiply, and add the
@@ -73,7 +77,7 @@ void bigint_multiply_raw ( const uint32_t *multiplicand0,
 			 * never overflow beyond the end of the
 			 * result, since:
 			 *
-			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
 			 */
 			__asm__ __volatile__ ( "umull %1, %2, %5, %6\n\t"
 					       "ldr %3, [%0]\n\t"
diff --git a/src/arch/arm32/include/bits/bigint.h b/src/arch/arm32/include/bits/bigint.h
index 103c6c489..e4b511da7 100644
--- a/src/arch/arm32/include/bits/bigint.h
+++ b/src/arch/arm32/include/bits/bigint.h
@@ -310,7 +310,9 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused,
 }
 
 extern void bigint_multiply_raw ( const uint32_t *multiplicand0,
+				  unsigned int multiplicand_size,
 				  const uint32_t *multiplier0,
-				  uint32_t *value0, unsigned int size );
+				  unsigned int multiplier_size,
+				  uint32_t *value0 );
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/arm64/core/arm64_bigint.c b/src/arch/arm64/core/arm64_bigint.c
index bc4ee9a00..7740f1aef 100644
--- a/src/arch/arm64/core/arm64_bigint.c
+++ b/src/arch/arm64/core/arm64_bigint.c
@@ -36,19 +36,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Multiply big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
  * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements
  */
 void bigint_multiply_raw ( const uint64_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const uint64_t *multiplier0,
-			   uint64_t *result0, unsigned int size ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
+			   unsigned int multiplier_size,
+			   uint64_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 	unsigned int i;
 	unsigned int j;
 	uint64_t multiplicand_element;
@@ -63,9 +67,9 @@ void bigint_multiply_raw ( const uint64_t *multiplicand0,
 	memset ( result, 0, sizeof ( *result ) );
 
 	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < size ; i++ ) {
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < size ; j++ ) {
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			multiplier_element = multiplier->element[j];
 			result_elements = &result->element[ i + j ];
 			/* Perform a single multiply, and add the
@@ -74,7 +78,7 @@ void bigint_multiply_raw ( const uint64_t *multiplicand0,
 			 * never overflow beyond the end of the
 			 * result, since:
 			 *
-			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
 			 */
 			__asm__ __volatile__ ( "mul %1, %6, %7\n\t"
 					       "umulh %2, %6, %7\n\t"
diff --git a/src/arch/arm64/include/bits/bigint.h b/src/arch/arm64/include/bits/bigint.h
index 79983b410..0d08bbd65 100644
--- a/src/arch/arm64/include/bits/bigint.h
+++ b/src/arch/arm64/include/bits/bigint.h
@@ -311,7 +311,9 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
 }
 
 extern void bigint_multiply_raw ( const uint64_t *multiplicand0,
+				  unsigned int multiplicand_size,
 				  const uint64_t *multiplier0,
-				  uint64_t *value0, unsigned int size );
+				  unsigned int multiplier_size,
+				  uint64_t *value0 );
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/loong64/core/loong64_bigint.c b/src/arch/loong64/core/loong64_bigint.c
index f42b86111..b428e22c3 100644
--- a/src/arch/loong64/core/loong64_bigint.c
+++ b/src/arch/loong64/core/loong64_bigint.c
@@ -37,19 +37,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Multiply big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
  * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements
  */
 void bigint_multiply_raw ( const uint64_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const uint64_t *multiplier0,
-			   uint64_t *result0, unsigned int size ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
+			   unsigned int multiplier_size,
+			   uint64_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 	unsigned int i;
 	unsigned int j;
 	uint64_t multiplicand_element;
@@ -64,9 +68,9 @@ void bigint_multiply_raw ( const uint64_t *multiplicand0,
 	memset ( result, 0, sizeof ( *result ) );
 
 	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < size ; i++ ) {
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < size ; j++ ) {
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			multiplier_element = multiplier->element[j];
 			result_elements = &result->element[ i + j ];
 			/* Perform a single multiply, and add the
@@ -75,7 +79,7 @@ void bigint_multiply_raw ( const uint64_t *multiplicand0,
 			 * never overflow beyond the end of the
 			 * result, since:
 			 *
-			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
 			 */
 			__asm__ __volatile__ ( "mul.d   %1, %6, %7\n\t"
 					       "mulh.du %2, %6, %7\n\t"
diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h
index 89e0b8679..a37ac73c9 100644
--- a/src/arch/loong64/include/bits/bigint.h
+++ b/src/arch/loong64/include/bits/bigint.h
@@ -330,7 +330,9 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
 }
 
 extern void bigint_multiply_raw ( const uint64_t *multiplicand0,
+				  unsigned int multiplicand_size,
 				  const uint64_t *multiplier0,
-				  uint64_t *value0, unsigned int size );
+				  unsigned int multiplier_size,
+				  uint64_t *value0 );
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/x86/core/x86_bigint.c b/src/arch/x86/core/x86_bigint.c
index 9a25bdad5..74e5da9a2 100644
--- a/src/arch/x86/core/x86_bigint.c
+++ b/src/arch/x86/core/x86_bigint.c
@@ -36,19 +36,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Multiply big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
  * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements
  */
 void bigint_multiply_raw ( const uint32_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const uint32_t *multiplier0,
-			   uint32_t *result0, unsigned int size ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
+			   unsigned int multiplier_size,
+			   uint32_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 	unsigned int i;
 	unsigned int j;
 	uint32_t multiplicand_element;
@@ -62,9 +66,9 @@ void bigint_multiply_raw ( const uint32_t *multiplicand0,
 	memset ( result, 0, sizeof ( *result ) );
 
 	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < size ; i++ ) {
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < size ; j++ ) {
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			multiplier_element = multiplier->element[j];
 			result_elements = &result->element[ i + j ];
 			/* Perform a single multiply, and add the
@@ -73,7 +77,7 @@ void bigint_multiply_raw ( const uint32_t *multiplicand0,
 			 * never overflow beyond the end of the
 			 * result, since:
 			 *
-			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
 			 */
 			__asm__ __volatile__ ( "mull %5\n\t"
 					       "addl %%eax, (%6,%2,4)\n\t"
diff --git a/src/arch/x86/include/bits/bigint.h b/src/arch/x86/include/bits/bigint.h
index 7443d6fdc..a6bc2ca1d 100644
--- a/src/arch/x86/include/bits/bigint.h
+++ b/src/arch/x86/include/bits/bigint.h
@@ -323,7 +323,9 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused,
 }
 
 extern void bigint_multiply_raw ( const uint32_t *multiplicand0,
+				  unsigned int multiplicand_size,
 				  const uint32_t *multiplier0,
-				  uint32_t *value0, unsigned int size );
+				  unsigned int multiplier_size,
+				  uint32_t *value0 );
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 2f99f8445..36138dd64 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -184,10 +184,11 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v result		Big integer to hold result
  */
 #define bigint_multiply( multiplicand, multiplier, result ) do {	\
-	unsigned int size = bigint_size (multiplicand);			\
+	unsigned int multiplicand_size = bigint_size (multiplicand);	\
+	unsigned int multiplier_size = bigint_size (multiplier);	\
 	bigint_multiply_raw ( (multiplicand)->element,			\
-			      (multiplier)->element, (result)->element,	\
-			      size );					\
+			      multiplicand_size, (multiplier)->element,	\
+			      multiplier_size, (result)->element );	\
 	} while ( 0 )
 
 /**
@@ -283,9 +284,10 @@ void bigint_shrink_raw ( const bigint_element_t *source0,
 			 unsigned int source_size, bigint_element_t *dest0,
 			 unsigned int dest_size );
 void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const bigint_element_t *multiplier0,
-			   bigint_element_t *result0,
-			   unsigned int size );
+			   unsigned int multiplier_size,
+			   bigint_element_t *result0 );
 void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 			       const bigint_element_t *multiplier0,
 			       const bigint_element_t *modulus0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 8d40c3188..02568dffb 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -150,15 +150,17 @@ void bigint_shrink_sample ( const bigint_element_t *source0,
 }
 
 void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
+			      unsigned int multiplicand_size,
 			      const bigint_element_t *multiplier0,
-			      bigint_element_t *result0,
-			      unsigned int size ) {
-	const bigint_t ( size ) *multiplicand __attribute__ (( may_alias ))
-		= ( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) *multiplier __attribute__ (( may_alias ))
-		= ( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) *result __attribute__ (( may_alias ))
-		= ( ( void * ) result0 );
+			      unsigned int multiplier_size,
+			      bigint_element_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 
 	bigint_multiply ( multiplicand, multiplier, result );
 }
@@ -430,17 +432,18 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	static const uint8_t multiplier_raw[] = multiplier;		\
 	static const uint8_t expected_raw[] = expected;			\
 	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
-	unsigned int size =						\
+	unsigned int multiplicand_size =				\
 		bigint_required_size ( sizeof ( multiplicand_raw ) );	\
-	bigint_t ( size ) multiplicand_temp;				\
-	bigint_t ( size ) multiplier_temp;				\
-	bigint_t ( size * 2 ) result_temp;				\
+	unsigned int multiplier_size =					\
+		bigint_required_size ( sizeof ( multiplier_raw ) );	\
+	bigint_t ( multiplicand_size ) multiplicand_temp;		\
+	bigint_t ( multiplier_size ) multiplier_temp;			\
+	bigint_t ( multiplicand_size + multiplier_size ) result_temp;	\
 	{} /* Fix emacs alignment */					\
 									\
-	assert ( bigint_size ( &multiplier_temp ) ==			\
-		 bigint_size ( &multiplicand_temp ) );			\
 	assert ( bigint_size ( &result_temp ) ==			\
-		 ( 2 * bigint_size ( &multiplicand_temp ) ) );		\
+		 ( bigint_size ( &multiplicand_temp ) +			\
+		   bigint_size ( &multiplier_temp ) ) );		\
 	bigint_init ( &multiplicand_temp, multiplicand_raw,		\
 		      sizeof ( multiplicand_raw ) );			\
 	bigint_init ( &multiplier_temp, multiplier_raw,			\
@@ -1373,6 +1376,12 @@ static void bigint_test_exec ( void ) {
 			     BIGINT ( 0x67, 0x3c, 0x5a, 0x16 ),
 			     BIGINT ( 0x3c, 0xdb, 0x7f, 0xae, 0x12, 0x7e,
 				      0xef, 0x16 ) );
+	bigint_multiply_ok ( BIGINT ( 0x39, 0x1f, 0xc8, 0x6a ),
+			     BIGINT ( 0xba, 0x39, 0x4a, 0xb8, 0xac, 0xb3,
+				      0x4f, 0x64, 0x28, 0x46, 0xa6, 0x99 ),
+			     BIGINT ( 0x29, 0x8d, 0xe0, 0x5d, 0x08, 0xea,
+				      0x0d, 0xc7, 0x82, 0x5d, 0xba, 0x96,
+				      0x1c, 0xef, 0x83, 0x5a ) );
 	bigint_multiply_ok ( BIGINT ( 0xe8, 0x08, 0x0b, 0xe9, 0x29, 0x36,
 				      0xea, 0x51, 0x1d, 0x75, 0x1a, 0xd5,
 				      0xba, 0xc6, 0xa0, 0xf3, 0x48, 0x5c,

From 13e390d54edde17c8e22b0f6d8897c273a91c5d0 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 19 Jan 2024 12:29:29 +0000
Subject: [PATCH 052/237] [crypto] Add bigint_copy() as a convenient wrapper
 macro

Big integers may be efficiently copied using bigint_shrink() (which
will always copy only the size of the destination integer), but this
is potentially confusing to a reader of the code.

Provide bigint_copy() as an alias for bigint_shrink() so that the
intention of the calling code may be more obvious.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/bigint.h | 13 +++++++++++++
 src/tests/bigint_test.c   | 10 ++++++++++
 2 files changed, 23 insertions(+)

diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 36138dd64..820d306b8 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -8,6 +8,8 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <assert.h>
+
 /**
  * Define a big-integer type
  *
@@ -176,6 +178,17 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 			    (dest)->element, dest_size );		\
 	} while ( 0 )
 
+/**
+ * Copy big integer
+ *
+ * @v source		Source big integer
+ * @v dest		Destination big integer
+ */
+#define bigint_copy( source, dest ) do {				\
+	build_assert ( sizeof ( *(source) ) == sizeof ( *(dest) ) );	\
+	bigint_shrink ( (source), (dest) );				\
+	} while ( 0 )
+
 /**
  * Multiply big integers
  *
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 02568dffb..484c59134 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -149,6 +149,16 @@ void bigint_shrink_sample ( const bigint_element_t *source0,
 	bigint_shrink ( source, dest );
 }
 
+void bigint_copy_sample ( const bigint_element_t *source0,
+			  bigint_element_t *dest0, unsigned int size ) {
+	const bigint_t ( size ) *source __attribute__ (( may_alias ))
+		= ( ( const void * ) source0 );
+	bigint_t ( size ) *dest __attribute__ (( may_alias ))
+		= ( ( void * ) dest0 );
+
+	bigint_copy ( source, dest );
+}
+
 void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
 			      unsigned int multiplicand_size,
 			      const bigint_element_t *multiplier0,

From bac13ba1f658a1e742b9ceb958e670086affebe7 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 19 Jan 2024 12:34:02 +0000
Subject: [PATCH 053/237] [crypto] Add bigint_swap() to conditionally swap big
 integers

Add a helper function bigint_swap() that can be used to conditionally
swap a pair of big integers in constant time.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       | 25 ++++++++++++++++++
 src/include/ipxe/bigint.h | 15 +++++++++++
 src/tests/bigint_test.c   | 54 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 94 insertions(+)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index ac9670ef6..656f979e5 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -50,6 +50,31 @@ static struct profiler bigint_mod_multiply_rescale_profiler __profiler =
 static struct profiler bigint_mod_multiply_subtract_profiler __profiler =
 	{ .name = "bigint_mod_multiply.subtract" };
 
+/**
+ * Conditionally swap big integers (in constant time)
+ *
+ * @v first0		Element 0 of big integer to be conditionally swapped
+ * @v second0		Element 0 of big integer to be conditionally swapped
+ * @v size		Number of elements in big integers
+ * @v swap		Swap first and second big integers
+ */
+void bigint_swap_raw ( bigint_element_t *first0, bigint_element_t *second0,
+		       unsigned int size, int swap ) {
+	bigint_element_t mask;
+	bigint_element_t xor;
+	unsigned int i;
+
+	/* Construct mask */
+	mask = ( ( bigint_element_t ) ( ! swap ) - 1 );
+
+	/* Conditionally swap elements */
+	for ( i = 0 ; i < size ; i++ ) {
+		xor = ( mask & ( first0[i] ^ second0[i] ) );
+		first0[i] ^= xor;
+		second0[i] ^= xor;
+	}
+}
+
 /**
  * Perform modular multiplication of big integers
  *
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 820d306b8..3dc344dff 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -189,6 +189,19 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	bigint_shrink ( (source), (dest) );				\
 	} while ( 0 )
 
+/**
+ * Conditionally swap big integers (in constant time)
+ *
+ * @v first		Big integer to be conditionally swapped
+ * @v second		Big integer to be conditionally swapped
+ * @v swap		Swap first and second big integers
+ */
+#define bigint_swap( first, second, swap ) do {				\
+	unsigned int size = bigint_size (first);			\
+	bigint_swap_raw ( (first)->element, (second)->element, size,	\
+			  (swap) );					\
+	} while ( 0 )
+
 /**
  * Multiply big integers
  *
@@ -296,6 +309,8 @@ void bigint_grow_raw ( const bigint_element_t *source0,
 void bigint_shrink_raw ( const bigint_element_t *source0,
 			 unsigned int source_size, bigint_element_t *dest0,
 			 unsigned int dest_size );
+void bigint_swap_raw ( bigint_element_t *first0, bigint_element_t *second0,
+		       unsigned int size, int swap );
 void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 			   unsigned int multiplicand_size,
 			   const bigint_element_t *multiplier0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 484c59134..f09d7c76a 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -159,6 +159,16 @@ void bigint_copy_sample ( const bigint_element_t *source0,
 	bigint_copy ( source, dest );
 }
 
+void bigint_swap_sample ( bigint_element_t *first0, bigint_element_t *second0,
+			  unsigned int size, int swap ) {
+	bigint_t ( size ) *first __attribute__ (( may_alias ))
+		= ( ( void * ) first0 );
+	bigint_t ( size ) *second __attribute__ (( may_alias ))
+		= ( ( void * ) second0 );
+
+	bigint_swap ( first, second, swap );
+}
+
 void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
 			      unsigned int multiplicand_size,
 			      const bigint_element_t *multiplier0,
@@ -430,6 +440,42 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	ok ( max_set_bit == (expected) );				\
 	} while ( 0 )
 
+/**
+ * Report result of big integer swap test
+ *
+ * @v first		Big integer to be conditionally swapped
+ * @v second		Big integer to be conditionally swapped
+ */
+#define bigint_swap_ok( first, second ) do {				\
+	static const uint8_t first_raw[] = first;			\
+	static const uint8_t second_raw[] = second;			\
+	uint8_t temp[ sizeof ( first_raw ) ];				\
+	unsigned int size = bigint_required_size ( sizeof ( temp) );	\
+	bigint_t ( size ) first_temp;					\
+	bigint_t ( size ) second_temp;					\
+	{} /* Fix emacs alignment */					\
+									\
+	assert ( sizeof ( first_raw ) == sizeof ( temp ) );		\
+	assert ( sizeof ( second_raw ) == sizeof ( temp ) );		\
+	bigint_init ( &first_temp, first_raw, sizeof ( first_raw ) );	\
+	bigint_init ( &second_temp, second_raw, sizeof ( second_raw ) );\
+	bigint_swap ( &first_temp, &second_temp, 0 );			\
+	bigint_done ( &first_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, first_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_done ( &second_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, second_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_swap ( &first_temp, &second_temp, 1 );			\
+	bigint_done ( &first_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, second_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_done ( &second_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, first_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_swap ( &first_temp, &second_temp, 1 );			\
+	bigint_done ( &first_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, first_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_done ( &second_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, second_raw, sizeof ( temp ) ) == 0 );	\
+	} while ( 0 )
+
 /**
  * Report result of big integer multiplication test
  *
@@ -1373,6 +1419,14 @@ static void bigint_test_exec ( void ) {
 					 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 					 0xff, 0xff ),
 				1024 );
+	bigint_swap_ok ( BIGINT ( 0x68, 0x65, 0x6c, 0x6c, 0x6f ),
+			 BIGINT ( 0x77, 0x6f, 0x72, 0x6c, 0x64 ) );
+	bigint_swap_ok ( BIGINT ( 0xc8, 0x1c, 0x31, 0xd7, 0x13, 0x69, 0x47,
+				  0x32, 0xb0, 0x0a, 0xf7, 0x2d, 0xb9, 0xc3,
+				  0x35, 0x96 ),
+			 BIGINT ( 0x8b, 0x1d, 0x8f, 0x21, 0x76, 0x16, 0x4c,
+				  0xf8, 0xb2, 0x63, 0xed, 0x89, 0x5e, 0x6b,
+				  0x35, 0x7c ) );
 	bigint_multiply_ok ( BIGINT ( 0xf0 ),
 			     BIGINT ( 0xeb ),
 			     BIGINT ( 0xdc, 0x50 ) );

From 908174ec7e49d845c8b52a07a3db32534eade37d Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 19 Jan 2024 16:35:02 +0000
Subject: [PATCH 054/237] [loong64] Replace broken big integer arithmetic
 implementations

The slightly incomprehensible LoongArch64 implementation for
bigint_subtract() is observed to produce incorrect results for some
input values.

Replace the suspicious LoongArch64 implementations of bigint_add(),
bigint_subtract(), bigint_rol() and bigint_ror(), and add a test case
for a subtraction that was producing an incorrect result with the
previous implementation.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/loong64/include/bits/bigint.h | 184 ++++++++++++++-----------
 src/tests/bigint_test.c                |   9 ++
 2 files changed, 115 insertions(+), 78 deletions(-)

diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h
index a37ac73c9..bec748beb 100644
--- a/src/arch/loong64/include/bits/bigint.h
+++ b/src/arch/loong64/include/bits/bigint.h
@@ -53,34 +53,37 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 	uint64_t *discard_value;
 	uint64_t discard_addend_i;
 	uint64_t discard_value_i;
+	uint64_t discard_carry;
+	uint64_t discard_temp;
 	unsigned int discard_size;
-	__asm__ __volatile__ ( "move   $t0, $zero\n"
-			       "1:\n\t"
-			       "ld.d   %3, %0, 0\n\t"
+
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load addend[i] and value[i] */
+			       "ld.d %3, %0, 0\n\t"
+			       "ld.d %4, %1, 0\n\t"
+			       /* Add carry flag and addend */
+			       "add.d %4, %4, %5\n\t"
+			       "sltu %6, %4, %5\n\t"
+			       "add.d %4, %4, %3\n\t"
+			       "sltu %5, %4, %3\n\t"
+			       "or %5, %5, %6\n\t"
+			       /* Store value[i] */
+			       "st.d %4, %1, 0\n\t"
+			       /* Loop */
 			       "addi.d %0, %0, 8\n\t"
-			       "ld.d   %4, %1, 0\n\t"
-
-			       "add.d  %4, %4, $t0\n\t"
-			       "sltu   $t0, %4, $t0\n\t"
-
-			       "add.d  %4, %4, %3\n\t"
-			       "sltu   $t1, %4, %3\n\t"
-
-			       "or     $t0, $t0, $t1\n\t"
-			       "st.d   %4,  %1, 0\n\t"
 			       "addi.d %1, %1, 8\n\t"
 			       "addi.w %2, %2, -1\n\t"
-			       "bnez   %2, 1b"
+			       "bnez %2, 1b\n\t"
 			       : "=r" ( discard_addend ),
 				 "=r" ( discard_value ),
 				 "=r" ( discard_size ),
 				 "=r" ( discard_addend_i ),
 				 "=r" ( discard_value_i ),
+				 "=r" ( discard_carry ),
+				 "=r" ( discard_temp ),
 				 "+m" ( *value )
-			       : "0" ( addend0 ),
-				 "1" ( value0 ),
-				 "2" ( size )
-			       : "t0", "t1" );
+			       : "0" ( addend0 ), "1" ( value0 ),
+				 "2" ( size ), "5" ( 0 ) );
 }
 
 /**
@@ -93,35 +96,43 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 static inline __attribute__ (( always_inline )) void
 bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 		      unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
 	uint64_t *discard_subtrahend;
 	uint64_t *discard_value;
 	uint64_t discard_subtrahend_i;
 	uint64_t discard_value_i;
+	uint64_t discard_carry;
+	uint64_t discard_temp;
 	unsigned int discard_size;
-	unsigned int flag = 0;
 
-	discard_subtrahend = (uint64_t*) subtrahend0;
-	discard_value = value0;
-	discard_size = size;
-
-	do {
-		discard_subtrahend_i = *discard_subtrahend;
-		discard_subtrahend++;
-		discard_value_i = *discard_value;
-
-		discard_value_i = discard_value_i - discard_subtrahend_i - flag;
-
-		if ( *discard_value < (discard_subtrahend_i + flag)) {
-			flag = 1;
-		} else {
-			flag = 0;
-		}
-
-		*discard_value = discard_value_i;
-
-		discard_value++;
-		discard_size -= 1;
-	} while (discard_size != 0);
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load subtrahend[i] and value[i] */
+			       "ld.d %3, %0, 0\n\t"
+			       "ld.d %4, %1, 0\n\t"
+			       /* Subtract carry flag and subtrahend */
+			       "sltu %6, %4, %5\n\t"
+			       "sub.d %4, %4, %5\n\t"
+			       "sltu %5, %4, %3\n\t"
+			       "sub.d %4, %4, %3\n\t"
+			       "or %5, %5, %6\n\t"
+			       /* Store value[i] */
+			       "st.d %4, %1, 0\n\t"
+			       /* Loop */
+			       "addi.d %0, %0, 8\n\t"
+			       "addi.d %1, %1, 8\n\t"
+			       "addi.w %2, %2, -1\n\t"
+			       "bnez %2, 1b\n\t"
+			       : "=r" ( discard_subtrahend ),
+				 "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_subtrahend_i ),
+				 "=r" ( discard_value_i ),
+				 "=r" ( discard_carry ),
+				 "=r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "0" ( subtrahend0 ), "1" ( value0 ),
+				 "2" ( size ), "5" ( 0 ) );
 }
 
 /**
@@ -132,30 +143,37 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
  */
 static inline __attribute__ (( always_inline )) void
 bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
 	uint64_t *discard_value;
 	uint64_t discard_value_i;
+	uint64_t discard_carry;
+	uint64_t discard_temp;
 	unsigned int discard_size;
-	uint64_t current_value_i;
-	unsigned int flag = 0;
 
-	discard_value = value0;
-	discard_size = size;
-	do {
-		discard_value_i = *discard_value;
-		current_value_i = discard_value_i;
-
-		discard_value_i += discard_value_i + flag;
-
-		if (discard_value_i < current_value_i) {
-			flag = 1;
-		} else {
-			flag = 0;
-		}
-
-		*discard_value = discard_value_i;
-		discard_value++;
-		discard_size -= 1;
-	} while ( discard_size != 0 );
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load value[i] */
+			       "ld.d %2, %0, 0\n\t"
+			       /* Shift left */
+			       "rotri.d %2, %2, 63\n\t"
+			       "andi %4, %2, 1\n\t"
+			       "xor %2, %2, %4\n\t"
+			       "or %2, %2, %3\n\t"
+			       "move %3, %4\n\t"
+			       /* Store value[i] */
+			       "st.d %2, %0, 0\n\t"
+			       /* Loop  */
+			       "addi.d %0, %0, 8\n\t"
+			       "addi.w %1, %1, -1\n\t"
+			       "bnez %1, 1b\n\t"
+			       : "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_value_i ),
+				 "=r" ( discard_carry ),
+				 "=r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "0" ( value0 ), "1" ( size ), "3" ( 0 )
+			       : "cc" );
 }
 
 /**
@@ -166,27 +184,37 @@ bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
  */
 static inline __attribute__ (( always_inline )) void
 bigint_ror_raw ( uint64_t *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
 	uint64_t *discard_value;
 	uint64_t discard_value_i;
-	uint64_t discard_value_j;
+	uint64_t discard_carry;
+	uint64_t discard_temp;
 	unsigned int discard_size;
 
-	discard_value = value0;
-	discard_size = size;
-
-	discard_value_j = 0;
-
-	do {
-		discard_size -= 1;
-
-		discard_value_i = *(discard_value + discard_size);
-
-		discard_value_j = (discard_value_j << 63) | (discard_value_i >> 1);
-
-		*(discard_value + discard_size) = discard_value_j;
-
-		discard_value_j = discard_value_i;
-	} while ( discard_size > 0 );
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load value[i] */
+			       "ld.d %2, %0, -8\n\t"
+			       /* Shift right */
+			       "andi %4, %2, 1\n\t"
+			       "xor %2, %2, %4\n\t"
+			       "or %2, %2, %3\n\t"
+			       "move %3, %4\n\t"
+			       "rotri.d %2, %2, 1\n\t"
+			       /* Store value[i] */
+			       "st.d %2, %0, -8\n\t"
+			       /* Loop  */
+			       "addi.d %0, %0, -8\n\t"
+			       "addi.w %1, %1, -1\n\t"
+			       "bnez %1, 1b\n\t"
+			       : "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_value_i ),
+				 "=r" ( discard_carry ),
+				 "=r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "0" ( value0 + size ), "1" ( size ), "3" ( 0 )
+			       : "cc" );
 }
 
 /**
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index f09d7c76a..76aca1059 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -714,6 +714,15 @@ static void bigint_test_exec ( void ) {
 	bigint_subtract_ok ( BIGINT ( 0xbb, 0x77, 0x32, 0x5a ),
 			     BIGINT ( 0x5a, 0xd5, 0xfe, 0x28 ),
 			     BIGINT ( 0x9f, 0x5e, 0xcb, 0xce ) );
+	bigint_subtract_ok ( BIGINT ( 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+				      0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+				      0xff, 0xff, 0xff, 0xff, 0xff, 0xff ),
+			     BIGINT ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x2a ),
+			     BIGINT ( 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x2b ) );
 	bigint_subtract_ok ( BIGINT ( 0x7b, 0xaa, 0x16, 0xcf, 0x15, 0x87,
 				      0xe0, 0x4f, 0x2c, 0xa3, 0xec, 0x2f,
 				      0x46, 0xfb, 0x83, 0xc6, 0xe0, 0xee,

From 2eea04c02ca902f8068b6ad0dd522e6e9a81691c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 19 Jan 2024 12:36:11 +0000
Subject: [PATCH 055/237] [crypto] Add X25519 key exchange algorithm

Add an implementation of the X25519 key exchange algorithm as defined
in RFC7748.

This implementation is inspired by and partially based upon the paper
"Implementing Curve25519/X25519: A Tutorial on Elliptic Curve
Cryptography" by Martin Kleppmann, available for download from
https://www.cl.cam.ac.uk/teaching/2122/Crypto/curve25519.pdf

The underlying modular addition, subtraction, and multiplication
operations are completely redesigned for substantially improved
efficiency compared to the TweetNaCl implementation studied in that
paper (approximately 5x-10x faster and with 70% less memory usage).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/x25519.c       | 808 ++++++++++++++++++++++++++++++++++++++
 src/include/ipxe/x25519.h |  91 +++++
 src/tests/tests.c         |   1 +
 src/tests/x25519_test.c   | 571 +++++++++++++++++++++++++++
 4 files changed, 1471 insertions(+)
 create mode 100644 src/crypto/x25519.c
 create mode 100644 src/include/ipxe/x25519.h
 create mode 100644 src/tests/x25519_test.c

diff --git a/src/crypto/x25519.c b/src/crypto/x25519.c
new file mode 100644
index 000000000..750a2a71c
--- /dev/null
+++ b/src/crypto/x25519.c
@@ -0,0 +1,808 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * X25519 key exchange
+ *
+ * This implementation is inspired by and partially based upon the
+ * paper "Implementing Curve25519/X25519: A Tutorial on Elliptic Curve
+ * Cryptography" by Martin Kleppmann, available for download from
+ * https://www.cl.cam.ac.uk/teaching/2122/Crypto/curve25519.pdf
+ *
+ * The underlying modular addition, subtraction, and multiplication
+ * operations are completely redesigned for substantially improved
+ * efficiency compared to the TweetNaCl implementation studied in that
+ * paper.
+ *
+ *				       TweetNaCl	    iPXE
+ *				       ---------	    ----
+ *
+ * Storage size of each big integer	     128	      40
+ * (in bytes)
+ *
+ * Stack usage for key exchange		    1144	     360
+ * (in bytes, large objects only)
+ *
+ * Cost of big integer addition		      16	       5
+ * (in number of 64-bit additions)
+ *
+ * Cost of big integer multiplication	     273	      31
+ * (in number of 64-bit multiplications)
+ *
+ * The implementation is constant-time (provided that the underlying
+ * big integer operations are also constant-time).
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <assert.h>
+#include <ipxe/init.h>
+#include <ipxe/x25519.h>
+
+/** X25519 reduction constant
+ *
+ * The X25519 field prime is p=2^255-19.  This gives us:
+ *
+ *               p = 2^255 - 19
+ *           2^255 = p + 19
+ *           2^255 = 19          (mod p)
+ *       k * 2^255 = k * 19      (mod p)
+ *
+ * We can therefore reduce a value modulo p by taking the high-order
+ * bits of the value from bit 255 and above, multiplying by 19, and
+ * adding this to the low-order 255 bits of the value.
+ *
+ * This would be cumbersome to do in practice since it would require
+ * partitioning the value at a 255-bit boundary (and hence would
+ * require some shifting and masking operations).  However, we can
+ * note that:
+ *
+ *       k * 2^255 = k * 19      (mod p)
+ *   k * 2 * 2^255 = k * 2 * 19  (mod p)
+ *       k * 2^256 = k * 38      (mod p)
+ *
+ * We can therefore simplify the reduction to taking the high order
+ * bits of the value from bit 256 and above, multiplying by 38, and
+ * adding this to the low-order 256 bits of the value.
+ *
+ * Since 256 will inevitably be a multiple of the big integer element
+ * size (typically 32 or 64 bits), this avoids the need to perform any
+ * shifting or masking operations.
+ */
+#define X25519_REDUCE_256 38
+
+/** X25519 multiplication step 1 result
+ *
+ * Step 1 of X25519 multiplication is to compute the product of two
+ * X25519 unsigned 258-bit integers.
+ *
+ * Both multiplication inputs are limited to 258 bits, and so the
+ * product will have at most 516 bits.
+ */
+union x25519_multiply_step1 {
+	/** Raw product
+	 *
+	 * Big integer multiplication produces a result with a number
+	 * of elements equal to the sum of the number of elements in
+	 * each input.
+	 */
+	bigint_t ( X25519_SIZE + X25519_SIZE ) product;
+	/** Partition into low-order and high-order bits
+	 *
+	 * Reduction modulo p requires separating the low-order 256
+	 * bits from the remaining high-order bits.
+	 *
+	 * Since the value will never exceed 516 bits (see above),
+	 * there will be at most 260 high-order bits.
+	 */
+	struct {
+		/** Low-order 256 bits */
+		bigint_t ( bigint_required_size ( ( 256 /* bits */ + 7 ) / 8 ) )
+			low_256bit;
+		/** High-order 260 bits */
+		bigint_t ( bigint_required_size ( ( 260 /* bits */ + 7 ) / 8 ) )
+			high_260bit;
+	} __attribute__ (( packed )) parts;
+};
+
+/** X25519 multiplication step 2 result
+ *
+ * Step 2 of X25519 multiplication is to multiply the high-order 260
+ * bits from step 1 with the 6-bit reduction constant 38, and to add
+ * this to the low-order 256 bits from step 1.
+ *
+ * The multiplication inputs are limited to 260 and 6 bits
+ * respectively, and so the product will have at most 266 bits.  After
+ * adding the low-order 256 bits from step 1, the result will have at
+ * most 267 bits.
+ */
+union x25519_multiply_step2 {
+	/** Raw product
+	 *
+	 * Big integer multiplication produces a result with a number
+	 * of elements equal to the sum of the number of elements in
+	 * each input.
+	 */
+	bigint_t ( bigint_required_size ( ( 260 /* bits */ + 7 ) / 8 ) +
+		   bigint_required_size ( ( 6 /* bits */ + 7 ) / 8 ) ) product;
+	/** Big integer value
+	 *
+	 * The value will never exceed 267 bits (see above), and so
+	 * may be consumed as a normal X25519 big integer.
+	 */
+	x25519_t value;
+	/** Partition into low-order and high-order bits
+	 *
+	 * Reduction modulo p requires separating the low-order 256
+	 * bits from the remaining high-order bits.
+	 *
+	 * Since the value will never exceed 267 bits (see above),
+	 * there will be at most 11 high-order bits.
+	 */
+	struct {
+		/** Low-order 256 bits */
+		bigint_t ( bigint_required_size ( ( 256 /* bits */ + 7 ) / 8 ) )
+			low_256bit;
+		/** High-order 11 bits */
+		bigint_t ( bigint_required_size ( ( 11 /* bits */ + 7 ) / 8 ) )
+			high_11bit;
+	} __attribute__ (( packed )) parts;
+};
+
+/** X25519 multiplication step 3 result
+ *
+ * Step 3 of X25519 multiplication is to multiply the high-order 11
+ * bits from step 2 with the 6-bit reduction constant 38, and to add
+ * this to the low-order 256 bits from step 2.
+ *
+ * The multiplication inputs are limited to 11 and 6 bits
+ * respectively, and so the product will have at most 17 bits.  After
+ * adding the low-order 256 bits from step 2, the result will have at
+ * most 257 bits.
+ */
+union x25519_multiply_step3 {
+	/** Raw product
+	 *
+	 * Big integer multiplication produces a result with a number
+	 * of elements equal to the sum of the number of elements in
+	 * each input.
+	 */
+	bigint_t ( bigint_required_size ( ( 11 /* bits */ + 7 ) / 8 ) +
+		   bigint_required_size ( ( 6 /* bits */ + 7 ) / 8 ) ) product;
+	/** Big integer value
+	 *
+	 * The value will never exceed 267 bits (see above), and so
+	 * may be consumed as a normal X25519 big integer.
+	 */
+	x25519_t value;
+};
+
+/** X25519 multiplication temporary working space
+ *
+ * We overlap the buffers used by each step of the multiplication
+ * calculation to reduce the total stack space required:
+ *
+ * |--------------------------------------------------------|
+ * | <- pad -> | <------------ step 1 result -------------> |
+ * |           | <- low 256 bits -> | <-- high 260 bits --> |
+ * | <------- step 2 result ------> | <-- step 3 result --> |
+ * |--------------------------------------------------------|
+ */
+union x25519_multiply_workspace {
+	/** Step 1 result */
+	struct {
+		/** Padding to avoid collision between steps 1 and 2
+		 *
+		 * The step 2 multiplication consumes the high 260
+		 * bits of step 1, and so the step 2 multiplication
+		 * result must not overlap this portion of the step 1
+		 * result.
+		 */
+		uint8_t pad[ sizeof ( union x25519_multiply_step2 ) -
+			     offsetof ( union x25519_multiply_step1,
+					parts.high_260bit ) ];
+		/** Step 1 result */
+		union x25519_multiply_step1 step1;
+	} __attribute__ (( packed ));
+	/** Steps 2 and 3 results */
+	struct {
+		/** Step 2 result */
+		union x25519_multiply_step2 step2;
+		/** Step 3 result */
+		union x25519_multiply_step3 step3;
+	} __attribute__ (( packed ));
+};
+
+/** An X25519 elliptic curve point in projective coordinates
+ *
+ * A point (x,y) on the Montgomery curve used in X25519 is represented
+ * using projective coordinates (X/Z,Y/Z) so that intermediate
+ * calculations may be performed on both numerator and denominator
+ * separately, with the division step performed only once at the end
+ * of the calculation.
+ *
+ * The group operation calculation is performed using a Montgomery
+ * ladder as:
+ *
+ *   X[2i]   = ( X[i]^2 - Z[i]^2 )^2
+ *   X[2i+1] = ( X[i] * X[i+1] - Z[i] * Z[i+1] )^2
+ *   Z[2i]   = 4 * X[i] * Z[i] * ( X[i]^2 + A * X[i] * Z[i] + Z[i]^2 )
+ *   Z[2i+1] = X[0] * ( X[i] * Z[i+1] - X[i+1] * Z[i] ) ^ 2
+ *
+ * It is therefore not necessary to store (or use) the value of Y.
+ */
+struct x25519_projective {
+	/** X coordinate */
+	union x25519_quad257 X;
+	/** Z coordinate */
+	union x25519_quad257 Z;
+};
+
+/** An X25519 Montgomery ladder step */
+struct x25519_step {
+	/** X[n]/Z[n] */
+	struct x25519_projective x_n;
+	/** X[n+1]/Z[n+1] */
+	struct x25519_projective x_n1;
+};
+
+/** Constant p=2^255-19 (the finite field prime) */
+static const uint8_t x25519_p_raw[] = {
+	0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xed
+};
+
+/** Constant p=2^255-19 (the finite field prime) */
+static x25519_t x25519_p;
+
+/** Constant 2p=2^256-38 */
+static x25519_t x25519_2p;
+
+/** Constant 4p=2^257-76 */
+static x25519_t x25519_4p;
+
+/** Reduction constant (used during multiplication) */
+static const uint8_t x25519_reduce_256_raw[] = { X25519_REDUCE_256 };
+
+/** Reduction constant (used during multiplication) */
+static bigint_t ( bigint_required_size ( sizeof ( x25519_reduce_256_raw ) ) )
+	x25519_reduce_256;
+
+/** Constant 121665 (used in the Montgomery ladder) */
+static const uint8_t x25519_121665_raw[] = { 0x01, 0xdb, 0x41 };
+
+/** Constant 121665 (used in the Montgomery ladder) */
+static union x25519_oct258 x25519_121665;
+
+/**
+ * Initialise constants
+ *
+ */
+static void x25519_init_constants ( void ) {
+
+	/* Construct constant p */
+	bigint_init ( &x25519_p, x25519_p_raw, sizeof ( x25519_p_raw ) );
+
+	/* Construct constant 2p */
+	bigint_copy ( &x25519_p, &x25519_2p );
+	bigint_add ( &x25519_p, &x25519_2p );
+
+	/* Construct constant 4p */
+	bigint_copy ( &x25519_2p, &x25519_4p );
+	bigint_add ( &x25519_2p, &x25519_4p );
+
+	/* Construct reduction constant */
+	bigint_init ( &x25519_reduce_256, x25519_reduce_256_raw,
+		      sizeof ( x25519_reduce_256_raw ) );
+
+	/* Construct constant 121665 */
+	bigint_init ( &x25519_121665.value, x25519_121665_raw,
+		      sizeof ( x25519_121665_raw ) );
+}
+
+/** Initialisation function */
+struct init_fn x25519_init_fn __init_fn ( INIT_NORMAL ) = {
+	.initialise = x25519_init_constants,
+};
+
+/**
+ * Add big integers modulo field prime
+ *
+ * @v augend		Big integer to add
+ * @v addend		Big integer to add
+ * @v result		Big integer to hold result (may overlap augend)
+ */
+static inline __attribute__ (( always_inline )) void
+x25519_add ( const union x25519_quad257 *augend,
+	     const union x25519_quad257 *addend,
+	     union x25519_oct258 *result ) {
+	int copy;
+
+	/* Copy augend if necessary */
+	copy = ( result != &augend->oct258 );
+	build_assert ( __builtin_constant_p ( copy ) );
+	if ( copy ) {
+		build_assert ( result != &addend->oct258 );
+		bigint_copy ( &augend->oct258.value, &result->value );
+	}
+
+	/* Perform addition
+	 *
+	 * Both inputs are in the range [0,4p-1] and the resulting
+	 * sum is therefore in the range [0,8p-2].
+	 *
+	 * This range lies within the range [0,8p-1] and the result is
+	 * therefore a valid X25519 unsigned 258-bit integer, as
+	 * required.
+	 */
+	bigint_add ( &addend->value, &result->value );
+}
+
+/**
+ * Subtract big integers modulo field prime
+ *
+ * @v minuend		Big integer from which to subtract
+ * @v subtrahend	Big integer to subtract
+ * @v result		Big integer to hold result (may overlap minuend)
+ */
+static inline __attribute__ (( always_inline )) void
+x25519_subtract ( const union x25519_quad257 *minuend,
+		  const union x25519_quad257 *subtrahend,
+		  union x25519_oct258 *result ) {
+	int copy;
+
+	/* Copy minuend if necessary */
+	copy = ( result != &minuend->oct258 );
+	build_assert ( __builtin_constant_p ( copy ) );
+	if ( copy ) {
+		build_assert ( result != &subtrahend->oct258 );
+		bigint_copy ( &minuend->oct258.value, &result->value );
+	}
+
+	/* Perform subtraction
+	 *
+	 * Both inputs are in the range [0,4p-1] and the resulting
+	 * difference is therefore in the range [1-4p,4p-1].
+	 *
+	 * This range lies partially outside the range [0,8p-1] and
+	 * the result is therefore not yet a valid X25519 unsigned
+	 * 258-bit integer.
+	 */
+	bigint_subtract ( &subtrahend->value, &result->value );
+
+	/* Add constant multiple of field prime p
+	 *
+	 * Add the constant 4p to the result.  This brings the result
+	 * within the range [1,8p-1] (without changing the value
+	 * modulo p).
+	 *
+	 * This range lies within the range [0,8p-1] and the result is
+	 * therefore now a valid X25519 unsigned 258-bit integer, as
+	 * required.
+	 */
+	bigint_add ( &x25519_4p, &result->value );
+}
+
+/**
+ * Multiply big integers modulo field prime
+ *
+ * @v multiplicand	Big integer to be multiplied
+ * @v multiplier	Big integer to be multiplied
+ * @v result		Big integer to hold result (may overlap either input)
+ */
+void x25519_multiply ( const union x25519_oct258 *multiplicand,
+		       const union x25519_oct258 *multiplier,
+		       union x25519_quad257 *result ) {
+	union x25519_multiply_workspace tmp;
+	union x25519_multiply_step1 *step1 = &tmp.step1;
+	union x25519_multiply_step2 *step2 = &tmp.step2;
+	union x25519_multiply_step3 *step3 = &tmp.step3;
+
+	/* Step 1: perform raw multiplication
+	 *
+	 *   step1 = multiplicand * multiplier
+	 *
+	 * Both inputs are 258-bit numbers and the step 1 result is
+	 * therefore 258+258=516 bits.
+	 */
+	static_assert ( sizeof ( step1->product ) >= sizeof ( step1->parts ) );
+	bigint_multiply ( &multiplicand->value, &multiplier->value,
+			  &step1->product );
+
+	/* Step 2: reduce high-order 516-256=260 bits of step 1 result
+	 *
+	 * Use the identity 2^256=38 (mod p) to reduce the high-order
+	 * bits of the step 1 result.  We split the 516-bit result
+	 * from step 1 into its low-order 256 bits and high-order 260
+	 * bits:
+	 *
+	 *   step1 = step1(low 256 bits) + step1(high 260 bits) * 2^256
+	 *
+	 * and then perform the calculation:
+	 *
+	 *   step2 = step1                                              (mod p)
+	 *         = step1(low 256 bits) + step1(high 260 bits) * 2^256 (mod p)
+	 *         = step1(low 256 bits) + step1(high 260 bits) * 38    (mod p)
+	 *
+	 * There are 6 bits in the constant value 38.  The step 2
+	 * multiplication product will therefore have 260+6=266 bits,
+	 * and the step 2 result (after the addition) will therefore
+	 * have 267 bits.
+	 */
+	static_assert ( sizeof ( step2->product ) >= sizeof ( step2->value ) );
+	static_assert ( sizeof ( step2->product ) >= sizeof ( step2->parts ) );
+	bigint_grow ( &step1->parts.low_256bit, &result->value );
+	bigint_multiply ( &step1->parts.high_260bit, &x25519_reduce_256,
+			  &step2->product );
+	bigint_add ( &result->value, &step2->value );
+
+	/* Step 3: reduce high-order 267-256=11 bits of step 2 result
+	 *
+	 * Use the identity 2^256=38 (mod p) again to reduce the
+	 * high-order bits of the step 2 result.  As before, we split
+	 * the 267-bit result from step 2 into its low-order 256 bits
+	 * and high-order 11 bits:
+	 *
+	 *   step2 = step2(low 256 bits) + step2(high 11 bits) * 2^256
+	 *
+	 * and then perform the calculation:
+	 *
+	 *   step3 = step2                                             (mod p)
+	 *         = step2(low 256 bits) + step2(high 11 bits) * 2^256 (mod p)
+	 *         = step2(low 256 bits) + step2(high 11 bits) * 38    (mod p)
+	 *
+	 * There are 6 bits in the constant value 38.  The step 3
+	 * multiplication product will therefore have 11+6=19 bits,
+	 * and the step 3 result (after the addition) will therefore
+	 * have 257 bits.
+	 *
+	 * A loose upper bound for the step 3 result (after the
+	 * addition) is given by:
+	 *
+	 *   step3 < ( 2^256 - 1 ) + ( 2^19 - 1 )
+	 *         < ( 2^257 - 2^256 - 1 ) + ( 2^19 - 1 )
+	 *         < ( 2^257 - 76 ) - 2^256 + 2^19 + 74
+	 *         < 4 * ( 2^255 - 19 ) - 2^256 + 2^19 + 74
+	 *         < 4p - 2^256 + 2^19 + 74
+	 *
+	 * and so the step 3 result is strictly less than 4p, and
+	 * therefore lies within the range [0,4p-1].
+	 */
+	memset ( &step3->value, 0, sizeof ( step3->value ) );
+	bigint_grow ( &step2->parts.low_256bit, &result->value );
+	bigint_multiply ( &step2->parts.high_11bit, &x25519_reduce_256,
+			  &step3->product );
+	bigint_add ( &step3->value, &result->value );
+
+	/* Step 1 calculates the product of the input operands, and
+	 * each subsequent step reduces the number of bits in the
+	 * result while preserving this value (modulo p).  The final
+	 * result is therefore equal to the product of the input
+	 * operands (modulo p), as required.
+	 *
+	 * The step 3 result lies within the range [0,4p-1] and the
+	 * final result is therefore a valid X25519 unsigned 257-bit
+	 * integer, as required.
+	 */
+}
+
+/**
+ * Compute multiplicative inverse
+ *
+ * @v invertend		Big integer to be inverted
+ * @v result		Big integer to hold result (may not overlap input)
+ */
+void x25519_invert ( const union x25519_oct258 *invertend,
+		     union x25519_quad257 *result ) {
+	int i;
+
+	/* Sanity check */
+	assert ( invertend != &result->oct258 );
+
+	/* Calculate inverse as x^(-1)=x^(p-2) where p is the field prime
+	 *
+	 * The field prime is p=2^255-19 and so:
+	 *
+	 *   p - 2 = 2^255 - 21
+	 *         = (2^255 - 1) - 2^4 - 2^2
+	 *
+	 * i.e. p-2 is a 254-bit number in which all bits are set
+	 * apart from bit 2 and bit 4.
+	 *
+	 * We use the square-and-multiply method to compute x^(p-2).
+	 */
+	bigint_copy ( &invertend->value, &result->value );
+	for ( i = 253 ; i >= 0 ; i-- ) {
+
+		/* Square running total */
+		x25519_multiply ( &result->oct258, &result->oct258, result );
+
+		/* For each set bit in the exponent, multiply by invertend */
+		if ( ( i != 2 ) && ( i != 4 ) ) {
+			x25519_multiply ( invertend, &result->oct258, result );
+		}
+	}
+}
+
+/**
+ * Reduce big integer via conditional subtraction
+ *
+ * @v subtrahend	Big integer to subtract
+ * @v value		Big integer to be subtracted from, if possible
+ */
+static void x25519_reduce_by ( const x25519_t *subtrahend, x25519_t *value ) {
+	unsigned int max_bit = ( ( 8 * sizeof ( *value ) ) - 1 );
+	x25519_t tmp;
+
+	/* Conditionally subtract subtrahend
+	 *
+	 * Subtract the subtrahend, discarding the result (in constant
+	 * time) if the subtraction underflows.
+	 */
+	bigint_copy ( value, &tmp );
+	bigint_subtract ( subtrahend, value );
+	bigint_swap ( value, &tmp, bigint_bit_is_set ( value, max_bit ) );
+}
+
+/**
+ * Reduce big integer to canonical range
+ *
+ * @v value		Big integer to be reduced
+ */
+void x25519_reduce ( union x25519_quad257 *value ) {
+
+	/* Conditionally subtract 2p
+	 *
+	 * Subtract twice the field prime, discarding the result (in
+	 * constant time) if the subtraction underflows.
+	 *
+	 * The input value is in the range [0,4p-1].  After this
+	 * conditional subtraction, the value is in the range
+	 * [0,2p-1].
+	 */
+	x25519_reduce_by ( &x25519_2p, &value->value );
+
+	/* Conditionally subtract p
+	 *
+	 * Subtract the field prime, discarding the result (in
+	 * constant time) if the subtraction underflows.
+	 *
+	 * The value is already in the range [0,2p-1].  After this
+	 * conditional subtraction, the value is in the range [0,p-1]
+	 * and is therefore the canonical representation.
+	 */
+	x25519_reduce_by ( &x25519_p, &value->value );
+}
+
+/**
+ * Compute next step of the Montgomery ladder
+ *
+ * @v base		Base point
+ * @v bit		Bit value
+ * @v step		Ladder step
+ */
+static void x25519_step ( const union x25519_quad257 *base, int bit,
+			  struct x25519_step *step ) {
+	union x25519_quad257 *a = &step->x_n.X;
+	union x25519_quad257 *b = &step->x_n1.X;
+	union x25519_quad257 *c = &step->x_n.Z;
+	union x25519_quad257 *d = &step->x_n1.Z;
+	union x25519_oct258 e;
+	union x25519_quad257 f;
+	union x25519_oct258 *v1_e;
+	union x25519_oct258 *v2_a;
+	union x25519_oct258 *v3_c;
+	union x25519_oct258 *v4_b;
+	union x25519_quad257 *v5_d;
+	union x25519_quad257 *v6_f;
+	union x25519_quad257 *v7_a;
+	union x25519_quad257 *v8_c;
+	union x25519_oct258 *v9_e;
+	union x25519_oct258 *v10_a;
+	union x25519_quad257 *v11_b;
+	union x25519_oct258 *v12_c;
+	union x25519_quad257 *v13_a;
+	union x25519_oct258 *v14_a;
+	union x25519_quad257 *v15_c;
+	union x25519_quad257 *v16_a;
+	union x25519_quad257 *v17_d;
+	union x25519_quad257 *v18_b;
+
+	/* See the referenced paper "Implementing Curve25519/X25519: A
+	 * Tutorial on Elliptic Curve Cryptography" for the reasoning
+	 * behind this calculation.
+	 */
+
+	/* Reuse storage locations for intermediate results where possible */
+	v1_e = &e;
+	v2_a = container_of ( &a->value, union x25519_oct258, value );
+	v3_c = container_of ( &c->value, union x25519_oct258, value );
+	v4_b = container_of ( &b->value, union x25519_oct258, value );
+	v5_d = d;
+	v6_f = &f;
+	v7_a = a;
+	v8_c = c;
+	v9_e = &e;
+	v10_a = container_of ( &a->value, union x25519_oct258, value );
+	v11_b = b;
+	v12_c = container_of ( &c->value, union x25519_oct258, value );
+	v13_a = a;
+	v14_a = container_of ( &a->value, union x25519_oct258, value );
+	v15_c = c;
+	v16_a = a;
+	v17_d = d;
+	v18_b = b;
+
+	/* Select inputs */
+	bigint_swap ( &a->value, &b->value, bit );
+	bigint_swap ( &c->value, &d->value, bit );
+
+	/* v1 = a + c */
+	x25519_add ( a, c, v1_e );
+
+	/* v2 = a - c */
+	x25519_subtract ( a, c, v2_a );
+
+	/* v3 = b + d */
+	x25519_add ( b, d, v3_c );
+
+	/* v4 = b - d */
+	x25519_subtract ( b, d, v4_b );
+
+	/* v5 = v1^2 = (a + c)^2 = a^2 + 2ac + c^2 */
+	x25519_multiply ( v1_e, v1_e, v5_d );
+
+	/* v6 = v2^2 = (a - c)^2 = a^2 - 2ac + c^2 */
+	x25519_multiply ( v2_a, v2_a, v6_f );
+
+	/* v7 = v3 * v2 = (b + d) * (a - c) = ab - bc + ad - cd */
+	x25519_multiply ( v3_c, v2_a, v7_a );
+
+	/* v8 = v4 * v1 = (b - d) * (a + c) = ab + bc - ad - cd */
+	x25519_multiply ( v4_b, v1_e, v8_c );
+
+	/* v9 = v7 + v8 = 2 * (ab - cd) */
+	x25519_add ( v7_a, v8_c, v9_e );
+
+	/* v10 = v7 - v8 = 2 * (ad - bc) */
+	x25519_subtract ( v7_a, v8_c, v10_a );
+
+	/* v11 = v10^2 = 4 * (ad - bc)^2 */
+	x25519_multiply ( v10_a, v10_a, v11_b );
+
+	/* v12 = v5 - v6 = (a + c)^2 - (a - c)^2 = 4ac */
+	x25519_subtract ( v5_d, v6_f, v12_c );
+
+	/* v13 = v12 * 121665 = 486660ac = (A-2) * ac */
+	x25519_multiply ( v12_c, &x25519_121665, v13_a );
+
+	/* v14 = v13 + v5 = (A-2) * ac + a^2 + 2ac + c^2 = a^2 + A * ac + c^2 */
+	x25519_add ( v13_a, v5_d, v14_a );
+
+	/* v15 = v12 * v14 = 4ac * (a^2 + A * ac + c^2) */
+	x25519_multiply ( v12_c, v14_a, v15_c );
+
+	/* v16 = v5 * v6 = (a + c)^2 * (a - c)^2 = (a^2 - c^2)^2 */
+	x25519_multiply ( &v5_d->oct258, &v6_f->oct258, v16_a );
+
+	/* v17 = v11 * base = 4 * base * (ad - bc)^2 */
+	x25519_multiply ( &v11_b->oct258, &base->oct258, v17_d );
+
+	/* v18 = v9^2 = 4 * (ab - cd)^2 */
+	x25519_multiply ( v9_e, v9_e, v18_b );
+
+	/* Select outputs */
+	bigint_swap ( &a->value, &b->value, bit );
+	bigint_swap ( &c->value, &d->value, bit );
+}
+
+/**
+ * Multiply X25519 elliptic curve point
+ *
+ * @v base		Base point
+ * @v scalar		Scalar multiple
+ * @v result		Point to hold result (may overlap base point)
+ */
+static void x25519_ladder ( const union x25519_quad257 *base,
+			    struct x25519_value *scalar,
+			    union x25519_quad257 *result ) {
+	static const uint8_t zero[] = { 0 };
+	static const uint8_t one[] = { 1 };
+	struct x25519_step step;
+	union x25519_quad257 *tmp;
+	int bit;
+	int i;
+
+	/* Initialise ladder */
+	bigint_init ( &step.x_n.X.value, one, sizeof ( one ) );
+	bigint_init ( &step.x_n.Z.value, zero, sizeof ( zero ) );
+	bigint_copy ( &base->value, &step.x_n1.X.value );
+	bigint_init ( &step.x_n1.Z.value, one, sizeof ( one ) );
+
+	/* Use ladder */
+	for ( i = 254 ; i >= 0 ; i-- ) {
+		bit = ( ( scalar->raw[ i / 8 ] >> ( i % 8 ) ) & 1 );
+		x25519_step ( base, bit, &step );
+	}
+
+	/* Convert back to affine coordinate */
+	tmp = &step.x_n1.X;
+	x25519_invert ( &step.x_n.Z.oct258, tmp );
+	x25519_multiply ( &step.x_n.X.oct258, &tmp->oct258, result );
+	x25519_reduce ( result );
+}
+
+/**
+ * Reverse X25519 value endianness
+ *
+ * @v value		Value to reverse
+ */
+static void x25519_reverse ( struct x25519_value *value ) {
+	uint8_t *low = value->raw;
+	uint8_t *high = &value->raw[ sizeof ( value->raw ) - 1 ];
+	uint8_t tmp;
+
+	/* Reverse bytes */
+	do {
+		tmp = *low;
+		*low = *high;
+		*high = tmp;
+	} while ( ++low < --high );
+}
+
+/**
+ * Calculate X25519 key
+ *
+ * @v base		Base point
+ * @v scalar		Scalar multiple
+ * @v result		Point to hold result (may overlap base point)
+ */
+void x25519_key ( const struct x25519_value *base,
+		  const struct x25519_value *scalar,
+		  struct x25519_value *result ) {
+	struct x25519_value *tmp = result;
+	union x25519_quad257 point;
+
+	/* Reverse base point and clear high bit as required by RFC7748 */
+	memcpy ( tmp, base, sizeof ( *tmp ) );
+	x25519_reverse ( tmp );
+	tmp->raw[0] &= 0x7f;
+	bigint_init ( &point.value, tmp->raw, sizeof ( tmp->raw ) );
+
+	/* Clamp scalar as required by RFC7748 */
+	memcpy ( tmp, scalar, sizeof ( *tmp ) );
+	tmp->raw[0] &= 0xf8;
+	tmp->raw[31] |= 0x40;
+
+	/* Multiply elliptic curve point */
+	x25519_ladder ( &point, tmp, &point );
+
+	/* Reverse result */
+	bigint_done ( &point.value, result->raw, sizeof ( result->raw ) );
+	x25519_reverse ( result );
+}
diff --git a/src/include/ipxe/x25519.h b/src/include/ipxe/x25519.h
new file mode 100644
index 000000000..7a86c1134
--- /dev/null
+++ b/src/include/ipxe/x25519.h
@@ -0,0 +1,91 @@
+#ifndef _IPXE_X25519_H
+#define _IPXE_X25519_H
+
+/** @file
+ *
+ * X25519 key exchange
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/bigint.h>
+
+/** X25519 unsigned big integer size
+ *
+ * X25519 uses the finite field of integers modulo the prime
+ * p=2^255-19.  The canonical representations of integers in this
+ * field therefore require only 255 bits.
+ *
+ * For internal calculations we use big integers containing up to 267
+ * bits, since this ends up allowing us to avoid some unnecessary (and
+ * expensive) intermediate reductions modulo p.
+ */
+#define X25519_SIZE bigint_required_size ( ( 267 /* bits */ + 7 ) / 8 )
+
+/** An X25519 unsigned big integer used in internal calculations */
+typedef bigint_t ( X25519_SIZE ) x25519_t;
+
+/** An X25519 unsigned 258-bit integer
+ *
+ * This is an unsigned integer N in the finite field of integers
+ * modulo the prime p=2^255-19.
+ *
+ * In this representation, N is encoded as any big integer that is in
+ * the same congruence class as N (i.e that has the same value as N
+ * modulo p) and that lies within the 258-bit range [0,8p-1].
+ *
+ * This type can be used as an input for multiplication (but not for
+ * addition or subtraction).
+ *
+ * Addition or subtraction will produce an output of this type.
+ */
+union x25519_oct258 {
+	/** Big integer value */
+	x25519_t value;
+};
+
+/** An X25519 unsigned 257-bit integer
+ *
+ * This is an unsigned integer N in the finite field of integers
+ * modulo the prime p=2^255-19.
+ *
+ * In this representation, N is encoded as any big integer that is in
+ * the same congruence class as N (i.e that has the same value as N
+ * modulo p) and that lies within the 257-bit range [0,4p-1].
+ *
+ * This type can be used as an input for addition, subtraction, or
+ * multiplication.
+ *
+ * Multiplication will produce an output of this type.
+ */
+union x25519_quad257 {
+	/** Big integer value */
+	x25519_t value;
+	/** X25519 unsigned 258-bit integer
+	 *
+	 * Any value in the range [0,4p-1] is automatically also
+	 * within the range [0,8p-1] and so may be consumed as an
+	 * unsigned 258-bit integer.
+	 */
+	const union x25519_oct258 oct258;
+};
+
+/** An X25519 32-byte value */
+struct x25519_value {
+	/** Raw value */
+	uint8_t raw[32];
+};
+
+extern void x25519_multiply ( const union x25519_oct258 *multiplicand,
+			      const union x25519_oct258 *multiplier,
+			      union x25519_quad257 *result );
+extern void x25519_invert ( const union x25519_oct258 *invertend,
+			    union x25519_quad257 *result );
+extern void x25519_reduce ( union x25519_quad257 *value );
+extern void x25519_key ( const struct x25519_value *base,
+			 const struct x25519_value *scalar,
+			 struct x25519_value *result );
+
+#endif /* _IPXE_X25519_H */
diff --git a/src/tests/tests.c b/src/tests/tests.c
index fbdf562c6..41c199c0b 100644
--- a/src/tests/tests.c
+++ b/src/tests/tests.c
@@ -81,3 +81,4 @@ REQUIRE_OBJECT ( hmac_test );
 REQUIRE_OBJECT ( dhe_test );
 REQUIRE_OBJECT ( gcm_test );
 REQUIRE_OBJECT ( nap_test );
+REQUIRE_OBJECT ( x25519_test );
diff --git a/src/tests/x25519_test.c b/src/tests/x25519_test.c
new file mode 100644
index 000000000..3d781f913
--- /dev/null
+++ b/src/tests/x25519_test.c
@@ -0,0 +1,571 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * X25519 key exchange test
+ *
+ * Full key exchange test vectors are taken from RFC7748.
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdint.h>
+#include <string.h>
+#include <ipxe/x25519.h>
+#include <ipxe/test.h>
+
+/** Define inline multiplicand */
+#define MULTIPLICAND(...) { __VA_ARGS__ }
+
+/** Define inline multiplier */
+#define MULTIPLIER(...) { __VA_ARGS__ }
+
+/** Define inline invertend */
+#define INVERTEND(...) { __VA_ARGS__ }
+
+/** Define inline base point */
+#define BASE(...) { __VA_ARGS__ }
+
+/** Define inline scalar multiple */
+#define SCALAR(...) { __VA_ARGS__ }
+
+/** Define inline expected result */
+#define EXPECTED(...) { __VA_ARGS__ }
+
+/** An X25519 multiplication self-test */
+struct x25519_multiply_test {
+	/** Multiplicand */
+	const void *multiplicand;
+	/** Length of multiplicand */
+	size_t multiplicand_len;
+	/** Multiplier */
+	const void *multiplier;
+	/** Length of multiplier */
+	size_t multiplier_len;
+	/** Expected result */
+	const void *expected;
+	/** Length of expected result */
+	size_t expected_len;
+};
+
+/**
+ * Define an X25519 multiplication test
+ *
+ * @v name		Test name
+ * @v MULTIPLICAND	258-bit multiplicand
+ * @v MULTIPLIER	258-bit multiplier
+ * @v EXPECTED		255-bit expected result
+ * @ret test		X25519 multiplication test
+ */
+#define X25519_MULTIPLY_TEST( name, MULTIPLICAND, MULTIPLIER,		\
+			      EXPECTED )				\
+	static const uint8_t name ## _multiplicand[] = MULTIPLICAND;	\
+	static const uint8_t name ## _multiplier[] = MULTIPLIER;	\
+	static const uint8_t name ## _expected[] = EXPECTED;		\
+	static struct x25519_multiply_test name = {			\
+		.multiplicand = name ## _multiplicand,			\
+		.multiplicand_len = sizeof ( name ## _multiplicand ),	\
+		.multiplier = name ## _multiplier,			\
+		.multiplier_len = sizeof ( name ## _multiplier ),	\
+		.expected = name ## _expected,				\
+		.expected_len = sizeof ( name ## _expected ),		\
+	}
+
+/** An X25519 multiplicative inversion self-test */
+struct x25519_invert_test {
+	/** Invertend */
+	const void *invertend;
+	/** Length of invertend */
+	size_t invertend_len;
+	/** Expected result */
+	const void *expected;
+	/** Length of expected result */
+	size_t expected_len;
+};
+
+/**
+ * Define an X25519 multiplicative inversion test
+ *
+ * @v name		Test name
+ * @v INVERTEND		258-bit invertend
+ * @v EXPECTED		255-bit expected result
+ * @ret test		X25519 multiplicative inversion test
+ */
+#define X25519_INVERT_TEST( name, INVERTEND, EXPECTED )			\
+	static const uint8_t name ## _invertend[] = INVERTEND;		\
+	static const uint8_t name ## _expected[] = EXPECTED;		\
+	static struct x25519_invert_test name = {			\
+		.invertend = name ## _invertend,			\
+		.invertend_len = sizeof ( name ## _invertend ),		\
+		.expected = name ## _expected,				\
+		.expected_len = sizeof ( name ## _expected ),		\
+	}
+
+/** An X25519 key exchange self-test */
+struct x25519_key_test {
+	/** Base */
+	struct x25519_value base;
+	/** Scalar */
+	struct x25519_value scalar;
+	/** Expected result */
+	struct x25519_value expected;
+	/** Number of iterations */
+	unsigned int count;
+};
+
+/**
+ * Define an X25519 key exchange test
+ *
+ * @v name		Test name
+ * @v COUNT		Number of iterations
+ * @v BASE		Base point
+ * @v SCALAR		Scalar multiple
+ * @v EXPECTED		Expected result
+ * @ret test		X25519 key exchange test
+ */
+#define X25519_KEY_TEST( name, COUNT, BASE, SCALAR, EXPECTED )		\
+	static struct x25519_key_test name = {				\
+		.count = COUNT,						\
+		.base = { .raw = BASE },				\
+		.scalar = { .raw = SCALAR },				\
+		.expected = { .raw = EXPECTED },			\
+	}
+
+/**
+ * Report an X25519 multiplication test result
+ *
+ * @v test		X25519 multiplication test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void x25519_multiply_okx ( struct x25519_multiply_test *test,
+				  const char *file, unsigned int line ) {
+	union x25519_oct258 multiplicand;
+	union x25519_oct258 multiplier;
+	union x25519_quad257 expected;
+	union x25519_quad257 actual;
+
+	/* Construct big integers */
+	bigint_init ( &multiplicand.value, test->multiplicand,
+		      test->multiplicand_len );
+	DBGC ( test, "X25519 multiplicand:\n" );
+	DBGC_HDA ( test, 0, &multiplicand, sizeof ( multiplicand ) );
+	bigint_init ( &multiplier.value, test->multiplier,
+		      test->multiplier_len );
+	DBGC ( test, "X25519 multiplier:\n" );
+	DBGC_HDA ( test, 0, &multiplier, sizeof ( multiplier ) );
+	bigint_init ( &expected.value, test->expected, test->expected_len );
+	DBGC ( test, "X25519 expected product:\n" );
+	DBGC_HDA ( test, 0, &expected, sizeof ( expected ) );
+
+	/* Perform multiplication */
+	x25519_multiply ( &multiplicand, &multiplier, &actual );
+
+	/* Reduce result to allow for comparison */
+	x25519_reduce ( &actual );
+	DBGC ( test, "X25519 actual product:\n" );
+	DBGC_HDA ( test, 0, &actual, sizeof ( actual ) );
+
+	/* Compare against expected result */
+	okx ( memcmp ( &actual, &expected, sizeof ( expected ) ) == 0,
+	      file, line );
+}
+#define x25519_multiply_ok( test ) \
+	x25519_multiply_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Report an X25519 multiplicative inversion test result
+ *
+ * @v test		X25519 multiplicative inversion test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void x25519_invert_okx ( struct x25519_invert_test *test,
+				const char *file, unsigned int line ) {
+	static const uint8_t one[] = { 1 };
+	union x25519_oct258 invertend;
+	union x25519_quad257 expected;
+	union x25519_quad257 actual;
+	union x25519_quad257 product;
+	union x25519_quad257 identity;
+
+	/* Construct big integers */
+	bigint_init ( &invertend.value, test->invertend, test->invertend_len );
+	DBGC ( test, "X25519 invertend:\n" );
+	DBGC_HDA ( test, 0, &invertend, sizeof ( invertend ) );
+	bigint_init ( &expected.value, test->expected, test->expected_len );
+	DBGC ( test, "X25519 expected inverse:\n" );
+	DBGC_HDA ( test, 0, &expected, sizeof ( expected ) );
+	bigint_init ( &identity.value, one, sizeof ( one ) );
+
+	/* Perform inversion */
+	x25519_invert ( &invertend, &actual );
+
+	/* Multiply invertend by inverse */
+	x25519_multiply ( &invertend, &actual.oct258, &product );
+
+	/* Reduce results to allow for comparison */
+	x25519_reduce ( &actual );
+	DBGC ( test, "X25519 actual inverse:\n" );
+	DBGC_HDA ( test, 0, &actual, sizeof ( actual ) );
+	x25519_reduce ( &product );
+	DBGC ( test, "X25519 actual product:\n" );
+	DBGC_HDA ( test, 0, &product, sizeof ( product ) );
+
+	/* Compare against expected results */
+	okx ( memcmp ( &actual, &expected, sizeof ( expected ) ) == 0,
+	      file, line );
+	okx ( memcmp ( &product, &identity, sizeof ( identity ) ) == 0,
+	      file, line );
+}
+#define x25519_invert_ok( test ) \
+	x25519_invert_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Report an X25519 key exchange test result
+ *
+ * @v test		X25519 key exchange test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void x25519_key_okx ( struct x25519_key_test *test,
+			     const char *file, unsigned int line ) {
+	struct x25519_value base;
+	struct x25519_value scalar;
+	struct x25519_value actual;
+	unsigned int i;
+
+	/* Construct input values */
+	memcpy ( &base, &test->base, sizeof ( test->base ) );
+	memcpy ( &scalar, &test->scalar, sizeof ( test->scalar ) );
+	DBGC ( test, "X25519 base:\n" );
+	DBGC_HDA ( test, 0, &base, sizeof ( base ) );
+	DBGC ( test, "X25519 scalar:\n" );
+	DBGC_HDA ( test, 0, &scalar, sizeof ( scalar ) );
+	DBGC ( test, "X25519 expected result (x%d):\n", test->count );
+	DBGC_HDA ( test, 0, &test->expected, sizeof ( test->expected ) );
+
+	/* Calculate key */
+	for ( i = 0 ; i < test->count ; i++ ) {
+		x25519_key ( &base, &scalar, &actual );
+		memcpy ( &base, &scalar, sizeof ( base ) );
+		memcpy ( &scalar, &actual, sizeof ( scalar ) );
+	}
+	DBGC ( test, "X25519 actual result (x%d):\n", test->count );
+	DBGC_HDA ( test, 0, &actual, sizeof ( actual ) );
+
+	/* Compare against expected result */
+	okx ( memcmp ( &actual, &test->expected,
+		       sizeof ( test->expected ) ) == 0, file, line );
+}
+#define x25519_key_ok( test ) \
+	x25519_key_okx ( test, __FILE__, __LINE__ )
+
+/* Test multiplying small numbers */
+X25519_MULTIPLY_TEST ( multiply_small, MULTIPLICAND ( 6 ),
+		       MULTIPLIER ( 9 ), EXPECTED ( 6 * 9 ) );
+
+/* Test exact multiple of field prime */
+X25519_MULTIPLY_TEST ( multiply_k_p,
+	MULTIPLICAND ( 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		       0xff, 0xff, 0xff, 0xff, 0xed ),
+	MULTIPLIER ( 0x00, 0xe8, 0x0d, 0x83, 0xd4, 0xe9, 0x1e, 0xdd, 0x7a,
+		     0x45, 0x14, 0x87, 0xb7, 0xfc, 0x62, 0x54, 0x1f, 0xb2,
+		     0x97, 0x24, 0xde, 0xfa, 0xd3, 0xe7, 0x3e, 0x83, 0x93,
+		     0x60, 0xbc, 0x20, 0x97, 0x9b, 0x22 ),
+	EXPECTED ( 0x00 ) );
+
+/* 0x0223b8c1e9392456de3eb13b9046685257bdd640fb06671ad11c80317fa3b1799d *
+ * 0x006c031199972a846916419f828b9d2434e465e150bd9c66b3ad3c2d6d1a3d1fa7 =
+ * 0x1ba87e982f7c477616b4d5136ba54733e40081c1c2e27d864aa178ce893d1297 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_1,
+	MULTIPLICAND ( 0x02, 0x23, 0xb8, 0xc1, 0xe9, 0x39, 0x24, 0x56, 0xde,
+		       0x3e, 0xb1, 0x3b, 0x90, 0x46, 0x68, 0x52, 0x57, 0xbd,
+		       0xd6, 0x40, 0xfb, 0x06, 0x67, 0x1a, 0xd1, 0x1c, 0x80,
+		       0x31, 0x7f, 0xa3, 0xb1, 0x79, 0x9d ),
+	MULTIPLIER ( 0x00, 0x6c, 0x03, 0x11, 0x99, 0x97, 0x2a, 0x84, 0x69,
+		     0x16, 0x41, 0x9f, 0x82, 0x8b, 0x9d, 0x24, 0x34, 0xe4,
+		     0x65, 0xe1, 0x50, 0xbd, 0x9c, 0x66, 0xb3, 0xad, 0x3c,
+		     0x2d, 0x6d, 0x1a, 0x3d, 0x1f, 0xa7 ),
+	EXPECTED ( 0x1b, 0xa8, 0x7e, 0x98, 0x2f, 0x7c, 0x47, 0x76, 0x16, 0xb4,
+		   0xd5, 0x13, 0x6b, 0xa5, 0x47, 0x33, 0xe4, 0x00, 0x81, 0xc1,
+		   0xc2, 0xe2, 0x7d, 0x86, 0x4a, 0xa1, 0x78, 0xce, 0x89, 0x3d,
+		   0x12, 0x97 ) );
+
+/* 0x008fadc1a606cb0fb39a1de644815ef6d13b8faa1837f8a88b17fc695a07a0ca6e *
+ * 0x0196da1dac72ff5d2a386ecbe06b65a6a48b8148f6b38a088ca65ed389b74d0fb1 =
+ * 0x351f7bf75ef580249ed6f9ff3996463b0730a1d49b5d36b863e192591157e950 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_2,
+	MULTIPLICAND ( 0x00, 0x8f, 0xad, 0xc1, 0xa6, 0x06, 0xcb, 0x0f, 0xb3,
+		       0x9a, 0x1d, 0xe6, 0x44, 0x81, 0x5e, 0xf6, 0xd1, 0x3b,
+		       0x8f, 0xaa, 0x18, 0x37, 0xf8, 0xa8, 0x8b, 0x17, 0xfc,
+		       0x69, 0x5a, 0x07, 0xa0, 0xca, 0x6e ),
+	MULTIPLIER ( 0x01, 0x96, 0xda, 0x1d, 0xac, 0x72, 0xff, 0x5d, 0x2a,
+		     0x38, 0x6e, 0xcb, 0xe0, 0x6b, 0x65, 0xa6, 0xa4, 0x8b,
+		     0x81, 0x48, 0xf6, 0xb3, 0x8a, 0x08, 0x8c, 0xa6, 0x5e,
+		     0xd3, 0x89, 0xb7, 0x4d, 0x0f, 0xb1 ),
+	EXPECTED ( 0x35, 0x1f, 0x7b, 0xf7, 0x5e, 0xf5, 0x80, 0x24, 0x9e, 0xd6,
+		   0xf9, 0xff, 0x39, 0x96, 0x46, 0x3b, 0x07, 0x30, 0xa1, 0xd4,
+		   0x9b, 0x5d, 0x36, 0xb8, 0x63, 0xe1, 0x92, 0x59, 0x11, 0x57,
+		   0xe9, 0x50 ) );
+
+/* 0x016c307511b2b9437a28df6ec4ce4a2bbdc241330b01a9e71fde8a774bcf36d58b *
+ * 0x0117be31111a2a73ed562b0f79c37459eef50bea63371ecd7b27cd813047229389 =
+ * 0x6b43b5185965f8f0920f31ae1b2cefadd7b078fecf68dbeaa17b9c385b558329 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_3,
+	MULTIPLICAND ( 0x01, 0x6c, 0x30, 0x75, 0x11, 0xb2, 0xb9, 0x43, 0x7a,
+		       0x28, 0xdf, 0x6e, 0xc4, 0xce, 0x4a, 0x2b, 0xbd, 0xc2,
+		       0x41, 0x33, 0x0b, 0x01, 0xa9, 0xe7, 0x1f, 0xde, 0x8a,
+		       0x77, 0x4b, 0xcf, 0x36, 0xd5, 0x8b ),
+	MULTIPLIER ( 0x01, 0x17, 0xbe, 0x31, 0x11, 0x1a, 0x2a, 0x73, 0xed,
+		     0x56, 0x2b, 0x0f, 0x79, 0xc3, 0x74, 0x59, 0xee, 0xf5,
+		     0x0b, 0xea, 0x63, 0x37, 0x1e, 0xcd, 0x7b, 0x27, 0xcd,
+		     0x81, 0x30, 0x47, 0x22, 0x93, 0x89 ),
+	EXPECTED ( 0x6b, 0x43, 0xb5, 0x18, 0x59, 0x65, 0xf8, 0xf0, 0x92, 0x0f,
+		   0x31, 0xae, 0x1b, 0x2c, 0xef, 0xad, 0xd7, 0xb0, 0x78, 0xfe,
+		   0xcf, 0x68, 0xdb, 0xea, 0xa1, 0x7b, 0x9c, 0x38, 0x5b, 0x55,
+		   0x83, 0x29 ) );
+
+/* 0x020b1f9163ce9ff57f43b7a3a69a8dca03580d7b71d8f564135be6128e18c26797 *
+ * 0x018d5288f1142c3fe860e7a113ec1b8ca1f91e1d4c1ff49b7889463e85759cde66 =
+ * 0x28a77d3c8a14323d63b288dbd40315b3f192b8485d86a02cb87d3dfb7a0b5447 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_4,
+	MULTIPLICAND ( 0x02, 0x0b, 0x1f, 0x91, 0x63, 0xce, 0x9f, 0xf5, 0x7f,
+		       0x43, 0xb7, 0xa3, 0xa6, 0x9a, 0x8d, 0xca, 0x03, 0x58,
+		       0x0d, 0x7b, 0x71, 0xd8, 0xf5, 0x64, 0x13, 0x5b, 0xe6,
+		       0x12, 0x8e, 0x18, 0xc2, 0x67, 0x97 ),
+	MULTIPLIER ( 0x01, 0x8d, 0x52, 0x88, 0xf1, 0x14, 0x2c, 0x3f, 0xe8,
+		     0x60, 0xe7, 0xa1, 0x13, 0xec, 0x1b, 0x8c, 0xa1, 0xf9,
+		     0x1e, 0x1d, 0x4c, 0x1f, 0xf4, 0x9b, 0x78, 0x89, 0x46,
+		     0x3e, 0x85, 0x75, 0x9c, 0xde, 0x66 ),
+	EXPECTED ( 0x28, 0xa7, 0x7d, 0x3c, 0x8a, 0x14, 0x32, 0x3d, 0x63, 0xb2,
+		   0x88, 0xdb, 0xd4, 0x03, 0x15, 0xb3, 0xf1, 0x92, 0xb8, 0x48,
+		   0x5d, 0x86, 0xa0, 0x2c, 0xb8, 0x7d, 0x3d, 0xfb, 0x7a, 0x0b,
+		   0x54, 0x47 ) );
+
+/* 0x023139d32c93cd59bf5c941cf0dc98d2c1e2acf72f9e574f7aa0ee89aed453dd32 *
+ * 0x03146d3f31fc377a4c4a15544dc5e7ce8a3a578a8ea9488d990bbb259911ce5dd2 =
+ * 0x4bdb7a35c0a5182000aa67554741e88cfdf460a78c6fae07adf83d2f005d2767 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_5,
+	MULTIPLICAND ( 0x02, 0x31, 0x39, 0xd3, 0x2c, 0x93, 0xcd, 0x59, 0xbf,
+		       0x5c, 0x94, 0x1c, 0xf0, 0xdc, 0x98, 0xd2, 0xc1, 0xe2,
+		       0xac, 0xf7, 0x2f, 0x9e, 0x57, 0x4f, 0x7a, 0xa0, 0xee,
+		       0x89, 0xae, 0xd4, 0x53, 0xdd, 0x32 ),
+	MULTIPLIER ( 0x03, 0x14, 0x6d, 0x3f, 0x31, 0xfc, 0x37, 0x7a, 0x4c,
+		     0x4a, 0x15, 0x54, 0x4d, 0xc5, 0xe7, 0xce, 0x8a, 0x3a,
+		     0x57, 0x8a, 0x8e, 0xa9, 0x48, 0x8d, 0x99, 0x0b, 0xbb,
+		     0x25, 0x99, 0x11, 0xce, 0x5d, 0xd2 ),
+	EXPECTED ( 0x4b, 0xdb, 0x7a, 0x35, 0xc0, 0xa5, 0x18, 0x20, 0x00, 0xaa,
+		   0x67, 0x55, 0x47, 0x41, 0xe8, 0x8c, 0xfd, 0xf4, 0x60, 0xa7,
+		   0x8c, 0x6f, 0xae, 0x07, 0xad, 0xf8, 0x3d, 0x2f, 0x00, 0x5d,
+		   0x27, 0x67 ) );
+
+/* 0x01d58842dea2bc372f7412b29347294739614ff3d719db3ad0ddd1dfb23b982ef8 ^ -1 =
+ * 0x093ff51750809d181a9a5481c564e37cff618def8ec45f464b1a6e24f8b826bd (mod p)
+ */
+X25519_INVERT_TEST ( invert_1,
+	INVERTEND ( 0x01, 0xd5, 0x88, 0x42, 0xde, 0xa2, 0xbc, 0x37, 0x2f,
+		    0x74, 0x12, 0xb2, 0x93, 0x47, 0x29, 0x47, 0x39, 0x61,
+		    0x4f, 0xf3, 0xd7, 0x19, 0xdb, 0x3a, 0xd0, 0xdd, 0xd1,
+		    0xdf, 0xb2, 0x3b, 0x98, 0x2e, 0xf8 ),
+	EXPECTED ( 0x09, 0x3f, 0xf5, 0x17, 0x50, 0x80, 0x9d, 0x18, 0x1a, 0x9a,
+		   0x54, 0x81, 0xc5, 0x64, 0xe3, 0x7c, 0xff, 0x61, 0x8d, 0xef,
+		   0x8e, 0xc4, 0x5f, 0x46, 0x4b, 0x1a, 0x6e, 0x24, 0xf8, 0xb8,
+		   0x26, 0xbd ) );
+
+/* 0x02efc89849b3aa7efe4458a885ab9099a435a240ae5af305535ec42e0829a3b2e9 ^ -1 =
+ * 0x591607b163e89d0ac33a62c881e984a25d3826e3db5ce229af240dc58e5b579a (mod p)
+ */
+X25519_INVERT_TEST ( invert_2,
+	INVERTEND ( 0x02, 0xef, 0xc8, 0x98, 0x49, 0xb3, 0xaa, 0x7e, 0xfe,
+		    0x44, 0x58, 0xa8, 0x85, 0xab, 0x90, 0x99, 0xa4, 0x35,
+		    0xa2, 0x40, 0xae, 0x5a, 0xf3, 0x05, 0x53, 0x5e, 0xc4,
+		    0x2e, 0x08, 0x29, 0xa3, 0xb2, 0xe9 ),
+	EXPECTED ( 0x59, 0x16, 0x07, 0xb1, 0x63, 0xe8, 0x9d, 0x0a, 0xc3, 0x3a,
+		   0x62, 0xc8, 0x81, 0xe9, 0x84, 0xa2, 0x5d, 0x38, 0x26, 0xe3,
+		   0xdb, 0x5c, 0xe2, 0x29, 0xaf, 0x24, 0x0d, 0xc5, 0x8e, 0x5b,
+		   0x57, 0x9a ) );
+
+/* 0x003eabedcbbaa80dd488bd64072bcfbe01a28defe39bf0027312476f57a5e5a5ab ^ -1 =
+ * 0x7d87c2e565b27c5038181a0a7cae9ebe826c8afc1f77128a4d62cce96d2759a2 (mod p)
+ */
+X25519_INVERT_TEST ( invert_3,
+	INVERTEND ( 0x00, 0x3e, 0xab, 0xed, 0xcb, 0xba, 0xa8, 0x0d, 0xd4,
+		    0x88, 0xbd, 0x64, 0x07, 0x2b, 0xcf, 0xbe, 0x01, 0xa2,
+		    0x8d, 0xef, 0xe3, 0x9b, 0xf0, 0x02, 0x73, 0x12, 0x47,
+		    0x6f, 0x57, 0xa5, 0xe5, 0xa5, 0xab ),
+	EXPECTED ( 0x7d, 0x87, 0xc2, 0xe5, 0x65, 0xb2, 0x7c, 0x50, 0x38, 0x18,
+		   0x1a, 0x0a, 0x7c, 0xae, 0x9e, 0xbe, 0x82, 0x6c, 0x8a, 0xfc,
+		   0x1f, 0x77, 0x12, 0x8a, 0x4d, 0x62, 0xcc, 0xe9, 0x6d, 0x27,
+		   0x59, 0xa2 ) );
+
+/* 0x008e944239b02b61c4a3d70628ece66fa2fd5166e6451b4cf36123fdf77656af72 ^ -1 =
+ * 0x08e96161a0eee1b29af396f154950d5c715dc61aff66ee97377ab22adf3321d7 (mod p)
+ */
+X25519_INVERT_TEST ( invert_4,
+	INVERTEND ( 0x00, 0x8e, 0x94, 0x42, 0x39, 0xb0, 0x2b, 0x61, 0xc4,
+		    0xa3, 0xd7, 0x06, 0x28, 0xec, 0xe6, 0x6f, 0xa2, 0xfd,
+		    0x51, 0x66, 0xe6, 0x45, 0x1b, 0x4c, 0xf3, 0x61, 0x23,
+		    0xfd, 0xf7, 0x76, 0x56, 0xaf, 0x72 ),
+	EXPECTED ( 0x08, 0xe9, 0x61, 0x61, 0xa0, 0xee, 0xe1, 0xb2, 0x9a, 0xf3,
+		   0x96, 0xf1, 0x54, 0x95, 0x0d, 0x5c, 0x71, 0x5d, 0xc6, 0x1a,
+		   0xff, 0x66, 0xee, 0x97, 0x37, 0x7a, 0xb2, 0x2a, 0xdf, 0x33,
+		   0x21, 0xd7 ) );
+
+/* 0x00d261a7ab3aa2e4f90e51f30dc6a7ee39c4b032ccd7c524a55304317faf42e12f ^ -1 =
+ * 0x0738781c0aeabfbe6e840c85bd30996ef71bc54988ce16cedd5ab4f30c281597 (mod p)
+ */
+X25519_INVERT_TEST ( invert_5,
+	INVERTEND ( 0x00, 0xd2, 0x61, 0xa7, 0xab, 0x3a, 0xa2, 0xe4, 0xf9,
+		    0x0e, 0x51, 0xf3, 0x0d, 0xc6, 0xa7, 0xee, 0x39, 0xc4,
+		    0xb0, 0x32, 0xcc, 0xd7, 0xc5, 0x24, 0xa5, 0x53, 0x04,
+		    0x31, 0x7f, 0xaf, 0x42, 0xe1, 0x2f ),
+	EXPECTED ( 0x07, 0x38, 0x78, 0x1c, 0x0a, 0xea, 0xbf, 0xbe, 0x6e, 0x84,
+		   0x0c, 0x85, 0xbd, 0x30, 0x99, 0x6e, 0xf7, 0x1b, 0xc5, 0x49,
+		   0x88, 0xce, 0x16, 0xce, 0xdd, 0x5a, 0xb4, 0xf3, 0x0c, 0x28,
+		   0x15, 0x97 ) );
+
+/* Base:   0xe6db6867583030db3594c1a424b15f7c726624ec26b3353b10a903a6d0ab1c4c
+ * Scalar: 0xa546e36bf0527c9d3b16154b82465edd62144c0ac1fc5a18506a2244ba449ac4
+ * Result: 0xc3da55379de9c6908e94ea4df28d084f32eccf03491c71f754b4075577a28552
+ */
+X25519_KEY_TEST ( rfc7748_1, 1,
+	BASE ( 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, 0x35, 0x94,
+	       0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, 0x72, 0x66, 0x24, 0xec,
+	       0x26, 0xb3, 0x35, 0x3b, 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab,
+	       0x1c, 0x4c ),
+	SCALAR ( 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, 0x3b, 0x16,
+		 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, 0x62, 0x14, 0x4c, 0x0a,
+		 0xc1, 0xfc, 0x5a, 0x18, 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44,
+		 0x9a, 0xc4 ),
+	EXPECTED ( 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, 0x8e, 0x94,
+		   0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, 0x32, 0xec, 0xcf, 0x03,
+		   0x49, 0x1c, 0x71, 0xf7, 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2,
+		   0x85, 0x52 ) );
+
+/* Base:   0xe5210f12786811d3f4b7959d0538ae2c31dbe7106fc03c3efc4cd549c715a493
+ * Scalar: 0x4b66e9d4d1b4673c5ad22691957d6af5c11b6421e0ea01d42ca4169e7918ba0d
+ * Result: 0x95cbde9476e8907d7aade45cb4b873f88b595a68799fa152e6f8f7647aac7957
+ */
+X25519_KEY_TEST ( rfc7748_2, 1,
+	BASE ( 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3, 0xf4, 0xb7,
+	       0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c, 0x31, 0xdb, 0xe7, 0x10,
+	       0x6f, 0xc0, 0x3c, 0x3e, 0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15,
+	       0xa4, 0x93 ),
+	SCALAR ( 0x4b, 0x66, 0xe9, 0xd4, 0xd1, 0xb4, 0x67, 0x3c, 0x5a, 0xd2,
+		 0x26, 0x91, 0x95, 0x7d, 0x6a, 0xf5, 0xc1, 0x1b, 0x64, 0x21,
+		 0xe0, 0xea, 0x01, 0xd4, 0x2c, 0xa4, 0x16, 0x9e, 0x79, 0x18,
+		 0xba, 0x0d ),
+	EXPECTED ( 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, 0x7a, 0xad,
+		   0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, 0x8b, 0x59, 0x5a, 0x68,
+		   0x79, 0x9f, 0xa1, 0x52, 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac,
+		   0x79, 0x57 ) );
+
+/* Base:   0x0900000000000000000000000000000000000000000000000000000000000000
+ * Scalar: 0x0900000000000000000000000000000000000000000000000000000000000000
+ * Result: 0x422c8e7a6227d7bca1350b3e2bb7279f7897b87bb6854b783c60e80311ae3079
+ */
+X25519_KEY_TEST ( rfc7748_3, 1,
+	BASE ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00 ),
+	SCALAR ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00 ),
+	EXPECTED ( 0x42, 0x2c, 0x8e, 0x7a, 0x62, 0x27, 0xd7, 0xbc, 0xa1, 0x35,
+		   0x0b, 0x3e, 0x2b, 0xb7, 0x27, 0x9f, 0x78, 0x97, 0xb8, 0x7b,
+		   0xb6, 0x85, 0x4b, 0x78, 0x3c, 0x60, 0xe8, 0x03, 0x11, 0xae,
+		   0x30, 0x79 ) );
+
+/* Base:   0x0900000000000000000000000000000000000000000000000000000000000000
+ * Scalar: 0x0900000000000000000000000000000000000000000000000000000000000000
+ * Result: 0xb1a5a73158904c020866c13939dd7e1aa26852ee1d2609c92e5a8f1debe2150a
+ * (after 100 iterations)
+ *
+ * RFC7748 gives test vectors for 1000 and 1000000 iterations with
+ * these starting values.  This test case stops after 100 iterations
+ * to avoid a pointlessly slow test cycle in the common case of
+ * running tests under Valgrind.
+ */
+X25519_KEY_TEST ( rfc7748_4_100, 100,
+	BASE ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00 ),
+	SCALAR ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00 ),
+	EXPECTED ( 0xb1, 0xa5, 0xa7, 0x31, 0x58, 0x90, 0x4c, 0x02, 0x08, 0x66,
+		   0xc1, 0x39, 0x39, 0xdd, 0x7e, 0x1a, 0xa2, 0x68, 0x52, 0xee,
+		   0x1d, 0x26, 0x09, 0xc9, 0x2e, 0x5a, 0x8f, 0x1d, 0xeb, 0xe2,
+		   0x15, 0x0a ) );
+
+/**
+ * Perform X25519 self-tests
+ *
+ */
+static void x25519_test_exec ( void ) {
+
+	/* Perform multiplication tests */
+	x25519_multiply_ok ( &multiply_small );
+	x25519_multiply_ok ( &multiply_k_p );
+	x25519_multiply_ok ( &multiply_1 );
+	x25519_multiply_ok ( &multiply_2 );
+	x25519_multiply_ok ( &multiply_3 );
+	x25519_multiply_ok ( &multiply_4 );
+	x25519_multiply_ok ( &multiply_5 );
+
+	/* Perform multiplicative inversion tests */
+	x25519_invert_ok ( &invert_1 );
+	x25519_invert_ok ( &invert_2 );
+	x25519_invert_ok ( &invert_3 );
+	x25519_invert_ok ( &invert_4 );
+	x25519_invert_ok ( &invert_5 );
+
+	/* Perform key exchange tests */
+	x25519_key_ok ( &rfc7748_1 );
+	x25519_key_ok ( &rfc7748_2 );
+	x25519_key_ok ( &rfc7748_3 );
+	x25519_key_ok ( &rfc7748_4_100 );
+}
+
+/** X25519 self-test */
+struct self_test x25519_test __self_test = {
+	.name = "x25519",
+	.exec = x25519_test_exec,
+};

From de8a0821c7bc737e724fa3dfb6d89dc36f591d7a Mon Sep 17 00:00:00 2001
From: Joseph Wong <joseph.wong@broadcom.com>
Date: Thu, 18 Jan 2024 21:53:44 -0800
Subject: [PATCH 056/237] [bnxt] Add support for additional chip IDs

Add additional chip IDs that can be recognized as part of the thor
family.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/bnxt/bnxt.c | 4 +++-
 src/drivers/net/bnxt/bnxt.h | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/drivers/net/bnxt/bnxt.c b/src/drivers/net/bnxt/bnxt.c
index 605aea328..5dd217971 100644
--- a/src/drivers/net/bnxt/bnxt.c
+++ b/src/drivers/net/bnxt/bnxt.c
@@ -722,7 +722,9 @@ static int bnxt_hwrm_ver_get ( struct bnxt *bp )
 		 ( resp->dev_caps_cfg & SHORT_CMD_REQUIRED ) )
 		FLAG_SET ( bp->flags, BNXT_FLAG_HWRM_SHORT_CMD_SUPP );
 	bp->hwrm_max_ext_req_len = resp->max_ext_req_len;
-	if ( bp->chip_num == CHIP_NUM_57500 )
+	if ( ( bp->chip_num == CHIP_NUM_57508 ) ||
+	     ( bp->chip_num == CHIP_NUM_57504 ) ||
+	     ( bp->chip_num == CHIP_NUM_57502 ) )
 		bp->thor = 1;
 	dbg_fw_ver ( resp, bp->hwrm_cmd_timeout );
 	return STATUS_SUCCESS;
diff --git a/src/drivers/net/bnxt/bnxt.h b/src/drivers/net/bnxt/bnxt.h
index 2cbaec5e5..cf2dea8bc 100644
--- a/src/drivers/net/bnxt/bnxt.h
+++ b/src/drivers/net/bnxt/bnxt.h
@@ -868,4 +868,6 @@ struct bnxt {
 	FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR | \
 	FUNC_VF_CFG_REQ_ENABLES_DFLT_MAC_ADDR)
 
-#define CHIP_NUM_57500       0x1750
+#define CHIP_NUM_57508       0x1750
+#define CHIP_NUM_57504       0x1751
+#define CHIP_NUM_57502       0x1752

From 27398f136030efb8845c45fbe154e544feefd7e5 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 13:14:21 +0000
Subject: [PATCH 057/237] [crypto] Check for all-zeros result from X25519 key
 exchange

RFC7748 states that it is entirely optional for X25519 Diffie-Hellman
implementations to check whether or not the result is the all-zero
value (indicating that an attacker sent a malicious public key with a
small order).  RFC8422 states that implementations in TLS must abort
the handshake if the all-zero value is obtained.

Return an error if the all-zero value is obtained, so that the TLS
code will not require knowledge specific to the X25519 curve.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/x25519.c        | 11 +++++++---
 src/include/ipxe/errfile.h |  1 +
 src/include/ipxe/x25519.h  |  6 +++---
 src/tests/x25519_test.c    | 41 ++++++++++++++++++++++++++++++++------
 4 files changed, 47 insertions(+), 12 deletions(-)

diff --git a/src/crypto/x25519.c b/src/crypto/x25519.c
index 750a2a71c..d3a19bc8e 100644
--- a/src/crypto/x25519.c
+++ b/src/crypto/x25519.c
@@ -59,6 +59,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <stdint.h>
 #include <string.h>
 #include <assert.h>
+#include <errno.h>
 #include <ipxe/init.h>
 #include <ipxe/x25519.h>
 
@@ -781,10 +782,11 @@ static void x25519_reverse ( struct x25519_value *value ) {
  * @v base		Base point
  * @v scalar		Scalar multiple
  * @v result		Point to hold result (may overlap base point)
+ * @ret rc		Return status code
  */
-void x25519_key ( const struct x25519_value *base,
-		  const struct x25519_value *scalar,
-		  struct x25519_value *result ) {
+int x25519_key ( const struct x25519_value *base,
+		 const struct x25519_value *scalar,
+		 struct x25519_value *result ) {
 	struct x25519_value *tmp = result;
 	union x25519_quad257 point;
 
@@ -805,4 +807,7 @@ void x25519_key ( const struct x25519_value *base,
 	/* Reverse result */
 	bigint_done ( &point.value, result->raw, sizeof ( result->raw ) );
 	x25519_reverse ( result );
+
+	/* Fail if result was all zeros (as required by RFC8422) */
+	return ( bigint_is_zero ( &point.value ) ? -EPERM : 0 );
 }
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 320835a34..060a42a33 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -407,6 +407,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_efi_rng		      ( ERRFILE_OTHER | 0x005c0000 )
 #define ERRFILE_efi_shim	      ( ERRFILE_OTHER | 0x005d0000 )
 #define ERRFILE_efi_settings	      ( ERRFILE_OTHER | 0x005e0000 )
+#define ERRFILE_x25519		      ( ERRFILE_OTHER | 0x005f0000 )
 
 /** @} */
 
diff --git a/src/include/ipxe/x25519.h b/src/include/ipxe/x25519.h
index 7a86c1134..6524abb7d 100644
--- a/src/include/ipxe/x25519.h
+++ b/src/include/ipxe/x25519.h
@@ -84,8 +84,8 @@ extern void x25519_multiply ( const union x25519_oct258 *multiplicand,
 extern void x25519_invert ( const union x25519_oct258 *invertend,
 			    union x25519_quad257 *result );
 extern void x25519_reduce ( union x25519_quad257 *value );
-extern void x25519_key ( const struct x25519_value *base,
-			 const struct x25519_value *scalar,
-			 struct x25519_value *result );
+extern int x25519_key ( const struct x25519_value *base,
+			const struct x25519_value *scalar,
+			struct x25519_value *result );
 
 #endif /* _IPXE_X25519_H */
diff --git a/src/tests/x25519_test.c b/src/tests/x25519_test.c
index 3d781f913..3dfbd3393 100644
--- a/src/tests/x25519_test.c
+++ b/src/tests/x25519_test.c
@@ -136,6 +136,8 @@ struct x25519_key_test {
 	struct x25519_value expected;
 	/** Number of iterations */
 	unsigned int count;
+	/** Key exchange is expected to fail (i.e. produce all-zeroes) */
+	int fail;
 };
 
 /**
@@ -143,14 +145,16 @@ struct x25519_key_test {
  *
  * @v name		Test name
  * @v COUNT		Number of iterations
+ * @v FAIL		Expected failure status
  * @v BASE		Base point
  * @v SCALAR		Scalar multiple
  * @v EXPECTED		Expected result
  * @ret test		X25519 key exchange test
  */
-#define X25519_KEY_TEST( name, COUNT, BASE, SCALAR, EXPECTED )		\
+#define X25519_KEY_TEST( name, COUNT, FAIL, BASE, SCALAR, EXPECTED )	\
 	static struct x25519_key_test name = {				\
 		.count = COUNT,						\
+		.fail = FAIL,						\
 		.base = { .raw = BASE },				\
 		.scalar = { .raw = SCALAR },				\
 		.expected = { .raw = EXPECTED },			\
@@ -259,6 +263,7 @@ static void x25519_key_okx ( struct x25519_key_test *test,
 	struct x25519_value scalar;
 	struct x25519_value actual;
 	unsigned int i;
+	int rc;
 
 	/* Construct input values */
 	memcpy ( &base, &test->base, sizeof ( test->base ) );
@@ -272,7 +277,12 @@ static void x25519_key_okx ( struct x25519_key_test *test,
 
 	/* Calculate key */
 	for ( i = 0 ; i < test->count ; i++ ) {
-		x25519_key ( &base, &scalar, &actual );
+		rc = x25519_key ( &base, &scalar, &actual );
+		if ( test->fail ) {
+			okx ( rc != 0, file, line );
+		} else {
+			okx ( rc == 0, file, line );
+		}
 		memcpy ( &base, &scalar, sizeof ( base ) );
 		memcpy ( &scalar, &actual, sizeof ( scalar ) );
 	}
@@ -461,7 +471,7 @@ X25519_INVERT_TEST ( invert_5,
  * Scalar: 0xa546e36bf0527c9d3b16154b82465edd62144c0ac1fc5a18506a2244ba449ac4
  * Result: 0xc3da55379de9c6908e94ea4df28d084f32eccf03491c71f754b4075577a28552
  */
-X25519_KEY_TEST ( rfc7748_1, 1,
+X25519_KEY_TEST ( rfc7748_1, 1, 0,
 	BASE ( 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, 0x35, 0x94,
 	       0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, 0x72, 0x66, 0x24, 0xec,
 	       0x26, 0xb3, 0x35, 0x3b, 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab,
@@ -479,7 +489,7 @@ X25519_KEY_TEST ( rfc7748_1, 1,
  * Scalar: 0x4b66e9d4d1b4673c5ad22691957d6af5c11b6421e0ea01d42ca4169e7918ba0d
  * Result: 0x95cbde9476e8907d7aade45cb4b873f88b595a68799fa152e6f8f7647aac7957
  */
-X25519_KEY_TEST ( rfc7748_2, 1,
+X25519_KEY_TEST ( rfc7748_2, 1, 0,
 	BASE ( 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3, 0xf4, 0xb7,
 	       0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c, 0x31, 0xdb, 0xe7, 0x10,
 	       0x6f, 0xc0, 0x3c, 0x3e, 0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15,
@@ -497,7 +507,7 @@ X25519_KEY_TEST ( rfc7748_2, 1,
  * Scalar: 0x0900000000000000000000000000000000000000000000000000000000000000
  * Result: 0x422c8e7a6227d7bca1350b3e2bb7279f7897b87bb6854b783c60e80311ae3079
  */
-X25519_KEY_TEST ( rfc7748_3, 1,
+X25519_KEY_TEST ( rfc7748_3, 1, 0,
 	BASE ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -521,7 +531,7 @@ X25519_KEY_TEST ( rfc7748_3, 1,
  * to avoid a pointlessly slow test cycle in the common case of
  * running tests under Valgrind.
  */
-X25519_KEY_TEST ( rfc7748_4_100, 100,
+X25519_KEY_TEST ( rfc7748_4_100, 100, 0,
 	BASE ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -535,6 +545,24 @@ X25519_KEY_TEST ( rfc7748_4_100, 100,
 		   0x1d, 0x26, 0x09, 0xc9, 0x2e, 0x5a, 0x8f, 0x1d, 0xeb, 0xe2,
 		   0x15, 0x0a ) );
 
+/* Base:   2^255 - 19 + 1 (deliberately malicious public key)
+ * Scalar: 0x000102030405060708090a0b0c0d0e0f000102030405060708090a0b0c0d0e0f
+ * Result: Failure (all zeros)
+ */
+X25519_KEY_TEST ( malicious, 1, 1,
+	BASE ( 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	       0xff, 0x7f ),
+	SCALAR ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+		 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03,
+		 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d,
+		 0x0e, 0x0f ),
+	EXPECTED ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		   0x00, 0x00 ) );
+
 /**
  * Perform X25519 self-tests
  *
@@ -562,6 +590,7 @@ static void x25519_test_exec ( void ) {
 	x25519_key_ok ( &rfc7748_2 );
 	x25519_key_ok ( &rfc7748_3 );
 	x25519_key_ok ( &rfc7748_4_100 );
+	x25519_key_ok ( &malicious );
 }
 
 /** X25519 self-test */

From 17135c83fb056f93fe70bdb09dd05ecc08963eed Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 13:26:36 +0000
Subject: [PATCH 058/237] [crypto] Add an abstraction of an elliptic curve

Define an abstraction of an elliptic curve with a fixed generator and
one supported operation (scalar multiplication of a curve point).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/x25519.c       | 31 +++++++++++++++++++++++++++++++
 src/include/ipxe/crypto.h | 23 +++++++++++++++++++++++
 src/include/ipxe/x25519.h |  3 +++
 3 files changed, 57 insertions(+)

diff --git a/src/crypto/x25519.c b/src/crypto/x25519.c
index d3a19bc8e..d58f7168c 100644
--- a/src/crypto/x25519.c
+++ b/src/crypto/x25519.c
@@ -61,6 +61,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <assert.h>
 #include <errno.h>
 #include <ipxe/init.h>
+#include <ipxe/crypto.h>
 #include <ipxe/x25519.h>
 
 /** X25519 reduction constant
@@ -300,6 +301,11 @@ static const uint8_t x25519_121665_raw[] = { 0x01, 0xdb, 0x41 };
 /** Constant 121665 (used in the Montgomery ladder) */
 static union x25519_oct258 x25519_121665;
 
+/** Constant g=9 (the group generator) */
+static struct x25519_value x25519_generator = {
+	.raw = { 9, }
+};
+
 /**
  * Initialise constants
  *
@@ -811,3 +817,28 @@ int x25519_key ( const struct x25519_value *base,
 	/* Fail if result was all zeros (as required by RFC8422) */
 	return ( bigint_is_zero ( &point.value ) ? -EPERM : 0 );
 }
+
+/**
+ * Multiply scalar by curve point
+ *
+ * @v base		Base point (or NULL to use generator)
+ * @v scalar		Scalar multiple
+ * @v result		Result point to fill in
+ * @ret rc		Return status code
+ */
+static int x25519_curve_multiply ( const void *base, const void *scalar,
+				   void *result ) {
+
+	/* Use base point if applicable */
+	if ( ! base )
+		base = &x25519_generator;
+
+	return x25519_key ( base, scalar, result );
+}
+
+/** X25519 elliptic curve */
+struct elliptic_curve x25519_curve = {
+	.name = "x25519",
+	.keysize = sizeof ( struct x25519_value ),
+	.multiply = x25519_curve_multiply,
+};
diff --git a/src/include/ipxe/crypto.h b/src/include/ipxe/crypto.h
index a15d5eba0..9ee1c40cb 100644
--- a/src/include/ipxe/crypto.h
+++ b/src/include/ipxe/crypto.h
@@ -195,6 +195,23 @@ struct pubkey_algorithm {
 			  const void *public_key, size_t public_key_len );
 };
 
+/** An elliptic curve */
+struct elliptic_curve {
+	/** Curve name */
+	const char *name;
+	/** Key size */
+	size_t keysize;
+	/** Multiply scalar by curve point
+	 *
+	 * @v base		Base point (or NULL to use generator)
+	 * @v scalar		Scalar multiple
+	 * @v result		Result point to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * multiply ) ( const void *base, const void *scalar,
+			     void *result );
+};
+
 static inline void digest_init ( struct digest_algorithm *digest,
 				 void *ctx ) {
 	digest->init ( ctx );
@@ -302,6 +319,12 @@ static inline int pubkey_match ( struct pubkey_algorithm *pubkey,
 			       public_key_len );
 }
 
+static inline int elliptic_multiply ( struct elliptic_curve *curve,
+				      const void *base, const void *scalar,
+				      void *result ) {
+	return curve->multiply ( base, scalar, result );
+}
+
 extern void digest_null_init ( void *ctx );
 extern void digest_null_update ( void *ctx, const void *src, size_t len );
 extern void digest_null_final ( void *ctx, void *out );
diff --git a/src/include/ipxe/x25519.h b/src/include/ipxe/x25519.h
index 6524abb7d..fd7caeee9 100644
--- a/src/include/ipxe/x25519.h
+++ b/src/include/ipxe/x25519.h
@@ -11,6 +11,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/bigint.h>
+#include <ipxe/crypto.h>
 
 /** X25519 unsigned big integer size
  *
@@ -88,4 +89,6 @@ extern int x25519_key ( const struct x25519_value *base,
 			const struct x25519_value *scalar,
 			struct x25519_value *result );
 
+extern struct elliptic_curve x25519_curve;
+
 #endif /* _IPXE_X25519_H */

From 6f70e8be834e3531c9e8910c619ce9ed377f2081 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 13:38:15 +0000
Subject: [PATCH 059/237] [tls] Restructure construction of ClientHello message

Define an individual local structure for each extension and a single
structure for the list of extensions.  This makes it viable to add
extensions such as the Supported Elliptic Curves extension, which must
not be present if the list of curves is empty.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/tls.c | 171 +++++++++++++++++++++++++++++---------------------
 1 file changed, 99 insertions(+), 72 deletions(-)

diff --git a/src/net/tls.c b/src/net/tls.c
index 000a8a785..7f31c8f78 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -1121,6 +1121,57 @@ static int tls_client_hello ( struct tls_connection *tls,
 						 size_t len ) ) {
 	struct tls_session *session = tls->session;
 	size_t name_len = strlen ( session->name );
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint16_t len;
+			struct {
+				uint8_t type;
+				uint16_t len;
+				uint8_t name[name_len];
+			} __attribute__ (( packed )) list[1];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *server_name_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint8_t max;
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *max_fragment_length_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint16_t len;
+			struct tls_signature_hash_id
+				code[TLS_NUM_SIG_HASH_ALGORITHMS];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *signature_algorithms_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint8_t len;
+			uint8_t data[ tls->secure_renegotiation ?
+				      sizeof ( tls->verify.client ) :0 ];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *renegotiation_info_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint8_t data[session->ticket_len];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *session_ticket_ext;
+	struct {
+		typeof ( *server_name_ext ) server_name;
+		typeof ( *max_fragment_length_ext ) max_fragment_length;
+		typeof ( *signature_algorithms_ext ) signature_algorithms;
+		typeof ( *renegotiation_info_ext ) renegotiation_info;
+		typeof ( *session_ticket_ext ) session_ticket;
+	} __attribute__ (( packed )) *extensions;
 	struct {
 		uint32_t type_length;
 		uint16_t version;
@@ -1132,42 +1183,7 @@ static int tls_client_hello ( struct tls_connection *tls,
 		uint8_t compression_methods_len;
 		uint8_t compression_methods[1];
 		uint16_t extensions_len;
-		struct {
-			uint16_t server_name_type;
-			uint16_t server_name_len;
-			struct {
-				uint16_t len;
-				struct {
-					uint8_t type;
-					uint16_t len;
-					uint8_t name[name_len];
-				} __attribute__ (( packed )) list[1];
-			} __attribute__ (( packed )) server_name;
-			uint16_t max_fragment_length_type;
-			uint16_t max_fragment_length_len;
-			struct {
-				uint8_t max;
-			} __attribute__ (( packed )) max_fragment_length;
-			uint16_t signature_algorithms_type;
-			uint16_t signature_algorithms_len;
-			struct {
-				uint16_t len;
-				struct tls_signature_hash_id
-					code[TLS_NUM_SIG_HASH_ALGORITHMS];
-			} __attribute__ (( packed )) signature_algorithms;
-			uint16_t renegotiation_info_type;
-			uint16_t renegotiation_info_len;
-			struct {
-				uint8_t len;
-				uint8_t data[ tls->secure_renegotiation ?
-					      sizeof ( tls->verify.client ) :0];
-			} __attribute__ (( packed )) renegotiation_info;
-			uint16_t session_ticket_type;
-			uint16_t session_ticket_len;
-			struct {
-				uint8_t data[session->ticket_len];
-			} __attribute__ (( packed )) session_ticket;
-		} __attribute__ (( packed )) extensions;
+		typeof ( *extensions ) extensions;
 	} __attribute__ (( packed )) hello;
 	struct tls_cipher_suite *suite;
 	struct tls_signature_hash_algorithm *sighash;
@@ -1188,43 +1204,54 @@ static int tls_client_hello ( struct tls_connection *tls,
 		hello.cipher_suites[i++] = suite->code;
 	hello.compression_methods_len = sizeof ( hello.compression_methods );
 	hello.extensions_len = htons ( sizeof ( hello.extensions ) );
-	hello.extensions.server_name_type = htons ( TLS_SERVER_NAME );
-	hello.extensions.server_name_len
-		= htons ( sizeof ( hello.extensions.server_name ) );
-	hello.extensions.server_name.len
-		= htons ( sizeof ( hello.extensions.server_name.list ) );
-	hello.extensions.server_name.list[0].type = TLS_SERVER_NAME_HOST_NAME;
-	hello.extensions.server_name.list[0].len
-		= htons ( sizeof ( hello.extensions.server_name.list[0].name ));
-	memcpy ( hello.extensions.server_name.list[0].name, session->name,
-		 sizeof ( hello.extensions.server_name.list[0].name ) );
-	hello.extensions.max_fragment_length_type
-		= htons ( TLS_MAX_FRAGMENT_LENGTH );
-	hello.extensions.max_fragment_length_len
-		= htons ( sizeof ( hello.extensions.max_fragment_length ) );
-	hello.extensions.max_fragment_length.max
-		= TLS_MAX_FRAGMENT_LENGTH_4096;
-	hello.extensions.signature_algorithms_type
-		= htons ( TLS_SIGNATURE_ALGORITHMS );
-	hello.extensions.signature_algorithms_len
-		= htons ( sizeof ( hello.extensions.signature_algorithms ) );
-	hello.extensions.signature_algorithms.len
-		= htons ( sizeof ( hello.extensions.signature_algorithms.code));
+	extensions = &hello.extensions;
+
+	/* Construct server name extension */
+	server_name_ext = &extensions->server_name;
+	server_name_ext->type = htons ( TLS_SERVER_NAME );
+	server_name_ext->len = htons ( sizeof ( server_name_ext->data ) );
+	server_name_ext->data.len
+		= htons ( sizeof ( server_name_ext->data.list ) );
+	server_name_ext->data.list[0].type = TLS_SERVER_NAME_HOST_NAME;
+	server_name_ext->data.list[0].len
+		= htons ( sizeof ( server_name_ext->data.list[0].name ) );
+	memcpy ( server_name_ext->data.list[0].name, session->name,
+		 sizeof ( server_name_ext->data.list[0].name ) );
+
+	/* Construct maximum fragment length extension */
+	max_fragment_length_ext = &extensions->max_fragment_length;
+	max_fragment_length_ext->type = htons ( TLS_MAX_FRAGMENT_LENGTH );
+	max_fragment_length_ext->len
+		= htons ( sizeof ( max_fragment_length_ext->data ) );
+	max_fragment_length_ext->data.max = TLS_MAX_FRAGMENT_LENGTH_4096;
+
+	/* Construct supported signature algorithms extension */
+	signature_algorithms_ext = &extensions->signature_algorithms;
+	signature_algorithms_ext->type = htons ( TLS_SIGNATURE_ALGORITHMS );
+	signature_algorithms_ext->len
+		= htons ( sizeof ( signature_algorithms_ext->data ) );
+	signature_algorithms_ext->data.len
+		= htons ( sizeof ( signature_algorithms_ext->data.code ) );
 	i = 0 ; for_each_table_entry ( sighash, TLS_SIG_HASH_ALGORITHMS )
-		hello.extensions.signature_algorithms.code[i++] = sighash->code;
-	hello.extensions.renegotiation_info_type
-		= htons ( TLS_RENEGOTIATION_INFO );
-	hello.extensions.renegotiation_info_len
-		= htons ( sizeof ( hello.extensions.renegotiation_info ) );
-	hello.extensions.renegotiation_info.len
-		= sizeof ( hello.extensions.renegotiation_info.data );
-	memcpy ( hello.extensions.renegotiation_info.data, tls->verify.client,
-		 sizeof ( hello.extensions.renegotiation_info.data ) );
-	hello.extensions.session_ticket_type = htons ( TLS_SESSION_TICKET );
-	hello.extensions.session_ticket_len
-		= htons ( sizeof ( hello.extensions.session_ticket ) );
-	memcpy ( hello.extensions.session_ticket.data, session->ticket,
-		 sizeof ( hello.extensions.session_ticket.data ) );
+		signature_algorithms_ext->data.code[i++] = sighash->code;
+
+	/* Construct renegotiation information extension */
+	renegotiation_info_ext = &extensions->renegotiation_info;
+	renegotiation_info_ext->type = htons ( TLS_RENEGOTIATION_INFO );
+	renegotiation_info_ext->len
+		= htons ( sizeof ( renegotiation_info_ext->data ) );
+	renegotiation_info_ext->data.len
+		= sizeof ( renegotiation_info_ext->data.data );
+	memcpy ( renegotiation_info_ext->data.data, tls->verify.client,
+		 sizeof ( renegotiation_info_ext->data.data ) );
+
+	/* Construct session ticket extension */
+	session_ticket_ext = &extensions->session_ticket;
+	session_ticket_ext->type = htons ( TLS_SESSION_TICKET );
+	session_ticket_ext->len
+		= htons ( sizeof ( session_ticket_ext->data ) );
+	memcpy ( session_ticket_ext->data.data, session->ticket,
+		 sizeof ( session_ticket_ext->data.data ) );
 
 	return action ( tls, &hello, sizeof ( hello ) );
 }

From 989dbe0bc4e63a5843e0c23fb1fd25ba8403bc40 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 15:17:49 +0000
Subject: [PATCH 060/237] [tls] Generate key material after sending
 ClientKeyExchange

The construction of the key material for the pending cipher suites
from the TLS master secret must happen regardless of which key
exchange algorithm is in use, and the key material is not required to
send the ClientKeyExchange handshake (which is sent before changing
cipher suites).

Centralise the call to tls_generate_keys() after performing key
exchange via the selected algorithm.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/tls.c | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/src/net/tls.c b/src/net/tls.c
index 7f31c8f78..da16889fd 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -1363,13 +1363,6 @@ static int tls_send_client_key_exchange_pubkey ( struct tls_connection *tls ) {
 	tls_generate_master_secret ( tls, &pre_master_secret,
 				     sizeof ( pre_master_secret ) );
 
-	/* Generate keys */
-	if ( ( rc = tls_generate_keys ( tls ) ) != 0 ) {
-		DBGC ( tls, "TLS %p could not generate keys: %s\n",
-		       tls, strerror ( rc ) );
-		return rc;
-	}
-
 	/* Encrypt pre-master secret using server's public key */
 	memset ( &key_xchg, 0, sizeof ( key_xchg ) );
 	len = pubkey_encrypt ( pubkey, cipherspec->pubkey_ctx,
@@ -1567,13 +1560,6 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 		/* Generate master secret */
 		tls_generate_master_secret ( tls, pre_master_secret, len );
 
-		/* Generate keys */
-		if ( ( rc = tls_generate_keys ( tls ) ) != 0 ) {
-			DBGC ( tls, "TLS %p could not generate keys: %s\n",
-			       tls, strerror ( rc ) );
-			goto err_generate_keys;
-		}
-
 		/* Transmit Client Key Exchange record */
 		if ( ( rc = tls_send_handshake ( tls, key_xchg,
 						 sizeof ( *key_xchg ) ) ) !=0){
@@ -1581,7 +1567,6 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 		}
 
 	err_send_handshake:
-	err_generate_keys:
 	err_dhe_key:
 		free ( dynamic );
 	}
@@ -1608,9 +1593,23 @@ struct tls_key_exchange_algorithm tls_dhe_exchange_algorithm = {
 static int tls_send_client_key_exchange ( struct tls_connection *tls ) {
 	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec_pending;
 	struct tls_cipher_suite *suite = cipherspec->suite;
+	int rc;
 
 	/* Transmit Client Key Exchange record via key exchange algorithm */
-	return suite->exchange->exchange ( tls );
+	if ( ( rc = suite->exchange->exchange ( tls ) ) != 0 ) {
+		DBGC ( tls, "TLS %p could not exchange keys: %s\n",
+		       tls, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Generate keys from master secret */
+	if ( ( rc = tls_generate_keys ( tls ) ) != 0 ) {
+		DBGC ( tls, "TLS %p could not generate keys: %s\n",
+		       tls, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
 }
 
 /**

From 8e2469c861fd25ac4956ca6b3bc3ed4ab8d74308 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 15:23:39 +0000
Subject: [PATCH 061/237] [tls] Split out Diffie-Hellman parameter signature
 verification

DHE and ECDHE use essentially the same mechanism for verifying the
signature over the Diffie-Hellman parameters, though the format of the
parameters is different between the two methods.

Split out the verification of the parameter signature so that it may
be shared between the DHE and ECDHE key exchange algorithms.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/tls.c | 161 ++++++++++++++++++++++++++++----------------------
 1 file changed, 92 insertions(+), 69 deletions(-)

diff --git a/src/net/tls.c b/src/net/tls.c
index da16889fd..66ab1e9ce 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -1394,21 +1394,18 @@ struct tls_key_exchange_algorithm tls_pubkey_exchange_algorithm = {
 };
 
 /**
- * Transmit Client Key Exchange record using DHE key exchange
+ * Verify Diffie-Hellman parameter signature
  *
  * @v tls		TLS connection
+ * @v param_len		Diffie-Hellman parameter length
  * @ret rc		Return status code
  */
-static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
+static int tls_verify_dh_params ( struct tls_connection *tls,
+				  size_t param_len ) {
 	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec_pending;
 	struct pubkey_algorithm *pubkey;
 	struct digest_algorithm *digest;
 	int use_sig_hash = tls_version ( tls, TLS_VERSION_TLS_1_2 );
-	uint8_t private[ sizeof ( tls->client_random.random ) ];
-	const struct {
-		uint16_t len;
-		uint8_t data[0];
-	} __attribute__ (( packed )) *dh_val[3];
 	const struct {
 		struct tls_signature_hash_id sig_hash[use_sig_hash];
 		uint16_t signature_len;
@@ -1416,7 +1413,91 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 	} __attribute__ (( packed )) *sig;
 	const void *data;
 	size_t remaining;
+	int rc;
+
+	/* Signature follows parameters */
+	assert ( param_len <= tls->server_key_len );
+	data = ( tls->server_key + param_len );
+	remaining = ( tls->server_key_len - param_len );
+
+	/* Parse signature from ServerKeyExchange */
+	sig = data;
+	if ( ( sizeof ( *sig ) > remaining ) ||
+	     ( ntohs ( sig->signature_len ) > ( remaining -
+						sizeof ( *sig ) ) ) ) {
+		DBGC ( tls, "TLS %p received underlength ServerKeyExchange\n",
+		       tls );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -EINVAL_KEY_EXCHANGE;
+	}
+
+	/* Identify signature and hash algorithm */
+	if ( use_sig_hash ) {
+		pubkey = tls_signature_hash_pubkey ( sig->sig_hash[0] );
+		digest = tls_signature_hash_digest ( sig->sig_hash[0] );
+		if ( ( ! pubkey ) || ( ! digest ) ) {
+			DBGC ( tls, "TLS %p ServerKeyExchange unsupported "
+			       "signature and hash algorithm\n", tls );
+			return -ENOTSUP_SIG_HASH;
+		}
+		if ( pubkey != cipherspec->suite->pubkey ) {
+			DBGC ( tls, "TLS %p ServerKeyExchange incorrect "
+			       "signature algorithm %s (expected %s)\n", tls,
+			       pubkey->name, cipherspec->suite->pubkey->name );
+			return -EPERM_KEY_EXCHANGE;
+		}
+	} else {
+		pubkey = cipherspec->suite->pubkey;
+		digest = &md5_sha1_algorithm;
+	}
+
+	/* Verify signature */
+	{
+		const void *signature = sig->signature;
+		size_t signature_len = ntohs ( sig->signature_len );
+		uint8_t ctx[digest->ctxsize];
+		uint8_t hash[digest->digestsize];
+
+		/* Calculate digest */
+		digest_init ( digest, ctx );
+		digest_update ( digest, ctx, &tls->client_random,
+				sizeof ( tls->client_random ) );
+		digest_update ( digest, ctx, tls->server_random,
+				sizeof ( tls->server_random ) );
+		digest_update ( digest, ctx, tls->server_key, param_len );
+		digest_final ( digest, ctx, hash );
+
+		/* Verify signature */
+		if ( ( rc = pubkey_verify ( pubkey, cipherspec->pubkey_ctx,
+					    digest, hash, signature,
+					    signature_len ) ) != 0 ) {
+			DBGC ( tls, "TLS %p ServerKeyExchange failed "
+			       "verification\n", tls );
+			DBGC_HDA ( tls, 0, tls->server_key,
+				   tls->server_key_len );
+			return -EPERM_KEY_EXCHANGE;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Transmit Client Key Exchange record using DHE key exchange
+ *
+ * @v tls		TLS connection
+ * @ret rc		Return status code
+ */
+static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
+	uint8_t private[ sizeof ( tls->client_random.random ) ];
+	const struct {
+		uint16_t len;
+		uint8_t data[0];
+	} __attribute__ (( packed )) *dh_val[3];
+	const void *data;
+	size_t remaining;
 	size_t frag_len;
+	size_t param_len;
 	unsigned int i;
 	int rc;
 
@@ -1439,68 +1520,11 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 		data += frag_len;
 		remaining -= frag_len;
 	}
-	sig = data;
-	if ( ( sizeof ( *sig ) > remaining ) ||
-	     ( ntohs ( sig->signature_len ) > ( remaining -
-						sizeof ( *sig ) ) ) ) {
-		DBGC ( tls, "TLS %p received underlength ServerKeyExchange\n",
-		       tls );
-		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
-		rc = -EINVAL_KEY_EXCHANGE;
-		goto err_header;
-	}
+	param_len = ( tls->server_key_len - remaining );
 
-	/* Identify signature and hash algorithm */
-	if ( use_sig_hash ) {
-		pubkey = tls_signature_hash_pubkey ( sig->sig_hash[0] );
-		digest = tls_signature_hash_digest ( sig->sig_hash[0] );
-		if ( ( ! pubkey ) || ( ! digest ) ) {
-			DBGC ( tls, "TLS %p ServerKeyExchange unsupported "
-			       "signature and hash algorithm\n", tls );
-			rc = -ENOTSUP_SIG_HASH;
-			goto err_sig_hash;
-		}
-		if ( pubkey != cipherspec->suite->pubkey ) {
-			DBGC ( tls, "TLS %p ServerKeyExchange incorrect "
-			       "signature algorithm %s (expected %s)\n", tls,
-			       pubkey->name, cipherspec->suite->pubkey->name );
-			rc = -EPERM_KEY_EXCHANGE;
-			goto err_sig_hash;
-		}
-	} else {
-		pubkey = cipherspec->suite->pubkey;
-		digest = &md5_sha1_algorithm;
-	}
-
-	/* Verify signature */
-	{
-		const void *signature = sig->signature;
-		size_t signature_len = ntohs ( sig->signature_len );
-		uint8_t ctx[digest->ctxsize];
-		uint8_t hash[digest->digestsize];
-
-		/* Calculate digest */
-		digest_init ( digest, ctx );
-		digest_update ( digest, ctx, &tls->client_random,
-				sizeof ( tls->client_random ) );
-		digest_update ( digest, ctx, tls->server_random,
-				sizeof ( tls->server_random ) );
-		digest_update ( digest, ctx, tls->server_key,
-				( tls->server_key_len - remaining ) );
-		digest_final ( digest, ctx, hash );
-
-		/* Verify signature */
-		if ( ( rc = pubkey_verify ( pubkey, cipherspec->pubkey_ctx,
-					    digest, hash, signature,
-					    signature_len ) ) != 0 ) {
-			DBGC ( tls, "TLS %p ServerKeyExchange failed "
-			       "verification\n", tls );
-			DBGC_HDA ( tls, 0, tls->server_key,
-				   tls->server_key_len );
-			rc = -EPERM_KEY_EXCHANGE;
-			goto err_verify;
-		}
-	}
+	/* Verify parameter signature */
+	if ( ( rc = tls_verify_dh_params ( tls, param_len ) ) != 0 )
+		goto err_verify;
 
 	/* Generate Diffie-Hellman private key */
 	if ( ( rc = tls_generate_random ( tls, private,
@@ -1573,7 +1597,6 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
  err_alloc:
  err_random:
  err_verify:
- err_sig_hash:
  err_header:
 	return rc;
 }

From b234226dbc4f348c7e4a5c61bdf7b0f8f0aef16c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 15:09:49 +0000
Subject: [PATCH 062/237] [tls] Add support for Ephemeral Elliptic Curve
 Diffie-Hellman key exchange

Add support for the Ephemeral Elliptic Curve Diffie-Hellman (ECDHE)
key exchange algorithm.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/tls.h |  24 ++++++
 src/net/tls.c          | 169 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 193 insertions(+)

diff --git a/src/include/ipxe/tls.h b/src/include/ipxe/tls.h
index 30bb1c483..5c218f84f 100644
--- a/src/include/ipxe/tls.h
+++ b/src/include/ipxe/tls.h
@@ -119,6 +119,10 @@ struct tls_header {
 #define TLS_MAX_FRAGMENT_LENGTH_2048 3
 #define TLS_MAX_FRAGMENT_LENGTH_4096 4
 
+/* TLS named curve extension */
+#define TLS_NAMED_CURVE 10
+#define TLS_NAMED_CURVE_X25519 29
+
 /* TLS signature algorithms extension */
 #define TLS_SIGNATURE_ALGORITHMS 13
 
@@ -205,6 +209,25 @@ struct tls_cipher_suite {
 #define __tls_cipher_suite( pref )					\
 	__table_entry ( TLS_CIPHER_SUITES, pref )
 
+/** TLS named curved type */
+#define TLS_NAMED_CURVE_TYPE 3
+
+/** A TLS named curve */
+struct tls_named_curve {
+	/** Elliptic curve */
+	struct elliptic_curve *curve;
+	/** Numeric code (in network-endian order) */
+	uint16_t code;
+};
+
+/** TLS named curve table */
+#define TLS_NAMED_CURVES						\
+	__table ( struct tls_named_curve, "tls_named_curves" )
+
+/** Declare a TLS named curve */
+#define __tls_named_curve( pref )					\
+	__table_entry ( TLS_NAMED_CURVES, pref )
+
 /** A TLS cipher specification */
 struct tls_cipherspec {
 	/** Cipher suite */
@@ -425,6 +448,7 @@ struct tls_connection {
 
 extern struct tls_key_exchange_algorithm tls_pubkey_exchange_algorithm;
 extern struct tls_key_exchange_algorithm tls_dhe_exchange_algorithm;
+extern struct tls_key_exchange_algorithm tls_ecdhe_exchange_algorithm;
 
 extern int add_tls ( struct interface *xfer, const char *name,
 		     struct x509_root *root, struct private_key *key );
diff --git a/src/net/tls.c b/src/net/tls.c
index 66ab1e9ce..58fad65e1 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -158,6 +158,10 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define EINFO_ENOTSUP_VERSION						\
 	__einfo_uniqify ( EINFO_ENOTSUP, 0x04,				\
 			  "Unsupported protocol version" )
+#define ENOTSUP_CURVE __einfo_error ( EINFO_ENOTSUP_CURVE )
+#define EINFO_ENOTSUP_CURVE						\
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x05,				\
+			  "Unsupported elliptic curve" )
 #define EPERM_ALERT __einfo_error ( EINFO_EPERM_ALERT )
 #define EINFO_EPERM_ALERT						\
 	__einfo_uniqify ( EINFO_EPERM, 0x01,				\
@@ -1042,6 +1046,35 @@ tls_signature_hash_digest ( struct tls_signature_hash_id code ) {
 	return NULL;
 }
 
+/******************************************************************************
+ *
+ * Ephemeral Elliptic Curve Diffie-Hellman key exchange
+ *
+ ******************************************************************************
+ */
+
+/** Number of supported named curves */
+#define TLS_NUM_NAMED_CURVES table_num_entries ( TLS_NAMED_CURVES )
+
+/**
+ * Identify named curve
+ *
+ * @v named_curve	Named curve specification
+ * @ret curve		Named curve, or NULL
+ */
+static struct tls_named_curve *
+tls_find_named_curve ( unsigned int named_curve ) {
+	struct tls_named_curve *curve;
+
+	/* Identify named curve */
+	for_each_table_entry ( curve, TLS_NAMED_CURVES ) {
+		if ( curve->code == named_curve )
+			return curve;
+	}
+
+	return NULL;
+}
+
 /******************************************************************************
  *
  * Record handling
@@ -1165,12 +1198,22 @@ static int tls_client_hello ( struct tls_connection *tls,
 			uint8_t data[session->ticket_len];
 		} __attribute__ (( packed )) data;
 	} __attribute__ (( packed )) *session_ticket_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint16_t len;
+			uint16_t code[TLS_NUM_NAMED_CURVES];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *named_curve_ext;
 	struct {
 		typeof ( *server_name_ext ) server_name;
 		typeof ( *max_fragment_length_ext ) max_fragment_length;
 		typeof ( *signature_algorithms_ext ) signature_algorithms;
 		typeof ( *renegotiation_info_ext ) renegotiation_info;
 		typeof ( *session_ticket_ext ) session_ticket;
+		typeof ( *named_curve_ext )
+			named_curve[TLS_NUM_NAMED_CURVES ? 1 : 0];
 	} __attribute__ (( packed )) *extensions;
 	struct {
 		uint32_t type_length;
@@ -1187,6 +1230,7 @@ static int tls_client_hello ( struct tls_connection *tls,
 	} __attribute__ (( packed )) hello;
 	struct tls_cipher_suite *suite;
 	struct tls_signature_hash_algorithm *sighash;
+	struct tls_named_curve *curve;
 	unsigned int i;
 
 	/* Construct record */
@@ -1253,6 +1297,18 @@ static int tls_client_hello ( struct tls_connection *tls,
 	memcpy ( session_ticket_ext->data.data, session->ticket,
 		 sizeof ( session_ticket_ext->data.data ) );
 
+	/* Construct named curves extension, if applicable */
+	if ( sizeof ( extensions->named_curve ) ) {
+		named_curve_ext = &extensions->named_curve[0];
+		named_curve_ext->type = htons ( TLS_NAMED_CURVE );
+		named_curve_ext->len
+			= htons ( sizeof ( named_curve_ext->data ) );
+		named_curve_ext->data.len
+			= htons ( sizeof ( named_curve_ext->data.code ) );
+		i = 0 ; for_each_table_entry ( curve, TLS_NAMED_CURVES )
+			named_curve_ext->data.code[i++] = curve->code;
+	}
+
 	return action ( tls, &hello, sizeof ( hello ) );
 }
 
@@ -1607,6 +1663,119 @@ struct tls_key_exchange_algorithm tls_dhe_exchange_algorithm = {
 	.exchange = tls_send_client_key_exchange_dhe,
 };
 
+/**
+ * Transmit Client Key Exchange record using ECDHE key exchange
+ *
+ * @v tls		TLS connection
+ * @ret rc		Return status code
+ */
+static int tls_send_client_key_exchange_ecdhe ( struct tls_connection *tls ) {
+	struct tls_named_curve *curve;
+	const struct {
+		uint8_t curve_type;
+		uint16_t named_curve;
+		uint8_t public_len;
+		uint8_t public[0];
+	} __attribute__ (( packed )) *ecdh;
+	size_t param_len;
+	int rc;
+
+	/* Parse ServerKeyExchange record */
+	ecdh = tls->server_key;
+	if ( ( sizeof ( *ecdh ) > tls->server_key_len ) ||
+	     ( ecdh->public_len > ( tls->server_key_len - sizeof ( *ecdh ) ))){
+		DBGC ( tls, "TLS %p received underlength ServerKeyExchange\n",
+		       tls );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -EINVAL_KEY_EXCHANGE;
+	}
+	param_len = ( sizeof ( *ecdh ) + ecdh->public_len );
+
+	/* Verify parameter signature */
+	if ( ( rc = tls_verify_dh_params ( tls, param_len ) ) != 0 )
+		return rc;
+
+	/* Identify named curve */
+	if ( ecdh->curve_type != TLS_NAMED_CURVE_TYPE ) {
+		DBGC ( tls, "TLS %p unsupported curve type %d\n",
+		       tls, ecdh->curve_type );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -ENOTSUP_CURVE;
+	}
+	curve = tls_find_named_curve ( ecdh->named_curve );
+	if ( ! curve ) {
+		DBGC ( tls, "TLS %p unsupported named curve %d\n",
+		       tls, ntohs ( ecdh->named_curve ) );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -ENOTSUP_CURVE;
+	}
+
+	/* Check key length */
+	if ( ecdh->public_len != curve->curve->keysize ) {
+		DBGC ( tls, "TLS %p invalid %s key\n",
+		       tls, curve->curve->name );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -EINVAL_KEY_EXCHANGE;
+	}
+
+	/* Construct pre-master secret and ClientKeyExchange record */
+	{
+		size_t len = curve->curve->keysize;
+		uint8_t private[len];
+		uint8_t pre_master_secret[len];
+		struct {
+			uint32_t type_length;
+			uint8_t public_len;
+			uint8_t public[len];
+		} __attribute__ (( packed )) key_xchg;
+
+		/* Generate ephemeral private key */
+		if ( ( rc = tls_generate_random ( tls, private,
+						  sizeof ( private ) ) ) != 0){
+			return rc;
+		}
+
+		/* Calculate pre-master secret */
+		if ( ( rc = elliptic_multiply ( curve->curve,
+						ecdh->public, private,
+						pre_master_secret ) ) != 0 ) {
+			DBGC ( tls, "TLS %p could not exchange ECDHE key: %s\n",
+			       tls, strerror ( rc ) );
+			return rc;
+		}
+
+		/* Generate master secret */
+		tls_generate_master_secret ( tls, pre_master_secret, len );
+
+		/* Generate Client Key Exchange record */
+		key_xchg.type_length =
+			( cpu_to_le32 ( TLS_CLIENT_KEY_EXCHANGE ) |
+			  htonl ( sizeof ( key_xchg ) -
+				  sizeof ( key_xchg.type_length ) ) );
+		key_xchg.public_len = len;
+		if ( ( rc = elliptic_multiply ( curve->curve, NULL, private,
+						key_xchg.public ) ) != 0 ) {
+			DBGC ( tls, "TLS %p could not generate ECDHE key: %s\n",
+			       tls, strerror ( rc ) );
+			return rc;
+		}
+
+		/* Transmit Client Key Exchange record */
+		if ( ( rc = tls_send_handshake ( tls, &key_xchg,
+						 sizeof ( key_xchg ) ) ) !=0){
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+/** Ephemeral Elliptic Curve Diffie-Hellman key exchange algorithm */
+struct tls_key_exchange_algorithm tls_ecdhe_exchange_algorithm = {
+	.name = "ecdhe",
+	.exchange = tls_send_client_key_exchange_ecdhe,
+};
+
 /**
  * Transmit Client Key Exchange record
  *

From a881a26061a75bbca68a709eb40b396c4ef5656b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 18:00:31 +0000
Subject: [PATCH 063/237] [crypto] Add X25519 OID-identified algorithm and TLS
 named curve

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config_crypto.c       |  5 ++++
 src/config/crypto.h              |  3 +++
 src/crypto/mishmash/oid_x25519.c | 45 ++++++++++++++++++++++++++++++++
 src/include/ipxe/asn1.h          |  7 +++++
 4 files changed, 60 insertions(+)
 create mode 100644 src/crypto/mishmash/oid_x25519.c

diff --git a/src/config/config_crypto.c b/src/config/config_crypto.c
index fa1996a55..72b709202 100644
--- a/src/config/config_crypto.c
+++ b/src/config/config_crypto.c
@@ -83,6 +83,11 @@ REQUIRE_OBJECT ( oid_sha512_224 );
 REQUIRE_OBJECT ( oid_sha512_256 );
 #endif
 
+/* X25519 */
+#if defined ( CRYPTO_CURVE_X25519 )
+REQUIRE_OBJECT ( oid_x25519 );
+#endif
+
 /* RSA and MD5 */
 #if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_MD5 )
 REQUIRE_OBJECT ( rsa_md5 );
diff --git a/src/config/crypto.h b/src/config/crypto.h
index 76bf14d41..637a06c0c 100644
--- a/src/config/crypto.h
+++ b/src/config/crypto.h
@@ -48,6 +48,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** SHA-512/256 digest algorithm */
 //#define CRYPTO_DIGEST_SHA512_256
 
+/** X25519 elliptic curve */
+#define CRYPTO_CURVE_X25519
+
 /** Margin of error (in seconds) allowed in signed timestamps
  *
  * We default to allowing a reasonable margin of error: 12 hours to
diff --git a/src/crypto/mishmash/oid_x25519.c b/src/crypto/mishmash/oid_x25519.c
new file mode 100644
index 000000000..2f8aa065b
--- /dev/null
+++ b/src/crypto/mishmash/oid_x25519.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/x25519.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "x25519" object identifier */
+static uint8_t oid_x25519[] = { ASN1_OID_X25519 };
+
+/** "x25519" OID-identified algorithm */
+struct asn1_algorithm x25519_algorithm __asn1_algorithm = {
+	.name = "x25519",
+	.curve = &x25519_curve,
+	.oid = ASN1_CURSOR ( oid_x25519 ),
+};
+
+/** X25519 named curve */
+struct tls_named_curve tls_x25519_named_curve __tls_named_curve ( 01 ) = {
+	.curve = &x25519_curve,
+	.code = htons ( TLS_NAMED_CURVE_X25519 ),
+};
diff --git a/src/include/ipxe/asn1.h b/src/include/ipxe/asn1.h
index 452fcef0b..ac7ea5604 100644
--- a/src/include/ipxe/asn1.h
+++ b/src/include/ipxe/asn1.h
@@ -187,6 +187,11 @@ struct asn1_builder_header {
 	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 2 ),		\
 	ASN1_OID_SINGLE ( 26 )
 
+/** ASN.1 OID for id-x25519 (1.3.101.110) */
+#define ASN1_OID_X25519						\
+	ASN1_OID_INITIAL ( 1, 3 ), ASN1_OID_SINGLE ( 101 ),	\
+	ASN1_OID_SINGLE ( 110 )
+
 /** ASN.1 OID for id-sha256 (2.16.840.1.101.3.4.2.1) */
 #define ASN1_OID_SHA256						\
 	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
@@ -312,6 +317,8 @@ struct asn1_algorithm {
 	struct pubkey_algorithm *pubkey;
 	/** Digest algorithm (if applicable) */
 	struct digest_algorithm *digest;
+	/** Elliptic curve (if applicable) */
+	struct elliptic_curve *curve;
 };
 
 /** ASN.1 OID-identified algorithms */

From 8f6a9399b3dc5af227cbd6185eff077b6e9d0e37 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 16:48:46 +0000
Subject: [PATCH 064/237] [tls] Make key exchange algorithms selectable via
 build configuration

Allow the choice of key exchange algorithms to be controlled via build
configuration options in config/crypto.h, as is already done for the
choices of public-key algorithms, cipher algorithms, and digest
algorithms.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config_crypto.c                   | 40 ++++++++++---
 src/config/crypto.h                          |  6 ++
 src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c   | 61 ++++++++++++++++++++
 src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c | 60 +++++++++++++++++++
 src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c | 45 +++++++++++++++
 src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c | 45 +++++++++++++++
 src/crypto/mishmash/rsa_aes_cbc_sha1.c       | 30 ----------
 src/crypto/mishmash/rsa_aes_cbc_sha256.c     | 30 ----------
 src/crypto/mishmash/rsa_aes_gcm_sha256.c     | 15 -----
 src/crypto/mishmash/rsa_aes_gcm_sha384.c     | 15 -----
 10 files changed, 249 insertions(+), 98 deletions(-)
 create mode 100644 src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c
 create mode 100644 src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c
 create mode 100644 src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c
 create mode 100644 src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c

diff --git a/src/config/config_crypto.c b/src/config/config_crypto.c
index 72b709202..efcd5af69 100644
--- a/src/config/config_crypto.c
+++ b/src/config/config_crypto.c
@@ -119,25 +119,49 @@ REQUIRE_OBJECT ( rsa_sha512 );
 #endif
 
 /* RSA, AES-CBC, and SHA-1 */
-#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_CBC ) && \
-    defined ( CRYPTO_DIGEST_SHA1 )
+#if defined ( CRYPTO_EXCHANGE_PUBKEY ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA1 )
 REQUIRE_OBJECT ( rsa_aes_cbc_sha1 );
 #endif
 
 /* RSA, AES-CBC, and SHA-256 */
-#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_CBC ) && \
-    defined ( CRYPTO_DIGEST_SHA256 )
+#if defined ( CRYPTO_EXCHANGE_PUBKEY ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA256 )
 REQUIRE_OBJECT ( rsa_aes_cbc_sha256 );
 #endif
 
 /* RSA, AES-GCM, and SHA-256 */
-#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_GCM ) && \
-    defined ( CRYPTO_DIGEST_SHA256 )
+#if defined ( CRYPTO_EXCHANGE_PUBKEY ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA256 )
 REQUIRE_OBJECT ( rsa_aes_gcm_sha256 );
 #endif
 
 /* RSA, AES-GCM, and SHA-384 */
-#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_GCM ) && \
-    defined ( CRYPTO_DIGEST_SHA384 )
+#if defined ( CRYPTO_EXCHANGE_PUBKEY ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA384 )
 REQUIRE_OBJECT ( rsa_aes_gcm_sha384 );
 #endif
+
+/* DHE, RSA, AES-CBC, and SHA-1 */
+#if defined ( CRYPTO_EXCHANGE_DHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA1 )
+REQUIRE_OBJECT ( dhe_rsa_aes_cbc_sha1 );
+#endif
+
+/* DHE, RSA, AES-CBC, and SHA-256 */
+#if defined ( CRYPTO_EXCHANGE_DHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( dhe_rsa_aes_cbc_sha256 );
+#endif
+
+/* DHE, RSA, AES-GCM, and SHA-256 */
+#if defined ( CRYPTO_EXCHANGE_DHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( dhe_rsa_aes_gcm_sha256 );
+#endif
+
+/* DHE, RSA, AES-GCM, and SHA-384 */
+#if defined ( CRYPTO_EXCHANGE_DHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA384 )
+REQUIRE_OBJECT ( dhe_rsa_aes_gcm_sha384 );
+#endif
diff --git a/src/config/crypto.h b/src/config/crypto.h
index 637a06c0c..ccf22df6b 100644
--- a/src/config/crypto.h
+++ b/src/config/crypto.h
@@ -12,6 +12,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** Minimum TLS version */
 #define TLS_VERSION_MIN TLS_VERSION_TLS_1_1
 
+/** Public-key exchange algorithm */
+#define CRYPTO_EXCHANGE_PUBKEY
+
+/** DHE key exchange algorithm */
+#define CRYPTO_EXCHANGE_DHE
+
 /** RSA public-key algorithm */
 #define CRYPTO_PUBKEY_RSA
 
diff --git a/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c
new file mode 100644
index 000000000..95a87f9ac
--- /dev/null
+++ b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha1.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_DHE_RSA_WITH_AES_128_CBC_SHA cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 05 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA1_DIGEST_SIZE,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+	.handshake = &sha256_algorithm,
+};
+
+/** TLS_DHE_RSA_WITH_AES_256_CBC_SHA cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 06 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA1_DIGEST_SIZE,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c
new file mode 100644
index 000000000..b60dff422
--- /dev/null
+++ b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 03 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA256_DIGEST_SIZE,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
+
+/** TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 04 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA256_DIGEST_SIZE,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c
new file mode 100644
index 000000000..9d0395091
--- /dev/null
+++ b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2022 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 01 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 4,
+	.record_iv_len = 8,
+	.mac_len = 0,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_gcm_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c
new file mode 100644
index 000000000..6acd4b1d7
--- /dev/null
+++ b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2022 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha512.h>
+#include <ipxe/tls.h>
+
+/** TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 02 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 4,
+	.record_iv_len = 8,
+	.mac_len = 0,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_gcm_algorithm,
+	.digest = &sha384_algorithm,
+	.handshake = &sha384_algorithm,
+};
diff --git a/src/crypto/mishmash/rsa_aes_cbc_sha1.c b/src/crypto/mishmash/rsa_aes_cbc_sha1.c
index 9f8193de0..fccb04a9b 100644
--- a/src/crypto/mishmash/rsa_aes_cbc_sha1.c
+++ b/src/crypto/mishmash/rsa_aes_cbc_sha1.c
@@ -30,36 +30,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sha256.h>
 #include <ipxe/tls.h>
 
-/** TLS_DHE_RSA_WITH_AES_128_CBC_SHA cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 05 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA ),
-	.key_len = ( 128 / 8 ),
-	.fixed_iv_len = 0,
-	.record_iv_len = AES_BLOCKSIZE,
-	.mac_len = SHA1_DIGEST_SIZE,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_cbc_algorithm,
-	.digest = &sha1_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
-/** TLS_DHE_RSA_WITH_AES_256_CBC_SHA cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 06 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA ),
-	.key_len = ( 256 / 8 ),
-	.fixed_iv_len = 0,
-	.record_iv_len = AES_BLOCKSIZE,
-	.mac_len = SHA1_DIGEST_SIZE,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_cbc_algorithm,
-	.digest = &sha1_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
 /** TLS_RSA_WITH_AES_128_CBC_SHA cipher suite */
 struct tls_cipher_suite
 tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 15 ) = {
diff --git a/src/crypto/mishmash/rsa_aes_cbc_sha256.c b/src/crypto/mishmash/rsa_aes_cbc_sha256.c
index d0dc84964..ac379b4b1 100644
--- a/src/crypto/mishmash/rsa_aes_cbc_sha256.c
+++ b/src/crypto/mishmash/rsa_aes_cbc_sha256.c
@@ -29,36 +29,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sha256.h>
 #include <ipxe/tls.h>
 
-/** TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 03 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 ),
-	.key_len = ( 128 / 8 ),
-	.fixed_iv_len = 0,
-	.record_iv_len = AES_BLOCKSIZE,
-	.mac_len = SHA256_DIGEST_SIZE,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_cbc_algorithm,
-	.digest = &sha256_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
-/** TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 04 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 ),
-	.key_len = ( 256 / 8 ),
-	.fixed_iv_len = 0,
-	.record_iv_len = AES_BLOCKSIZE,
-	.mac_len = SHA256_DIGEST_SIZE,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_cbc_algorithm,
-	.digest = &sha256_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
 /** TLS_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
 struct tls_cipher_suite
 tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 13 ) = {
diff --git a/src/crypto/mishmash/rsa_aes_gcm_sha256.c b/src/crypto/mishmash/rsa_aes_gcm_sha256.c
index cf9c4c279..e08deff84 100644
--- a/src/crypto/mishmash/rsa_aes_gcm_sha256.c
+++ b/src/crypto/mishmash/rsa_aes_gcm_sha256.c
@@ -29,21 +29,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sha256.h>
 #include <ipxe/tls.h>
 
-/** TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 01 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 ),
-	.key_len = ( 128 / 8 ),
-	.fixed_iv_len = 4,
-	.record_iv_len = 8,
-	.mac_len = 0,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_gcm_algorithm,
-	.digest = &sha256_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
 /** TLS_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
 struct tls_cipher_suite
 tls_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 11 ) = {
diff --git a/src/crypto/mishmash/rsa_aes_gcm_sha384.c b/src/crypto/mishmash/rsa_aes_gcm_sha384.c
index 10a977f7f..0b307b420 100644
--- a/src/crypto/mishmash/rsa_aes_gcm_sha384.c
+++ b/src/crypto/mishmash/rsa_aes_gcm_sha384.c
@@ -29,21 +29,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sha512.h>
 #include <ipxe/tls.h>
 
-/** TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 02 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 ),
-	.key_len = ( 256 / 8 ),
-	.fixed_iv_len = 4,
-	.record_iv_len = 8,
-	.mac_len = 0,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_gcm_algorithm,
-	.digest = &sha384_algorithm,
-	.handshake = &sha384_algorithm,
-};
-
 /** TLS_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
 struct tls_cipher_suite
 tls_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 12 ) = {

From 963ec1c4f379a49cf37d01472a770fff8e47470c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jan 2024 17:42:16 +0000
Subject: [PATCH 065/237] [tls] Add ECDHE cipher suites

Add ECDHE variants of the existing cipher suites, and lower the
priority of the non-ECDHE variants.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config_crypto.c                    | 30 +++++++++
 src/config/crypto.h                           |  3 +
 src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c    |  4 +-
 src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c  |  4 +-
 src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c  |  2 +-
 src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c  |  2 +-
 src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha1.c  | 61 +++++++++++++++++++
 .../mishmash/ecdhe_rsa_aes_cbc_sha256.c       | 45 ++++++++++++++
 .../mishmash/ecdhe_rsa_aes_cbc_sha384.c       | 45 ++++++++++++++
 .../mishmash/ecdhe_rsa_aes_gcm_sha256.c       | 45 ++++++++++++++
 .../mishmash/ecdhe_rsa_aes_gcm_sha384.c       | 45 ++++++++++++++
 src/crypto/mishmash/rsa_aes_cbc_sha1.c        |  4 +-
 src/crypto/mishmash/rsa_aes_cbc_sha256.c      |  4 +-
 src/crypto/mishmash/rsa_aes_gcm_sha256.c      |  2 +-
 src/crypto/mishmash/rsa_aes_gcm_sha384.c      |  2 +-
 src/include/ipxe/tls.h                        |  6 ++
 16 files changed, 292 insertions(+), 12 deletions(-)
 create mode 100644 src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha1.c
 create mode 100644 src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha256.c
 create mode 100644 src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha384.c
 create mode 100644 src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha256.c
 create mode 100644 src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha384.c

diff --git a/src/config/config_crypto.c b/src/config/config_crypto.c
index efcd5af69..5211224ab 100644
--- a/src/config/config_crypto.c
+++ b/src/config/config_crypto.c
@@ -165,3 +165,33 @@ REQUIRE_OBJECT ( dhe_rsa_aes_gcm_sha256 );
     defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA384 )
 REQUIRE_OBJECT ( dhe_rsa_aes_gcm_sha384 );
 #endif
+
+/* ECDHE, RSA, AES-CBC, and SHA-1 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA1 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_cbc_sha1 );
+#endif
+
+/* ECDHE, RSA, AES-CBC, and SHA-256 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_cbc_sha256 );
+#endif
+
+/* ECDHE, RSA, AES-CBC, and SHA-384 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA384 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_cbc_sha384 );
+#endif
+
+/* ECDHE, RSA, AES-GCM, and SHA-256 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_gcm_sha256 );
+#endif
+
+/* ECDHE, RSA, AES-GCM, and SHA-384 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA384 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_gcm_sha384 );
+#endif
diff --git a/src/config/crypto.h b/src/config/crypto.h
index ccf22df6b..589c4f0da 100644
--- a/src/config/crypto.h
+++ b/src/config/crypto.h
@@ -18,6 +18,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** DHE key exchange algorithm */
 #define CRYPTO_EXCHANGE_DHE
 
+/** ECDHE key exchange algorithm */
+#define CRYPTO_EXCHANGE_ECDHE
+
 /** RSA public-key algorithm */
 #define CRYPTO_PUBKEY_RSA
 
diff --git a/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c
index 95a87f9ac..05e409f7a 100644
--- a/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c
+++ b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c
@@ -32,7 +32,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** TLS_DHE_RSA_WITH_AES_128_CBC_SHA cipher suite */
 struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 05 ) = {
+tls_dhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 15 ) = {
 	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 0,
@@ -47,7 +47,7 @@ tls_dhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 05 ) = {
 
 /** TLS_DHE_RSA_WITH_AES_256_CBC_SHA cipher suite */
 struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 06 ) = {
+tls_dhe_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 16 ) = {
 	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 0,
diff --git a/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c
index b60dff422..6ce428642 100644
--- a/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c
+++ b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c
@@ -31,7 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 03 ) = {
+tls_dhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 13 ) = {
 	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 0,
@@ -46,7 +46,7 @@ tls_dhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 03 ) = {
 
 /** TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 04 ) = {
+tls_dhe_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 14 ) = {
 	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 0,
diff --git a/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c
index 9d0395091..dc5cad9f8 100644
--- a/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c
+++ b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c
@@ -31,7 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 01 ) = {
+tls_dhe_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 11 ) = {
 	.code = htons ( TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 4,
diff --git a/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c
index 6acd4b1d7..0448255f3 100644
--- a/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c
+++ b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c
@@ -31,7 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
 struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 02 ) = {
+tls_dhe_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 12 ) = {
 	.code = htons ( TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 4,
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha1.c b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha1.c
new file mode 100644
index 000000000..c23f65cc0
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha1.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha1.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 05 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA1_DIGEST_SIZE,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+	.handshake = &sha256_algorithm,
+};
+
+/** TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 06 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA1_DIGEST_SIZE,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha256.c b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha256.c
new file mode 100644
index 000000000..431e2e304
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha256.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 03 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA256_DIGEST_SIZE,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha384.c b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha384.c
new file mode 100644
index 000000000..c52976809
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha384.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha512.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_256_cbc_sha384 __tls_cipher_suite ( 04 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA384_DIGEST_SIZE,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha384_algorithm,
+	.handshake = &sha384_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha256.c b/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha256.c
new file mode 100644
index 000000000..4f4e38c69
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha256.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 01 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 4,
+	.record_iv_len = 8,
+	.mac_len = 0,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_gcm_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha384.c b/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha384.c
new file mode 100644
index 000000000..0bc7c305f
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha384.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha512.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 02 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 4,
+	.record_iv_len = 8,
+	.mac_len = 0,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_gcm_algorithm,
+	.digest = &sha384_algorithm,
+	.handshake = &sha384_algorithm,
+};
diff --git a/src/crypto/mishmash/rsa_aes_cbc_sha1.c b/src/crypto/mishmash/rsa_aes_cbc_sha1.c
index fccb04a9b..0862fb5ac 100644
--- a/src/crypto/mishmash/rsa_aes_cbc_sha1.c
+++ b/src/crypto/mishmash/rsa_aes_cbc_sha1.c
@@ -32,7 +32,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** TLS_RSA_WITH_AES_128_CBC_SHA cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 15 ) = {
+tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 25 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 0,
@@ -47,7 +47,7 @@ tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 15 ) = {
 
 /** TLS_RSA_WITH_AES_256_CBC_SHA cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 16 ) = {
+tls_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 26 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 0,
diff --git a/src/crypto/mishmash/rsa_aes_cbc_sha256.c b/src/crypto/mishmash/rsa_aes_cbc_sha256.c
index ac379b4b1..e5928db82 100644
--- a/src/crypto/mishmash/rsa_aes_cbc_sha256.c
+++ b/src/crypto/mishmash/rsa_aes_cbc_sha256.c
@@ -31,7 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** TLS_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 13 ) = {
+tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 23 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA256 ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 0,
@@ -46,7 +46,7 @@ tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 13 ) = {
 
 /** TLS_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 14 ) = {
+tls_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 24 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA256 ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 0,
diff --git a/src/crypto/mishmash/rsa_aes_gcm_sha256.c b/src/crypto/mishmash/rsa_aes_gcm_sha256.c
index e08deff84..b18bbd844 100644
--- a/src/crypto/mishmash/rsa_aes_gcm_sha256.c
+++ b/src/crypto/mishmash/rsa_aes_gcm_sha256.c
@@ -31,7 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** TLS_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 11 ) = {
+tls_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 21 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_128_GCM_SHA256 ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 4,
diff --git a/src/crypto/mishmash/rsa_aes_gcm_sha384.c b/src/crypto/mishmash/rsa_aes_gcm_sha384.c
index 0b307b420..06558aaed 100644
--- a/src/crypto/mishmash/rsa_aes_gcm_sha384.c
+++ b/src/crypto/mishmash/rsa_aes_gcm_sha384.c
@@ -31,7 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** TLS_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 12 ) = {
+tls_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 22 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_256_GCM_SHA384 ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 4,
diff --git a/src/include/ipxe/tls.h b/src/include/ipxe/tls.h
index 5c218f84f..cf3277820 100644
--- a/src/include/ipxe/tls.h
+++ b/src/include/ipxe/tls.h
@@ -96,6 +96,12 @@ struct tls_header {
 #define TLS_RSA_WITH_AES_256_GCM_SHA384 0x009d
 #define TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 0x009e
 #define TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 0x009f
+#define TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA 0xc013
+#define TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA 0xc014
+#define TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 0xc027
+#define TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 0xc028
+#define TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 0xc02f
+#define TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 0xc030
 
 /* TLS hash algorithm identifiers */
 #define TLS_MD5_ALGORITHM 1

From 65d69d33da445afc7ff56857af1881cf73666be4 Mon Sep 17 00:00:00 2001
From: Ross Lagerwall <ross.lagerwall@citrix.com>
Date: Tue, 30 Jan 2024 10:52:29 +0000
Subject: [PATCH 066/237] [efi] Fix hang during ExitBootServices()

When ExitBootServices() invokes efi_shutdown_hook(), there may be
nothing to generate an interrupt since the timer is disabled in the
first step of ExitBootServices().  Additionally, for VMs OVMF masks
everything from the PIC (except the timer) by default.  This means
that calling cpu_nap() may hang indefinitely.  This was seen in
practice in netfront_reset() when running in a VM on XenServer.

Fix this by skipping the halt if an EFI shutdown is in progress.

Signed-off-by: Ross Lagerwall <ross.lagerwall@citrix.com>
Modified-by: Michael Brown <mcb30@ipxe.org>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm/interface/efi/efiarm_nap.c         | 6 +++++-
 src/arch/loong64/interface/efi/efiloong64_nap.c | 6 +++++-
 src/arch/x86/interface/efi/efix86_nap.c         | 6 +++++-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/arch/arm/interface/efi/efiarm_nap.c b/src/arch/arm/interface/efi/efiarm_nap.c
index 9ed638e9a..fba7a5d82 100644
--- a/src/arch/arm/interface/efi/efiarm_nap.c
+++ b/src/arch/arm/interface/efi/efiarm_nap.c
@@ -46,8 +46,12 @@ static void efiarm_cpu_nap ( void ) {
 	 * The EFI shell doesn't seem to bother sleeping the CPU; it
 	 * just sits there idly burning power.
 	 *
+	 * If a shutdown is in progess, there may be nothing to
+	 * generate an interrupt since the timer is disabled in the
+	 * first step of ExitBootServices().
 	 */
-	__asm__ __volatile__ ( "wfi" );
+	if ( ! efi_shutdown_in_progress )
+		__asm__ __volatile__ ( "wfi" );
 }
 
 PROVIDE_NAP ( efiarm, cpu_nap, efiarm_cpu_nap );
diff --git a/src/arch/loong64/interface/efi/efiloong64_nap.c b/src/arch/loong64/interface/efi/efiloong64_nap.c
index 5cd1c1b94..0a7609783 100644
--- a/src/arch/loong64/interface/efi/efiloong64_nap.c
+++ b/src/arch/loong64/interface/efi/efiloong64_nap.c
@@ -46,8 +46,12 @@ static void efiloong64_cpu_nap ( void ) {
 	 * The EFI shell doesn't seem to bother sleeping the CPU; it
 	 * just sits there idly burning power.
 	 *
+	 * If a shutdown is in progess, there may be nothing to
+	 * generate an interrupt since the timer is disabled in the
+	 * first step of ExitBootServices().
 	 */
-	__asm__ __volatile__ ( "idle 0" );
+	if ( ! efi_shutdown_in_progress )
+		__asm__ __volatile__ ( "idle 0" );
 }
 
 PROVIDE_NAP ( efiloong64, cpu_nap, efiloong64_cpu_nap );
diff --git a/src/arch/x86/interface/efi/efix86_nap.c b/src/arch/x86/interface/efi/efix86_nap.c
index 3ebf0bd68..296876b85 100644
--- a/src/arch/x86/interface/efi/efix86_nap.c
+++ b/src/arch/x86/interface/efi/efix86_nap.c
@@ -46,8 +46,12 @@ static void efix86_cpu_nap ( void ) {
 	 * The EFI shell doesn't seem to bother sleeping the CPU; it
 	 * just sits there idly burning power.
 	 *
+	 * If a shutdown is in progess, there may be nothing to
+	 * generate an interrupt since the timer is disabled in the
+	 * first step of ExitBootServices().
 	 */
-	__asm__ __volatile__ ( "hlt" );
+	if ( ! efi_shutdown_in_progress )
+		__asm__ __volatile__ ( "hlt" );
 }
 
 PROVIDE_NAP ( efix86, cpu_nap, efix86_cpu_nap );

From 0cc0f47443ef9711775a748c2b0fb40e38643733 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 31 Jan 2024 13:49:35 +0000
Subject: [PATCH 067/237] [tls] Tidy up error handling flow in
 tls_send_plaintext()

Coverity reported that tls_send_plaintext() failed to check the return
status from tls_generate_random(), which could potentially result in
uninitialised random data being used as the block initialisation
vector (instead of intentionally random data).

Add the missing return status check, and separate out the error
handling code paths (since on the successful exit code path there will
be no need to free either the plaintext or the ciphertext anyway).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/tls.c | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/src/net/tls.c b/src/net/tls.c
index 58fad65e1..5f89be452 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -2945,9 +2945,9 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 	} __attribute__ (( packed )) iv;
 	struct tls_auth_header authhdr;
 	struct tls_header *tlshdr;
-	void *plaintext = NULL;
-	size_t plaintext_len = len;
-	struct io_buffer *ciphertext = NULL;
+	void *plaintext;
+	size_t plaintext_len;
+	struct io_buffer *ciphertext;
 	size_t ciphertext_len;
 	size_t padding_len;
 	uint8_t mac[digest->digestsize];
@@ -2956,7 +2956,10 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 
 	/* Construct initialisation vector */
 	memcpy ( iv.fixed, cipherspec->fixed_iv, sizeof ( iv.fixed ) );
-	tls_generate_random ( tls, iv.record, sizeof ( iv.record ) );
+	if ( ( rc = tls_generate_random ( tls, iv.record,
+					  sizeof ( iv.record ) ) ) != 0 ) {
+		goto err_random;
+	}
 
 	/* Construct authentication data */
 	authhdr.seq = cpu_to_be64 ( tls->tx_seq );
@@ -2965,7 +2968,7 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 	authhdr.header.length = htons ( len );
 
 	/* Calculate padding length */
-	plaintext_len += suite->mac_len;
+	plaintext_len = ( len + suite->mac_len );
 	if ( is_block_cipher ( cipher ) ) {
 		padding_len = ( ( ( cipher->blocksize - 1 ) &
 				  -( plaintext_len + 1 ) ) + 1 );
@@ -2980,7 +2983,7 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 		DBGC ( tls, "TLS %p could not allocate %zd bytes for "
 		       "plaintext\n", tls, plaintext_len );
 		rc = -ENOMEM_TX_PLAINTEXT;
-		goto done;
+		goto err_plaintext;
 	}
 
 	/* Assemble plaintext */
@@ -3014,7 +3017,7 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 		DBGC ( tls, "TLS %p could not allocate %zd bytes for "
 		       "ciphertext\n", tls, ciphertext_len );
 		rc = -ENOMEM_TX_CIPHERTEXT;
-		goto done;
+		goto err_ciphertext;
 	}
 
 	/* Assemble ciphertext */
@@ -3039,15 +3042,22 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 				       iob_disown ( ciphertext ) ) ) != 0 ) {
 		DBGC ( tls, "TLS %p could not deliver ciphertext: %s\n",
 		       tls, strerror ( rc ) );
-		goto done;
+		goto err_deliver;
 	}
 
 	/* Update TX state machine to next record */
 	tls->tx_seq += 1;
 
- done:
-	free ( plaintext );
+	assert ( plaintext == NULL );
+	assert ( ciphertext == NULL );
+	return 0;
+
+ err_deliver:
 	free_iob ( ciphertext );
+ err_ciphertext:
+	free ( plaintext );
+ err_plaintext:
+ err_random:
 	return rc;
 }
 

From 36a27b22b1c4a587e66cc1eff1193f6c8c927b05 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 2 Feb 2024 17:09:06 +0000
Subject: [PATCH 068/237] [crypto] Fix stray references to AES

The CBC_CIPHER() macro contains some accidentally hardcoded references
to an underlying AES cipher, instead of using the cipher specified in
the macro parameters.

Fix by using the macro parameter as required.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/cbc.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/include/ipxe/cbc.h b/src/include/ipxe/cbc.h
index 382fc9036..f02e51937 100644
--- a/src/include/ipxe/cbc.h
+++ b/src/include/ipxe/cbc.h
@@ -77,19 +77,19 @@ static void _cbc_name ## _setiv ( void *ctx, const void *iv,		\
 				   size_t ivlen ) {			\
 	struct _cbc_name ## _context * _cbc_name ## _ctx = ctx;		\
 	cbc_setiv ( &_cbc_name ## _ctx->raw_ctx, iv, ivlen,		\
-		    &_raw_cipher, &aes_cbc_ctx->cbc_ctx );		\
+		    &_raw_cipher, &_cbc_name ## _ctx->cbc_ctx );	\
 }									\
 static void _cbc_name ## _encrypt ( void *ctx, const void *src,		\
 				    void *dst, size_t len ) {		\
 	struct _cbc_name ## _context * _cbc_name ## _ctx = ctx;		\
 	cbc_encrypt ( &_cbc_name ## _ctx->raw_ctx, src, dst, len,	\
-		      &_raw_cipher, &aes_cbc_ctx->cbc_ctx );		\
+		      &_raw_cipher, &_cbc_name ## _ctx->cbc_ctx );	\
 }									\
 static void _cbc_name ## _decrypt ( void *ctx, const void *src,		\
 				    void *dst, size_t len ) {		\
 	struct _cbc_name ## _context * _cbc_name ## _ctx = ctx;		\
 	cbc_decrypt ( &_cbc_name ## _ctx->raw_ctx, src, dst, len,	\
-		      &_raw_cipher, &aes_cbc_ctx->cbc_ctx );		\
+		      &_raw_cipher, &_cbc_name ## _ctx->cbc_ctx );	\
 }									\
 struct cipher_algorithm _cbc_cipher = {					\
 	.name		= #_cbc_name,					\

From af4583b214bfe98df82d6645387d6c78fd698d7f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 7 Feb 2024 21:16:47 +0000
Subject: [PATCH 069/237] [test] Remove dummy initialisation vector for
 ECB-mode AES tests

A block cipher in ECB mode has no concept of an initialisation vector,
and any data provided to cipher_setiv() for an ECB cipher will be
ignored.  There is no requirement within our cipher algorithm
abstraction for a dummy initialisation vector to be provided.

Remove the entirely spurious dummy 16-byte initialisation vector from
the ECB test cases.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/tests/aes_test.c | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/src/tests/aes_test.c b/src/tests/aes_test.c
index be119c8d8..46a052a2c 100644
--- a/src/tests/aes_test.c
+++ b/src/tests/aes_test.c
@@ -63,11 +63,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	      0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10,	\
 	      0xa3, 0x09, 0x14, 0xdf, 0xf4 )
 
-/** Dummy initialisation vector used for NIST ECB-mode test vectors */
-#define AES_IV_NIST_DUMMY						\
-	IV ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	\
-	     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 )
-
 /** Initialisation vector used for NIST CBC-mode test vectors */
 #define AES_IV_NIST_CBC							\
 	IV ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,	\
@@ -86,7 +81,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** AES-128-ECB (same test as AES-128-Core) */
 CIPHER_TEST ( aes_128_ecb, &aes_ecb_algorithm,
-	AES_KEY_NIST_128, AES_IV_NIST_DUMMY, ADDITIONAL(), AES_PLAINTEXT_NIST,
+	AES_KEY_NIST_128, IV(), ADDITIONAL(), AES_PLAINTEXT_NIST,
 	CIPHERTEXT ( 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60,
 		     0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97,
 		     0xf5, 0xd3, 0xd5, 0x85, 0x03, 0xb9, 0x69, 0x9d,
@@ -110,7 +105,7 @@ CIPHER_TEST ( aes_128_cbc, &aes_cbc_algorithm,
 
 /** AES-192-ECB (same test as AES-192-Core) */
 CIPHER_TEST ( aes_192_ecb, &aes_ecb_algorithm,
-	AES_KEY_NIST_192, AES_IV_NIST_DUMMY, ADDITIONAL(), AES_PLAINTEXT_NIST,
+	AES_KEY_NIST_192, IV(), ADDITIONAL(), AES_PLAINTEXT_NIST,
 	CIPHERTEXT ( 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f,
 		     0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc,
 		     0x97, 0x41, 0x04, 0x84, 0x6d, 0x0a, 0xd3, 0xad,
@@ -134,7 +129,7 @@ CIPHER_TEST ( aes_192_cbc, &aes_cbc_algorithm,
 
 /** AES-256-ECB (same test as AES-256-Core) */
 CIPHER_TEST ( aes_256_ecb, &aes_ecb_algorithm,
-	AES_KEY_NIST_256, AES_IV_NIST_DUMMY, ADDITIONAL(), AES_PLAINTEXT_NIST,
+	AES_KEY_NIST_256, IV(), ADDITIONAL(), AES_PLAINTEXT_NIST,
 	CIPHERTEXT ( 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c,
 		     0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8,
 		     0x59, 0x1c, 0xcb, 0x10, 0xd4, 0x10, 0xed, 0x26,

From e7ae51b0d75d9b9925748743b91405c99e5c7fec Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 7 Feb 2024 21:20:20 +0000
Subject: [PATCH 070/237] [crypto] Add implementation of the DES cipher

The DES block cipher dates back to the 1970s.  It is no longer
relevant for use in TLS cipher suites, but it is still used by the
MS-CHAPv2 authentication protocol which remains unfortunately common
for 802.1x port authentication.

Add an implementation of the DES block cipher, complete with the
extremely comprehensive test vectors published by NBS (the precursor
to NIST) in the form of an utterly adorable typewritten and hand-drawn
paper document.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/des.c           | 695 ++++++++++++++++++++++++++++
 src/include/ipxe/des.h     |  91 ++++
 src/include/ipxe/errfile.h |   1 +
 src/tests/des_test.c       | 898 +++++++++++++++++++++++++++++++++++++
 src/tests/tests.c          |   1 +
 5 files changed, 1686 insertions(+)
 create mode 100644 src/crypto/des.c
 create mode 100644 src/include/ipxe/des.h
 create mode 100644 src/tests/des_test.c

diff --git a/src/crypto/des.c b/src/crypto/des.c
new file mode 100644
index 000000000..6918bec3e
--- /dev/null
+++ b/src/crypto/des.c
@@ -0,0 +1,695 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * DES algorithm
+ *
+ * DES was not designed to be implemented in software, and therefore
+ * contains a large number of bit permutation operations that are
+ * essentially free in hardware (requiring only wires, no gates) but
+ * expensive in software.
+ *
+ * Since DES is no longer used as a practical block cipher for large
+ * volumes of data, we optimise for code size, and do not attempt to
+ * obtain fast throughput.
+ *
+ * The algorithm is specified in NIST SP 800-67, downloadable from
+ * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-67r2.pdf
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/rotate.h>
+#include <ipxe/crypto.h>
+#include <ipxe/ecb.h>
+#include <ipxe/cbc.h>
+#include <ipxe/init.h>
+#include <ipxe/des.h>
+
+/**
+ * DES shift schedule
+ *
+ * The DES shift schedule (ordered from round 16 down to round 1) is
+ * {1,2,2,2,2,2,2,1,2,2,2,2,2,2,1,1}.  In binary, this may be
+ * represented as {1,10,10,10,10,10,10,1,10,10,10,10,10,10,1,1} and
+ * concatenated (without padding) to produce a single binary integer
+ * 1101010101010110101010101011 (equal to 0x0d556aab in hexadecimal).
+ *
+ * This integer may then be consumed LSB-first, where a 1 bit
+ * indicates a shift and the generation of a round key, and a 0 bit
+ * indicates a shift without the generation of a round key.
+ */
+#define DES_SCHEDULE 0x0d556aab
+
+/**
+ * Define an element pair in a DES S-box
+ *
+ * @v x			Upper element of element pair
+ * @v y			Lower element of element pair
+ *
+ * DES S-box elements are 4-bit values.  We encode two values per
+ * byte, ordering the elements so that the six-bit input value may be
+ * used directly as a lookup index.
+ *
+ * Specifically, if the input value is {r1,c3,c2,c1,c0,r0}, where
+ * {r1,r0} is the table row index and {c3,c2,c1,c0} is the table
+ * column index (as used in the DES specification), then:
+ *
+ *   - {r1,c3,c2,c1,c0} is the byte index into the table
+ *
+ *   - (4*r0) is the required bit shift to extract the 4-bit value
+ */
+#define SBYTE( x, y ) ( ( (y) << 4 ) | (x) )
+
+/**
+ * Define a row pair in a DES S-box
+ *
+ * @v x0..xf		Upper row of row pair
+ * @v y0..yf		Lower row of row pair
+ */
+#define SBOX( x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, xa, xb, xc, xd, xe, xf,  \
+	      y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, ya, yb, yc, yd, ye, yf ) \
+	SBYTE ( x0, y0 ), SBYTE ( x1, y1 ), SBYTE ( x2, y2 ), SBYTE ( x3, y3 ),\
+	SBYTE ( x4, y4 ), SBYTE ( x5, y5 ), SBYTE ( x6, y6 ), SBYTE ( x7, y7 ),\
+	SBYTE ( x8, y8 ), SBYTE ( x9, y9 ), SBYTE ( xa, ya ), SBYTE ( xb, yb ),\
+	SBYTE ( xc, yc ), SBYTE ( xd, yd ), SBYTE ( xe, ye ), SBYTE ( xf, yf )
+
+/** DES S-boxes S1..S8 */
+static const uint8_t des_s[8][32] = { {
+	/* S1 */
+	SBOX ( 14,  4, 13,  1,  2, 15, 11,  8,  3, 10,  6, 12,  5,  9,  0,  7,
+		0, 15,  7,  4, 14,  2, 13,  1, 10,  6, 12, 11,  9,  5,  3,  8 ),
+	SBOX (  4,  1, 14,  8, 13,  6,  2, 11, 15, 12,  9,  7,  3, 10,  5,  0,
+	       15, 12,  8,  2,  4,  9,  1,  7,  5, 11,  3, 14, 10,  0,  6, 13 )
+}, {
+	/* S2 */
+	SBOX ( 15,  1,  8, 14,  6, 11,  3,  4,  9,  7,  2, 13, 12,  0,  5, 10,
+		3, 13,  4,  7, 15,  2,  8, 14, 12,  0,  1, 10,  6,  9, 11,  5 ),
+	SBOX (  0, 14,  7, 11, 10,  4, 13,  1,  5,  8, 12,  6,  9,  3,  2, 15,
+		13,  8, 10,  1,  3, 15,  4,  2, 11,  6,  7, 12,  0,  5, 14,  9 )
+}, {
+	/* S3 */
+	SBOX ( 10,  0,  9, 14,  6,  3, 15,  5,  1, 13, 12,  7, 11,  4,  2,  8,
+	       13,  7,  0,  9,  3,  4,  6, 10,  2,  8,  5, 14, 12, 11, 15,  1 ),
+	SBOX ( 13,  6,  4,  9,  8, 15,  3,  0, 11,  1,  2, 12,  5, 10, 14,  7,
+		1, 10, 13,  0,  6,  9,  8,  7,  4, 15, 14,  3, 11,  5,  2, 12 )
+}, {
+	/* S4 */
+	SBOX (  7, 13, 14,  3,  0,  6,  9, 10,  1,  2,  8,  5, 11, 12,  4, 15,
+	       13,  8, 11,  5,  6, 15,  0,  3,  4,  7,  2, 12,  1, 10, 14,  9 ),
+	SBOX ( 10,  6,  9,  0, 12, 11,  7, 13, 15,  1,  3, 14,  5,  2,  8,  4,
+		3, 15,  0,  6, 10,  1, 13,  8,  9,  4,  5, 11, 12,  7,  2, 14 )
+}, {
+	/* S5 */
+	SBOX (  2, 12,  4,  1,  7, 10, 11,  6,  8,  5,  3, 15, 13,  0, 14,  9,
+	       14, 11,  2, 12,  4,  7, 13,  1,  5,  0, 15, 10,  3,  9,  8,  6 ),
+	SBOX (  4,  2,  1, 11, 10, 13,  7,  8, 15,  9, 12,  5,  6,  3,  0, 14,
+	       11,  8, 12,  7,  1, 14,  2, 13,  6, 15,  0,  9, 10,  4,  5,  3 )
+}, {
+	/* S6 */
+	SBOX ( 12,  1, 10, 15,  9,  2,  6,  8,  0, 13,  3,  4, 14,  7,  5, 11,
+	       10, 15,  4,  2,  7, 12,  9,  5,  6,  1, 13, 14,  0, 11,  3,  8 ),
+	SBOX (  9, 14, 15,  5,  2,  8, 12,  3,  7,  0,  4, 10,  1, 13, 11,  6,
+		4,  3,  2, 12,  9,  5, 15, 10, 11, 14,  1,  7,  6,  0,  8, 13 )
+}, {
+	/* S7 */
+	SBOX (  4, 11,  2, 14, 15,  0,  8, 13,  3, 12,  9,  7,  5, 10,  6,  1,
+	       13,  0, 11,  7,  4,  9,  1, 10, 14,  3,  5, 12,  2, 15,  8,  6 ),
+	SBOX (  1,  4, 11, 13, 12,  3,  7, 14, 10, 15,  6,  8,  0,  5,  9,  2,
+		6, 11, 13,  8,  1,  4, 10,  7,  9,  5,  0, 15, 14,  2,  3, 12 )
+}, {
+	/* S8 */
+	SBOX ( 13,  2,  8,  4,  6, 15, 11,  1, 10,  9,  3, 14,  5,  0, 12,  7,
+		1, 15, 13,  8, 10,  3,  7,  4, 12,  5,  6, 11,  0, 14,  9,  2 ),
+	SBOX (  7, 11,  4,  1,  9, 12, 14,  2,  0,  6, 10, 13, 15,  3,  5,  8,
+		2,  1, 14,  7,  4, 10,  8, 13, 15, 12,  9,  0,  3,  5,  6, 11 )
+} };
+
+/**
+ * Define a bit index within permuted choice 2 (PC2)
+ *
+ * @v bit		Bit index
+ *
+ * Permuted choice 2 (PC2) is used to select bits from a concatenated
+ * pair of 28-bit registers ("C" and "D") as part of the key schedule.
+ * We store these as 32-bit registers and so must add 4 to indexes
+ * above 28.
+ */
+#define DES_PC2( x ) ( (x) + ( ( (x) > 28 ) ? 4 : 0 ) )
+
+/**
+ * Define six bits of permuted choice 2 (PC2)
+ *
+ * @v r1:r0		Bits corresponding to S-box row index
+ * @v c3:c0		Bits corresponding to S-box column index
+ *
+ * There are 8 steps within a DES round (one step per S-box).  Each
+ * step requires six bits of the round key, corresponding to the S-box
+ * input value {r1,c3,c2,c1,c0,r0}, where {r1,r0} is the table row
+ * index and {c3,c2,c1,c0} is the table column index.
+ *
+ * As an optimisation, we store the least significant of the 6 bits in
+ * the sign bit of a signed 8-bit value, and the remaining 5 bits in
+ * the least significant 5 bits of the 8-bit value.  See the comments
+ * in des_sbox() for further details.
+ */
+#define DES_PC2R( r1, c3, c2, c1, c0, r0 )				\
+	DES_PC2 ( r0 ), /* LSB stored in sign bit */			\
+	DES_PC2 ( r0 ), /* Unused bit */				\
+	DES_PC2 ( r0 ), /* Unused bit */				\
+	DES_PC2 ( r1 ),	/* Remaining 5 bits */				\
+	DES_PC2 ( c3 ),	/* ... */					\
+	DES_PC2 ( c2 ),	/* ... */					\
+	DES_PC2 ( c1 ),	/* ... */					\
+	DES_PC2 ( c0 ) 	/* ... */
+
+/**
+ * A DES systematic permutation generator
+ *
+ * Many of the permutations used in DES comprise systematic bit
+ * patterns.  We generate these permutations at runtime to save on
+ * code size.
+ */
+struct des_generator {
+	/** Permutation */
+	uint8_t *permutation;
+	/** Seed value */
+	uint32_t seed;
+};
+
+/**
+ * Define a DES permutation generator
+ *
+ * @v PERMUTATION	Permutation
+ * @v OFFSET		Fixed input bit offset (0 or 1)
+ * @v INV<n>		Input bit index bit <n> should be inverted
+ * @v BIT<n>		Source bit for input bit index bit <n>
+ * @ret generator	Permutation generator
+ */
+#define DES_GENERATOR( PERMUTATION, OFFSET, INV5, BIT5, INV4, BIT4,	\
+		       INV3, BIT3, INV2, BIT2, INV1, BIT1, INV0, BIT0 )	\
+	{								\
+		.permutation = (PERMUTATION),				\
+		.seed = ( ( (INV0) << 31 ) | ( (BIT0) << 28 ) |		\
+			  ( (INV1) << 27 ) | ( (BIT1) << 24 ) |		\
+			  ( (INV2) << 23 ) | ( (BIT2) << 20 ) |		\
+			  ( (INV3) << 19 ) | ( (BIT3) << 16 ) |		\
+			  ( (INV4) << 15 ) | ( (BIT4) << 12 ) |		\
+			  ( (INV5) << 11 ) | ( (BIT5) <<  8 ) |		\
+			  ( ( uint32_t ) sizeof (PERMUTATION) - 1 ) |	\
+			  (OFFSET) ),					\
+	}
+
+/** DES permuted choice 1 (PC1) "C" register */
+static uint8_t des_pc1c[29];
+
+/** DES permuted choice 1 (PC1) "D" register */
+static uint8_t des_pc1d[33];
+
+/** DES permuted choice 2 (PC2) */
+static const uint8_t des_pc2[65] = {
+	DES_PC2R ( 14, 17, 11, 24,  1,  5 ),
+	DES_PC2R (  3, 28, 15,  6, 21, 10 ),
+	DES_PC2R ( 23, 19, 12,  4, 26,  8 ),
+	DES_PC2R ( 16,  7, 27, 20, 13,  2 ),
+	DES_PC2R ( 41, 52, 31, 37, 47, 55 ),
+	DES_PC2R ( 30, 40, 51, 45, 33, 48 ),
+	DES_PC2R ( 44, 49, 39, 56, 34, 53 ),
+	DES_PC2R ( 46, 42, 50, 36, 29, 32 ),
+	0 /* terminator */
+};
+
+/** DES initial permutation (IP) */
+static uint8_t des_ip[65];
+
+/** DES data permutation (P) */
+static const uint8_t des_p[33] = {
+	16,  7, 20, 21, 29, 12, 28, 17,  1, 15, 23, 26,  5, 18, 31, 10,
+	 2,  8, 24, 14, 32, 27,  3,  9, 19, 13, 30,  6, 22, 11,  4, 25,
+	 0 /* terminator */
+};
+
+/** DES final / inverse initial permutation (FP / IP^-1) */
+static uint8_t des_fp[65];
+
+/** DES permutation generators */
+static struct des_generator des_generators[] = {
+
+	/* The DES initial permutation transforms the bit index
+	 * {x5,x4,x3,x2,x1,x0}+1 into {~x2,~x1,~x0,x4,x3,~x5}+1
+	 */
+	DES_GENERATOR ( des_ip, 1, 1, 2, 1, 1, 1, 0, 0, 4, 0, 3, 1, 5 ),
+
+	/* The DES final permutation transforms the bit index
+	 * {x5,x4,x3,x2,x1,x0}+1 into {~x0,x2,x1,~x5,~x4,~x3}+1
+	 *
+	 * There is an asymmetry in the DES block diagram for the last
+	 * of the 16 rounds, which is functionally equivalent to
+	 * performing 16 identical rounds and then swapping the left
+	 * and right halves before applying the final permutation.  We
+	 * may therefore account for this asymmetry by inverting the
+	 * MSB in each bit index, to point to the corresponding bit in
+	 * the other half.
+	 *
+	 * This is equivalent to using a permutation that transforms
+	 * {x5,x4,x3,x2,x1,x0}+1 into {x0,x2,x1,~x5,~x4,~x3}+1
+	 */
+	DES_GENERATOR ( des_fp, 1, 0, 0, 0, 2, 0, 1, 1, 5, 1, 4, 1, 3 ),
+
+	/* The "C" half of DES permuted choice 1 (PC1) transforms the
+	 * bit index {x5,x4,x3,x2,x1,x0}+1 into {~x2,~x1,~x0,x5,x4,x3}+1
+	 */
+	DES_GENERATOR ( des_pc1c, 1, 1, 2, 1, 1, 1, 0, 0, 5, 0, 4, 0, 3 ),
+
+	/* The "D" half of DES permuted choice 1 (PC1) transforms the
+	 * bit index {x5,x4,x3,x2,x1,x0}+1 into {~x2,~x1,~x0,~x5,~x4,~x3}+0
+	 *
+	 * Due to the idosyncratic design choice of using 28-bit
+	 * registers in the DES key expansion schedule, the final four
+	 * permutation values appear at indices [28:31] instead of
+	 * [24:27].  This is adjusted for in @c des_setkey().
+	 */
+	DES_GENERATOR ( des_pc1d, 0, 1, 2, 1, 1, 1, 0, 1, 5, 1, 4, 1, 3 ),
+};
+
+/**
+ * Generate DES permutation
+ *
+ * @v generator		Generator
+ */
+static __attribute__ (( noinline )) void
+des_generate ( struct des_generator *generator ) {
+	uint8_t *permutation = generator->permutation;
+	uint32_t seed = generator->seed;
+	unsigned int index = 0;
+	uint8_t accum;
+	uint8_t bit;
+
+	/* Generate permutations
+	 *
+	 * This loop is optimised for code size on a
+	 * register-constrained architecture such as i386.
+	 */
+	do {
+		/* Rotate seed to access MSB's bit descriptor */
+		seed = ror32 ( seed, 8 );
+
+		/* Initialise accumulator with six flag bits */
+		accum = 0xfc;
+
+		/* Accumulate bits until all six flag bits are cleared */
+		do {
+			/* Extract specified bit from index.  Use a
+			 * rotation instead of a shift, since this
+			 * will allow the mask to be elided.
+			 */
+			bit = ror8 ( index, ( seed & 0x07 ) );
+			seed = ror32 ( seed, 3 );
+
+			/* Toggle bit if applicable */
+			bit ^= seed;
+			seed = ror32 ( seed, 1 );
+
+			/* Add bit to accumulator and clear one flag bit */
+			accum <<= 1;
+			accum |= ( bit & 0x01 );
+
+		} while ( accum & 0x80 );
+
+		/* Add constant offset if applicable */
+		accum += ( seed & 0x01 );
+
+		/* Store permutation */
+		permutation[index] = accum;
+
+		/* Loop until reaching length (which is always even) */
+	} while ( ++index < ( seed & 0xfe ) );
+	DBGC2 ( permutation, "DES generated permutation %p:\n", permutation );
+	DBGC2_HDA ( permutation, 0, permutation,
+		    ( ( seed & 0xfe ) + 1 /* zero terminator */ ) );
+}
+
+/**
+ * Initialise permutations
+ */
+static void des_init ( void ) {
+	unsigned int i;
+
+	/* Generate all generated permutations */
+	for ( i = 0 ; i < ( sizeof ( des_generators ) /
+			    sizeof ( des_generators[0] ) ) ; i++ ) {
+		des_generate ( &des_generators[i] );
+	}
+}
+
+/** Initialisation function */
+struct init_fn des_init_fn __init_fn ( INIT_NORMAL ) = {
+	.initialise = des_init,
+};
+
+/**
+ * Perform bit permutation
+ *
+ * @v permutation	Bit permutation (zero-terminated)
+ * @v in		Input value
+ * @v out		Output value
+ */
+static void des_permute ( const uint8_t *permutation, const uint8_t *in,
+			  uint8_t *out ) {
+	uint8_t mask = 0x80;
+	uint8_t accum = 0;
+	unsigned int bit;
+
+	/* Extract individual input bits to construct output value */
+	while ( ( bit = *(permutation++) ) ) {
+		bit--;
+		if ( in[ bit / 8 ] & ( 0x80 >> ( bit % 8 ) ) )
+			accum |= mask;
+		*out = accum;
+		mask = ror8 ( mask, 1 );
+		if ( mask == 0x80 ) {
+			out++;
+			accum = 0;
+		}
+	}
+}
+
+/**
+ * Perform DES S-box substitution
+ *
+ * @v in		32-bit input value (native endian)
+ * @v rkey		48-bit round key
+ * @ret out		32-bit output value (native endian)
+ */
+static uint32_t des_sbox ( uint32_t in, const union des_round_key *rkey ) {
+	uint32_t out = 0;
+	uint32_t lookup;
+	int32_t key;
+	uint8_t sub;
+	unsigned int i;
+
+	/* Perform input expansion, key addition, and S-box substitution */
+	for ( i = 0 ; i < 8 ; i++ ) {
+
+		/* Rotate input and output */
+		out = rol32 ( out, 4 );
+		in = rol32 ( in, 4 );
+
+		/* Extract step key from relevant 6 bits of round key
+		 *
+		 * The least significant of the 6 bits (corresponding
+		 * to bit r0 in the S-box lookup index) is stored in
+		 * the sign bit of the step key byte.  It will
+		 * therefore be propagated via sign extension to the
+		 * MSB of the 32-bit step key.
+		 *
+		 * The remaining 5 of the 6 bits (corresponding to
+		 * bits {r1,c3,c2,c1,c0} in the S-box lookup index)
+		 * are stored in the least significant 5 bits of the
+		 * step key byte and will end up in the least
+		 * significant 5 bits of the 32-bit step key.
+		 */
+		key = rkey->step[i];
+
+		/* Add step key to input to produce S-box lookup index
+		 *
+		 * We do not ever perform an explicit expansion of the
+		 * input value from 32 to 48 bits.  Instead, we rotate
+		 * the 32-bit input value by 4 bits on each step, and
+		 * extract the relevant 6 bits.
+		 *
+		 * The least significant of the 6 bits (corresponding
+		 * to bit r0 in the S-box lookup index) is currently
+		 * in the MSB of the 32-bit (rotated) input value.
+		 *
+		 * The remaining 5 of the 6 bits (corresponding to
+		 * bits {r1,c3,c2,c1,c0} in the S-box lookup index)
+		 * are currently in the least significant 5 bits of
+		 * the 32-bit (rotated) input value.
+		 *
+		 * This aligns with the placement of the bits in the
+		 * step key (see above), and we can therefore perform
+		 * a single XOR to add the 6-bit step key to the
+		 * relevant 6 bits of the input value.
+		 */
+		lookup = ( in ^ key );
+
+		/* Look up S[i][in ^ key] from S-box
+		 *
+		 * We have bits {r1,c3,c2,c1,c0} in the least
+		 * significant 5 bits of the lookup index, and so can
+		 * use the masked lookup index directly as a byte
+		 * index into the relevant S-box to extract the byte
+		 * containing both {r1,c3,c2,c1,c0,'0'} and
+		 * {r1,c3,c2,c1,c0,'1'}.
+		 *
+		 * We then use the MSB of the 32-bit lookup index to
+		 * extract the relevant nibble for the full lookup
+		 * index {r1,c3,c2,c1,c0,r0}.
+		 */
+		sub = des_s[i][ lookup & 0x1f ];
+		sub >>= ( ( lookup >> 29 ) & 4 );
+		sub &= 0x0f;
+
+		/* Substitute S[i][input ^ key] into output */
+		out |= sub;
+	}
+
+	return out;
+}
+
+/**
+ * Perform a single DES round
+ *
+ * @v block		DES block
+ * @v rkey		48-bit round key
+ */
+static void des_round ( union des_block *block,
+			const union des_round_key *rkey ) {
+	union des_dword sbox;
+	uint32_t left;
+	uint32_t right;
+
+	/* Extract left and right halves L[n-1] and R[n-1] */
+	left = block->left.dword;
+	right = block->right.dword;
+	DBGC2 ( block, "DES L=%08x R=%08x K=%08x%08x", be32_to_cpu ( left ),
+		be32_to_cpu ( right ), be32_to_cpu ( rkey->dword[0] ),
+		be32_to_cpu ( rkey->dword[1] ) );
+
+	/* L[n] = R[n-1] */
+	block->left.dword = right;
+
+	/* Calculate Feistel function f(R[n-1], K[n]) */
+	sbox.dword = cpu_to_be32 ( des_sbox ( be32_to_cpu ( right ), rkey ) );
+	des_permute ( des_p, sbox.byte, block->right.byte );
+
+	/* R[n] = L[n-1] + f(R[n-1], K[n]) */
+	block->right.dword ^= left;
+	DBGC2 ( block, " => L=%08x R=%08x\n",
+		be32_to_cpu ( block->left.dword ),
+		be32_to_cpu ( block->right.dword ) );
+}
+
+/**
+ * Perform all DES rounds
+ *
+ * @v in		Input DES block
+ * @v out		Output DES block
+ * @v rkey		Starting 48-bit round key
+ * @v offset		Byte offset between round keys
+ */
+static void des_rounds ( const union des_block *in, union des_block *out,
+			 const union des_round_key *rkey,
+			 ssize_t offset ) {
+	union des_block tmp;
+	unsigned int i;
+
+	/* Apply initial permutation */
+	des_permute ( des_ip, in->byte, tmp.byte );
+
+	/* Perform all DES rounds, consuming keys in the specified order */
+	for ( i = 0 ; i < DES_ROUNDS ; i++ ) {
+		des_round ( &tmp, rkey );
+		rkey = ( ( ( void * ) rkey ) + offset );
+	}
+
+	/* Apply final permutation */
+	DBGC ( &tmp, "DES %scrypted %08x%08x => ",
+	       ( ( offset > 0 ) ? "en" : "de" ), be32_to_cpu ( in->dword[0] ),
+	       be32_to_cpu ( in->dword[1] ) );
+	des_permute ( des_fp, tmp.byte, out->byte );
+	DBGC ( &tmp, "%08x%08x\n", be32_to_cpu ( out->dword[0] ),
+	       be32_to_cpu ( out->dword[1] ) );
+}
+
+/**
+ * Rotate 28-bit word
+ *
+ * @v dword		28-bit dword value
+ * @ret dword		Rotated 28-bit dword value
+ */
+static uint32_t des_rol28 ( uint32_t dword ) {
+	int32_t sdword;
+
+	/* Convert to native-endian */
+	sdword = be32_to_cpu ( dword );
+
+	/* Signed shift right by 4 places to copy bit 31 to bits 27:31 */
+	sdword >>= 4;
+
+	/* Rotate left */
+	sdword = rol32 ( sdword, 1 );
+
+	/* Shift left by 4 places to restore bit positions */
+	sdword <<= 4;
+
+	/* Convert back to big-endian */
+	dword = cpu_to_be32 ( sdword );
+
+	return dword;
+}
+
+/**
+ * Set key
+ *
+ * @v ctx		Context
+ * @v key		Key
+ * @v keylen		Key length
+ * @ret rc		Return status code
+ */
+static int des_setkey ( void *ctx, const void *key, size_t keylen ) {
+	struct des_context *des = ctx;
+	union des_round_key *rkey = des->rkey;
+	union des_block reg;
+	uint32_t schedule;
+
+	/* Validate key length */
+	if ( keylen != DES_BLOCKSIZE )
+		return -EINVAL;
+	DBGC ( des, "DES %p new key:\n", des );
+	DBGC_HDA ( des, 0, key, keylen );
+
+	/* Apply permuted choice 1 */
+	des_permute ( des_pc1c, key, reg.c.byte );
+	des_permute ( des_pc1d, key, reg.d.byte );
+	reg.d.byte[3] <<= 4; /* see comment for @c des_pc1d */
+	DBGC2 ( des, "DES %p C[ 0]=%07x D[ 0]=%07x\n",
+		des, ( be32_to_cpu ( reg.c.dword ) >> 4 ),
+		( be32_to_cpu ( reg.d.dword ) >> 4 ) );
+
+	/* Generate round keys */
+	for ( schedule = DES_SCHEDULE ; schedule ; schedule >>= 1 ) {
+
+		/* Shift 28-bit words */
+		reg.c.dword = des_rol28 ( reg.c.dword );
+		reg.d.dword = des_rol28 ( reg.d.dword );
+
+		/* Skip rounds according to shift schedule */
+		if ( ! ( schedule & 1 ) )
+			continue;
+
+		/* Apply permuted choice 2 */
+		des_permute ( des_pc2, reg.byte, rkey->byte );
+		DBGC2 ( des, "DES %p C[%2zd]=%07x D[%2zd]=%07x K[%2zd]="
+			"%08x%08x\n", des, ( ( rkey - des->rkey ) + 1 ),
+			( be32_to_cpu ( reg.c.dword ) >> 4 ),
+			( ( rkey - des->rkey ) + 1 ),
+			( be32_to_cpu ( reg.d.dword ) >> 4 ),
+			( ( rkey - des->rkey ) + 1 ),
+			be32_to_cpu ( rkey->dword[0] ),
+			be32_to_cpu ( rkey->dword[1] ) );
+
+		/* Move to next key */
+		rkey++;
+	}
+
+	/* Sanity check */
+	assert ( rkey == &des->rkey[DES_ROUNDS] );
+
+	return 0;
+}
+
+/**
+ * Encrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to encrypt
+ * @v dst		Buffer for encrypted data
+ * @v len		Length of data
+ */
+static void des_encrypt ( void *ctx, const void *src, void *dst, size_t len ) {
+	struct des_context *des = ctx;
+
+	/* Sanity check */
+	assert ( len == DES_BLOCKSIZE );
+
+	/* Cipher using keys in forward direction */
+	des_rounds ( src, dst, &des->rkey[0], sizeof ( des->rkey[0] ) );
+}
+
+/**
+ * Decrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to decrypt
+ * @v dst		Buffer for decrypted data
+ * @v len		Length of data
+ */
+static void des_decrypt ( void *ctx, const void *src, void *dst, size_t len ) {
+	struct des_context *des = ctx;
+
+	/* Sanity check */
+	assert ( len == DES_BLOCKSIZE );
+
+	/* Cipher using keys in reverse direction */
+	des_rounds ( src, dst, &des->rkey[ DES_ROUNDS - 1 ],
+		     -sizeof ( des->rkey[0] ) );
+}
+
+/** Basic DES algorithm */
+struct cipher_algorithm des_algorithm = {
+	.name = "des",
+	.ctxsize = sizeof ( struct des_context ),
+	.blocksize = DES_BLOCKSIZE,
+	.alignsize = 0,
+	.authsize = 0,
+	.setkey = des_setkey,
+	.setiv = cipher_null_setiv,
+	.encrypt = des_encrypt,
+	.decrypt = des_decrypt,
+	.auth = cipher_null_auth,
+};
+
+/* DES in Electronic Codebook mode */
+ECB_CIPHER ( des_ecb, des_ecb_algorithm,
+	     des_algorithm, struct des_context, DES_BLOCKSIZE );
+
+/* DES in Cipher Block Chaining mode */
+CBC_CIPHER ( des_cbc, des_cbc_algorithm,
+	     des_algorithm, struct des_context, DES_BLOCKSIZE );
diff --git a/src/include/ipxe/des.h b/src/include/ipxe/des.h
new file mode 100644
index 000000000..755a90ea0
--- /dev/null
+++ b/src/include/ipxe/des.h
@@ -0,0 +1,91 @@
+#ifndef _IPXE_DES_H
+#define _IPXE_DES_H
+
+/** @file
+ *
+ * DES algorithm
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/crypto.h>
+
+/** A DES 32-bit dword value
+ *
+ * DES views data as 64-bit big-endian values, typically handled as a
+ * most-significant "left" half and a least-significant "right" half.
+ */
+union des_dword {
+	/** Raw bytes */
+	uint8_t byte[4];
+	/** 32-bit big-endian dword */
+	uint32_t dword;
+};
+
+/** A DES 64-bit block */
+union des_block {
+	/** Raw bytes */
+	uint8_t byte[8];
+	/** 32-bit big-endian dwords */
+	uint32_t dword[2];
+	/** Named left and right halves */
+	struct {
+		/** Left (most significant) half */
+		union des_dword left;
+		/** Right (least significant) half */
+		union des_dword right;
+	};
+	/** Named "C" and "D" halves */
+	struct {
+		/** "C" (most significant) half */
+		union des_dword c;
+		/** "D" (least significant) half */
+		union des_dword d;
+	};
+};
+
+/** DES blocksize */
+#define DES_BLOCKSIZE sizeof ( union des_block )
+
+/** A DES round key
+ *
+ * A DES round key is a 48-bit value, consumed as 8 groups of 6 bits.
+ * We store these as 8 separate bytes, for simplicity of consumption.
+ */
+union des_round_key {
+	/** Raw bytes */
+	uint8_t byte[8];
+	/** 32-bit big-endian dwords */
+	uint32_t dword[2];
+	/** 6-bit step key byte
+	 *
+	 * There are 8 steps within a DES round (one step per S-box).
+	 * Each step requires six bits of the round key.
+	 *
+	 * As an optimisation, we store the least significant of the 6
+	 * bits in the sign bit of a signed 8-bit value, and the
+	 * remaining 5 bits in the least significant 5 bits of the
+	 * 8-bit value.  See the comments in des_sbox() for further
+	 * details.
+	 */
+	int8_t step[8];
+};
+
+/** Number of DES rounds */
+#define DES_ROUNDS 16
+
+/** DES context */
+struct des_context {
+	/** Round keys */
+	union des_round_key rkey[DES_ROUNDS];
+};
+
+/** DES context size */
+#define DES_CTX_SIZE sizeof ( struct des_context )
+
+extern struct cipher_algorithm des_algorithm;
+extern struct cipher_algorithm des_ecb_algorithm;
+extern struct cipher_algorithm des_cbc_algorithm;
+
+#endif /* _IPXE_DES_H */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 060a42a33..f7a00dbe7 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -408,6 +408,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_efi_shim	      ( ERRFILE_OTHER | 0x005d0000 )
 #define ERRFILE_efi_settings	      ( ERRFILE_OTHER | 0x005e0000 )
 #define ERRFILE_x25519		      ( ERRFILE_OTHER | 0x005f0000 )
+#define ERRFILE_des		      ( ERRFILE_OTHER | 0x00600000 )
 
 /** @} */
 
diff --git a/src/tests/des_test.c b/src/tests/des_test.c
new file mode 100644
index 000000000..ffafbd810
--- /dev/null
+++ b/src/tests/des_test.c
@@ -0,0 +1,898 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * DES tests
+ *
+ * These test vectors are originally provided by NBS (the precursor of
+ * NIST) in SP 500-20, downloadable as a scan of the typewritten
+ * original from:
+ *
+ * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nbsspecialpublication500-20e1980.pdf
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <assert.h>
+#include <ipxe/des.h>
+#include <ipxe/test.h>
+#include "cipher_test.h"
+
+/** Define a DES 64-bit test value */
+#define DES_VALUE(value) {						\
+	( ( ( ( uint64_t ) (value) ) >> 56 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 48 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 40 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 32 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 24 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 16 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >>  8 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >>  0 ) & 0xff )			\
+	}
+
+/** Define a DES test */
+#define DES_TEST( name, key, plaintext, ciphertext )			\
+	CIPHER_TEST ( name, &des_ecb_algorithm,	DES_VALUE ( key ),	\
+		      IV(), ADDITIONAL(), DES_VALUE ( plaintext ),	\
+		      DES_VALUE ( ciphertext ), AUTH() )
+
+/* Sample round outputs (page 9) */
+DES_TEST ( des_round_sample,
+	   0x10316e028c8f3b4a, 0x0000000000000000, 0x82dcbafbdeab6602 );
+
+/* Test 1: Initial permutation and expansion tests
+ *
+ * "Set Key=0 and encrypt the 64-bit data vectors e[i]: i=1,...,64"
+ *
+ * Appendix B, page 28 ("IP and E test")
+ */
+DES_TEST ( des_test1_1,
+	   0x0101010101010101, 0x8000000000000000, 0x95f8a5e5dd31d900 );
+DES_TEST ( des_test1_2,
+	   0x0101010101010101, 0x4000000000000000, 0xdd7f121ca5015619 );
+DES_TEST ( des_test1_3,
+	   0x0101010101010101, 0x2000000000000000, 0x2e8653104f3834ea );
+DES_TEST ( des_test1_4,
+	   0x0101010101010101, 0x1000000000000000, 0x4bd388ff6cd81d4f );
+DES_TEST ( des_test1_5,
+	   0x0101010101010101, 0x0800000000000000, 0x20b9e767b2fb1456 );
+DES_TEST ( des_test1_6,
+	   0x0101010101010101, 0x0400000000000000, 0x55579380d77138ef );
+DES_TEST ( des_test1_7,
+	   0x0101010101010101, 0x0200000000000000, 0x6cc5defaaf04512f );
+DES_TEST ( des_test1_8,
+	   0x0101010101010101, 0x0100000000000000, 0x0d9f279ba5d87260 );
+DES_TEST ( des_test1_9,
+	   0x0101010101010101, 0x0080000000000000, 0xd9031b0271bd5a0a );
+DES_TEST ( des_test1_10,
+	   0x0101010101010101, 0x0040000000000000, 0x424250b37c3dd951 );
+DES_TEST ( des_test1_11,
+	   0x0101010101010101, 0x0020000000000000, 0xb8061b7ecd9a21e5 );
+DES_TEST ( des_test1_12,
+	   0x0101010101010101, 0x0010000000000000, 0xf15d0f286b65bd28 );
+DES_TEST ( des_test1_13,
+	   0x0101010101010101, 0x0008000000000000, 0xadd0cc8d6e5deba1 );
+DES_TEST ( des_test1_14,
+	   0x0101010101010101, 0x0004000000000000, 0xe6d5f82752ad63d1 );
+DES_TEST ( des_test1_15,
+	   0x0101010101010101, 0x0002000000000000, 0xecbfe3bd3f591a5e );
+DES_TEST ( des_test1_16,
+	   0x0101010101010101, 0x0001000000000000, 0xf356834379d165cd );
+DES_TEST ( des_test1_17,
+	   0x0101010101010101, 0x0000800000000000, 0x2b9f982f20037fa9 );
+DES_TEST ( des_test1_18,
+	   0x0101010101010101, 0x0000400000000000, 0x889de068a16f0be6 );
+DES_TEST ( des_test1_19,
+	   0x0101010101010101, 0x0000200000000000, 0xe19e275d846a1298 );
+DES_TEST ( des_test1_20,
+	   0x0101010101010101, 0x0000100000000000, 0x329a8ed523d71aec );
+DES_TEST ( des_test1_21,
+	   0x0101010101010101, 0x0000080000000000, 0xe7fce22557d23c97 );
+DES_TEST ( des_test1_22,
+	   0x0101010101010101, 0x0000040000000000, 0x12a9f5817ff2d65d );
+DES_TEST ( des_test1_23,
+	   0x0101010101010101, 0x0000020000000000, 0xa484c3ad38dc9c19 );
+DES_TEST ( des_test1_24,
+	   0x0101010101010101, 0x0000010000000000, 0xfbe00a8a1ef8ad72 );
+DES_TEST ( des_test1_25,
+	   0x0101010101010101, 0x0000008000000000, 0x750d079407521363 );
+DES_TEST ( des_test1_26,
+	   0x0101010101010101, 0x0000004000000000, 0x64feed9c724c2faf );
+DES_TEST ( des_test1_27,
+	   0x0101010101010101, 0x0000002000000000, 0xf02b263b328e2b60 );
+DES_TEST ( des_test1_28,
+	   0x0101010101010101, 0x0000001000000000, 0x9d64555a9a10b852 );
+DES_TEST ( des_test1_29,
+	   0x0101010101010101, 0x0000000800000000, 0xd106ff0bed5255d7 );
+DES_TEST ( des_test1_30,
+	   0x0101010101010101, 0x0000000400000000, 0xe1652c6b138c64a5 );
+DES_TEST ( des_test1_31,
+	   0x0101010101010101, 0x0000000200000000, 0xe428581186ec8f46 );
+DES_TEST ( des_test1_32,
+	   0x0101010101010101, 0x0000000100000000, 0xaeb5f5ede22d1a36 );
+DES_TEST ( des_test1_33,
+	   0x0101010101010101, 0x0000000080000000, 0xe943d7568aec0c5c );
+DES_TEST ( des_test1_34,
+	   0x0101010101010101, 0x0000000040000000, 0xdf98c8276f54b04b );
+DES_TEST ( des_test1_35,
+	   0x0101010101010101, 0x0000000020000000, 0xb160e4680f6c696f );
+DES_TEST ( des_test1_36,
+	   0x0101010101010101, 0x0000000010000000, 0xfa0752b07d9c4ab8 );
+DES_TEST ( des_test1_37,
+	   0x0101010101010101, 0x0000000008000000, 0xca3a2b036dbc8502 );
+DES_TEST ( des_test1_38,
+	   0x0101010101010101, 0x0000000004000000, 0x5e0905517bb59bcf );
+DES_TEST ( des_test1_39,
+	   0x0101010101010101, 0x0000000002000000, 0x814eeb3b91d90726 );
+DES_TEST ( des_test1_40,
+	   0x0101010101010101, 0x0000000001000000, 0x4d49db1532919c9f );
+DES_TEST ( des_test1_41,
+	   0x0101010101010101, 0x0000000000800000, 0x25eb5fc3f8cf0621 );
+DES_TEST ( des_test1_42,
+	   0x0101010101010101, 0x0000000000400000, 0xab6a20c0620d1c6f );
+DES_TEST ( des_test1_43,
+	   0x0101010101010101, 0x0000000000200000, 0x79e90dbc98f92cca );
+DES_TEST ( des_test1_44,
+	   0x0101010101010101, 0x0000000000100000, 0x866ecedd8072bb0e );
+DES_TEST ( des_test1_45,
+	   0x0101010101010101, 0x0000000000080000, 0x8b54536f2f3e64a8 );
+DES_TEST ( des_test1_46,
+	   0x0101010101010101, 0x0000000000040000, 0xea51d3975595b86b );
+DES_TEST ( des_test1_47,
+	   0x0101010101010101, 0x0000000000020000, 0xcaffc6ac4542de31 );
+DES_TEST ( des_test1_48,
+	   0x0101010101010101, 0x0000000000010000, 0x8dd45a2ddf90796c );
+DES_TEST ( des_test1_49,
+	   0x0101010101010101, 0x0000000000008000, 0x1029d55e880ec2d0 );
+DES_TEST ( des_test1_50,
+	   0x0101010101010101, 0x0000000000004000, 0x5d86cb23639dbea9 );
+DES_TEST ( des_test1_51,
+	   0x0101010101010101, 0x0000000000002000, 0x1d1ca853ae7c0c5f );
+DES_TEST ( des_test1_52,
+	   0x0101010101010101, 0x0000000000001000, 0xce332329248f3228 );
+DES_TEST ( des_test1_53,
+	   0x0101010101010101, 0x0000000000000800, 0x8405d1abe24fb942 );
+DES_TEST ( des_test1_54,
+	   0x0101010101010101, 0x0000000000000400, 0xe643d78090ca4207 );
+DES_TEST ( des_test1_55,
+	   0x0101010101010101, 0x0000000000000200, 0x48221b9937748a23 );
+DES_TEST ( des_test1_56,
+	   0x0101010101010101, 0x0000000000000100, 0xdd7c0bbd61fafd54 );
+DES_TEST ( des_test1_57,
+	   0x0101010101010101, 0x0000000000000080, 0x2fbc291a570db5c4 );
+DES_TEST ( des_test1_58,
+	   0x0101010101010101, 0x0000000000000040, 0xe07c30d7e4e26e12 );
+DES_TEST ( des_test1_59,
+	   0x0101010101010101, 0x0000000000000020, 0x0953e2258e8e90a1 );
+DES_TEST ( des_test1_60,
+	   0x0101010101010101, 0x0000000000000010, 0x5b711bc4ceebf2ee );
+DES_TEST ( des_test1_61,
+	   0x0101010101010101, 0x0000000000000008, 0xcc083f1e6d9e85f6 );
+DES_TEST ( des_test1_62,
+	   0x0101010101010101, 0x0000000000000004, 0xd2fd8867d50d2dfe );
+DES_TEST ( des_test1_63,
+	   0x0101010101010101, 0x0000000000000002, 0x06e7ea22ce92708f );
+DES_TEST ( des_test1_64,
+	   0x0101010101010101, 0x0000000000000001, 0x166b40b44aba4bd6 );
+
+/* Test 2: Inverse permutation and expansion tests
+ *
+ * "Set Key=0 and encrypt the results c[i] obtained in Test 1"
+ *
+ * Appendix B, page 28 ("IP and E test")
+ */
+DES_TEST ( des_test2_1,
+	   0x0101010101010101, 0x95f8a5e5dd31d900, 0x8000000000000000 );
+DES_TEST ( des_test2_2,
+	   0x0101010101010101, 0xdd7f121ca5015619, 0x4000000000000000 );
+DES_TEST ( des_test2_3,
+	   0x0101010101010101, 0x2e8653104f3834ea, 0x2000000000000000 );
+DES_TEST ( des_test2_4,
+	   0x0101010101010101, 0x4bd388ff6cd81d4f, 0x1000000000000000 );
+DES_TEST ( des_test2_5,
+	   0x0101010101010101, 0x20b9e767b2fb1456, 0x0800000000000000 );
+DES_TEST ( des_test2_6,
+	   0x0101010101010101, 0x55579380d77138ef, 0x0400000000000000 );
+DES_TEST ( des_test2_7,
+	   0x0101010101010101, 0x6cc5defaaf04512f, 0x0200000000000000 );
+DES_TEST ( des_test2_8,
+	   0x0101010101010101, 0x0d9f279ba5d87260, 0x0100000000000000 );
+DES_TEST ( des_test2_9,
+	   0x0101010101010101, 0xd9031b0271bd5a0a, 0x0080000000000000 );
+DES_TEST ( des_test2_10,
+	   0x0101010101010101, 0x424250b37c3dd951, 0x0040000000000000 );
+DES_TEST ( des_test2_11,
+	   0x0101010101010101, 0xb8061b7ecd9a21e5, 0x0020000000000000 );
+DES_TEST ( des_test2_12,
+	   0x0101010101010101, 0xf15d0f286b65bd28, 0x0010000000000000 );
+DES_TEST ( des_test2_13,
+	   0x0101010101010101, 0xadd0cc8d6e5deba1, 0x0008000000000000 );
+DES_TEST ( des_test2_14,
+	   0x0101010101010101, 0xe6d5f82752ad63d1, 0x0004000000000000 );
+DES_TEST ( des_test2_15,
+	   0x0101010101010101, 0xecbfe3bd3f591a5e, 0x0002000000000000 );
+DES_TEST ( des_test2_16,
+	   0x0101010101010101, 0xf356834379d165cd, 0x0001000000000000 );
+DES_TEST ( des_test2_17,
+	   0x0101010101010101, 0x2b9f982f20037fa9, 0x0000800000000000 );
+DES_TEST ( des_test2_18,
+	   0x0101010101010101, 0x889de068a16f0be6, 0x0000400000000000 );
+DES_TEST ( des_test2_19,
+	   0x0101010101010101, 0xe19e275d846a1298, 0x0000200000000000 );
+DES_TEST ( des_test2_20,
+	   0x0101010101010101, 0x329a8ed523d71aec, 0x0000100000000000 );
+DES_TEST ( des_test2_21,
+	   0x0101010101010101, 0xe7fce22557d23c97, 0x0000080000000000 );
+DES_TEST ( des_test2_22,
+	   0x0101010101010101, 0x12a9f5817ff2d65d, 0x0000040000000000 );
+DES_TEST ( des_test2_23,
+	   0x0101010101010101, 0xa484c3ad38dc9c19, 0x0000020000000000 );
+DES_TEST ( des_test2_24,
+	   0x0101010101010101, 0xfbe00a8a1ef8ad72, 0x0000010000000000 );
+DES_TEST ( des_test2_25,
+	   0x0101010101010101, 0x750d079407521363, 0x0000008000000000 );
+DES_TEST ( des_test2_26,
+	   0x0101010101010101, 0x64feed9c724c2faf, 0x0000004000000000 );
+DES_TEST ( des_test2_27,
+	   0x0101010101010101, 0xf02b263b328e2b60, 0x0000002000000000 );
+DES_TEST ( des_test2_28,
+	   0x0101010101010101, 0x9d64555a9a10b852, 0x0000001000000000 );
+DES_TEST ( des_test2_29,
+	   0x0101010101010101, 0xd106ff0bed5255d7, 0x0000000800000000 );
+DES_TEST ( des_test2_30,
+	   0x0101010101010101, 0xe1652c6b138c64a5, 0x0000000400000000 );
+DES_TEST ( des_test2_31,
+	   0x0101010101010101, 0xe428581186ec8f46, 0x0000000200000000 );
+DES_TEST ( des_test2_32,
+	   0x0101010101010101, 0xaeb5f5ede22d1a36, 0x0000000100000000 );
+DES_TEST ( des_test2_33,
+	   0x0101010101010101, 0xe943d7568aec0c5c, 0x0000000080000000 );
+DES_TEST ( des_test2_34,
+	   0x0101010101010101, 0xdf98c8276f54b04b, 0x0000000040000000 );
+DES_TEST ( des_test2_35,
+	   0x0101010101010101, 0xb160e4680f6c696f, 0x0000000020000000 );
+DES_TEST ( des_test2_36,
+	   0x0101010101010101, 0xfa0752b07d9c4ab8, 0x0000000010000000 );
+DES_TEST ( des_test2_37,
+	   0x0101010101010101, 0xca3a2b036dbc8502, 0x0000000008000000 );
+DES_TEST ( des_test2_38,
+	   0x0101010101010101, 0x5e0905517bb59bcf, 0x0000000004000000 );
+DES_TEST ( des_test2_39,
+	   0x0101010101010101, 0x814eeb3b91d90726, 0x0000000002000000 );
+DES_TEST ( des_test2_40,
+	   0x0101010101010101, 0x4d49db1532919c9f, 0x0000000001000000 );
+DES_TEST ( des_test2_41,
+	   0x0101010101010101, 0x25eb5fc3f8cf0621, 0x0000000000800000 );
+DES_TEST ( des_test2_42,
+	   0x0101010101010101, 0xab6a20c0620d1c6f, 0x0000000000400000 );
+DES_TEST ( des_test2_43,
+	   0x0101010101010101, 0x79e90dbc98f92cca, 0x0000000000200000 );
+DES_TEST ( des_test2_44,
+	   0x0101010101010101, 0x866ecedd8072bb0e, 0x0000000000100000 );
+DES_TEST ( des_test2_45,
+	   0x0101010101010101, 0x8b54536f2f3e64a8, 0x0000000000080000 );
+DES_TEST ( des_test2_46,
+	   0x0101010101010101, 0xea51d3975595b86b, 0x0000000000040000 );
+DES_TEST ( des_test2_47,
+	   0x0101010101010101, 0xcaffc6ac4542de31, 0x0000000000020000 );
+DES_TEST ( des_test2_48,
+	   0x0101010101010101, 0x8dd45a2ddf90796c, 0x0000000000010000 );
+DES_TEST ( des_test2_49,
+	   0x0101010101010101, 0x1029d55e880ec2d0, 0x0000000000008000 );
+DES_TEST ( des_test2_50,
+	   0x0101010101010101, 0x5d86cb23639dbea9, 0x0000000000004000 );
+DES_TEST ( des_test2_51,
+	   0x0101010101010101, 0x1d1ca853ae7c0c5f, 0x0000000000002000 );
+DES_TEST ( des_test2_52,
+	   0x0101010101010101, 0xce332329248f3228, 0x0000000000001000 );
+DES_TEST ( des_test2_53,
+	   0x0101010101010101, 0x8405d1abe24fb942, 0x0000000000000800 );
+DES_TEST ( des_test2_54,
+	   0x0101010101010101, 0xe643d78090ca4207, 0x0000000000000400 );
+DES_TEST ( des_test2_55,
+	   0x0101010101010101, 0x48221b9937748a23, 0x0000000000000200 );
+DES_TEST ( des_test2_56,
+	   0x0101010101010101, 0xdd7c0bbd61fafd54, 0x0000000000000100 );
+DES_TEST ( des_test2_57,
+	   0x0101010101010101, 0x2fbc291a570db5c4, 0x0000000000000080 );
+DES_TEST ( des_test2_58,
+	   0x0101010101010101, 0xe07c30d7e4e26e12, 0x0000000000000040 );
+DES_TEST ( des_test2_59,
+	   0x0101010101010101, 0x0953e2258e8e90a1, 0x0000000000000020 );
+DES_TEST ( des_test2_60,
+	   0x0101010101010101, 0x5b711bc4ceebf2ee, 0x0000000000000010 );
+DES_TEST ( des_test2_61,
+	   0x0101010101010101, 0xcc083f1e6d9e85f6, 0x0000000000000008 );
+DES_TEST ( des_test2_62,
+	   0x0101010101010101, 0xd2fd8867d50d2dfe, 0x0000000000000004 );
+DES_TEST ( des_test2_63,
+	   0x0101010101010101, 0x06e7ea22ce92708f, 0x0000000000000002 );
+DES_TEST ( des_test2_64,
+	   0x0101010101010101, 0x166b40b44aba4bd6, 0x0000000000000001 );
+
+/* Test 3: Data permutation tests
+ *
+ * "Set the plaintext to zero and process the 32 keys in PTEST"
+ *
+ * Appendix B, page 32 ("PTEST")
+ */
+DES_TEST ( des_test3_1,
+	   0x1046913489980131, 0x0000000000000000, 0x88d55e54f54c97b4 );
+DES_TEST ( des_test3_2,
+	   0x1007103489988020, 0x0000000000000000, 0x0c0cc00c83ea48fd );
+DES_TEST ( des_test3_3,
+	   0x10071034c8980120, 0x0000000000000000, 0x83bc8ef3a6570183 );
+DES_TEST ( des_test3_4,
+	   0x1046103489988020, 0x0000000000000000, 0xdf725dcad94ea2e9 );
+DES_TEST ( des_test3_5,
+	   0x1086911519190101, 0x0000000000000000, 0xe652b53b550be8b0 );
+DES_TEST ( des_test3_6,
+	   0x1086911519580101, 0x0000000000000000, 0xaf527120c485cbb0 );
+DES_TEST ( des_test3_7,
+	   0x5107b01519580101, 0x0000000000000000, 0x0f04ce393db926d5 );
+DES_TEST ( des_test3_8,
+	   0x1007b01519190101, 0x0000000000000000, 0xc9f00ffc74079067 );
+DES_TEST ( des_test3_9,
+	   0x3107915498080101, 0x0000000000000000, 0x7cfd82a593252b4e );
+DES_TEST ( des_test3_10,
+	   0x3107919498080101, 0x0000000000000000, 0xcb49a2f9e91363e3 );
+DES_TEST ( des_test3_11,
+	   0x10079115b9080140, 0x0000000000000000, 0x00b588be70d23f56 );
+DES_TEST ( des_test3_12,
+	   0x3107911598080140, 0x0000000000000000, 0x406a9a6ab43399ae );
+DES_TEST ( des_test3_13,
+	   0x1007d01589980101, 0x0000000000000000, 0x6cb773611dca9ada );
+DES_TEST ( des_test3_14,
+	   0x9107911589980101, 0x0000000000000000, 0x67fd21c17dbb5d70 );
+DES_TEST ( des_test3_15,
+	   0x9107d01589190101, 0x0000000000000000, 0x9592cb4110430787 );
+DES_TEST ( des_test3_16,
+	   0x1007d01598980120, 0x0000000000000000, 0xa6b7ff68a318ddd3 );
+DES_TEST ( des_test3_17,
+	   0x1007940498190101, 0x0000000000000000, 0x4d102196c914ca16 );
+DES_TEST ( des_test3_18,
+	   0x0107910491190401, 0x0000000000000000, 0x2dfa9f4573594965 );
+DES_TEST ( des_test3_19,
+	   0x0107910491190101, 0x0000000000000000, 0xb46604816c0e0774 );
+DES_TEST ( des_test3_20,
+	   0x0107940491190401, 0x0000000000000000, 0x6e7e6221a4f34e87 );
+DES_TEST ( des_test3_21,
+	   0x19079210981a0101, 0x0000000000000000, 0xaa85e74643233199 );
+DES_TEST ( des_test3_22,
+	   0x1007911998190801, 0x0000000000000000, 0x2e5a19db4d1962d6 );
+DES_TEST ( des_test3_23,
+	   0x10079119981a0801, 0x0000000000000000, 0x23a866a809d30894 );
+DES_TEST ( des_test3_24,
+	   0x1007921098190101, 0x0000000000000000, 0xd812d961f017d320 );
+DES_TEST ( des_test3_25,
+	   0x100791159819010b, 0x0000000000000000, 0x055605816e58608f );
+DES_TEST ( des_test3_26,
+	   0x1004801598190101, 0x0000000000000000, 0xabd88e8b1b7716f1 );
+DES_TEST ( des_test3_27,
+	   0x1004801598190102, 0x0000000000000000, 0x537ac95be69da1e1 );
+DES_TEST ( des_test3_28,
+	   0x1004801598190108, 0x0000000000000000, 0xaed0f6ae3c25cdd8 );
+DES_TEST ( des_test3_29,
+	   0x1002911498100104, 0x0000000000000000, 0xb3e35a5ee53e7b8d );
+DES_TEST ( des_test3_30,
+	   0x1002911598190104, 0x0000000000000000, 0x61c79c71921a2ef8 );
+DES_TEST ( des_test3_31,
+	   0x1002911598100201, 0x0000000000000000, 0xe2f5728f0995013c );
+DES_TEST ( des_test3_32,
+	   0x1002911698100101, 0x0000000000000000, 0x1aeac39a61f0a464 );
+
+/* Test 4: Key permutation tests
+ *
+ * "Set Data=0 and use the keys e[i]: i=1,...,64 ignoring i=8,16,...,64"
+ *
+ * Test 4 part 1 is the forward direction as described above.  Test 4
+ * part 2 ("set data=c[i] from part 1 ... then decipher") is carried
+ * out for us automatically, since CIPHER_TEST() performs both
+ * encryption and decryption tests.
+ *
+ * Appendix B, page 30 ("PC1 and PC2 test")
+ */
+DES_TEST ( des_test4_1,
+	   0x8001010101010101, 0x0000000000000000, 0x95a8d72813daa94d );
+DES_TEST ( des_test4_2,
+	   0x4001010101010101, 0x0000000000000000, 0x0eec1487dd8c26d5 );
+DES_TEST ( des_test4_3,
+	   0x2001010101010101, 0x0000000000000000, 0x7ad16ffb79c45926 );
+DES_TEST ( des_test4_4,
+	   0x1001010101010101, 0x0000000000000000, 0xd3746294ca6a6cf3 );
+DES_TEST ( des_test4_5,
+	   0x0801010101010101, 0x0000000000000000, 0x809f5f873c1fd761 );
+DES_TEST ( des_test4_6,
+	   0x0401010101010101, 0x0000000000000000, 0xc02faffec989d1fc );
+DES_TEST ( des_test4_7,
+	   0x0201010101010101, 0x0000000000000000, 0x4615aa1d33e72f10 );
+DES_TEST ( des_test4_8,
+	   0x0180010101010101, 0x0000000000000000, 0x2055123350c00858 );
+DES_TEST ( des_test4_9,
+	   0x0140010101010101, 0x0000000000000000, 0xdf3b99d6577397c8 );
+DES_TEST ( des_test4_10,
+	   0x0120010101010101, 0x0000000000000000, 0x31fe17369b5288c9 );
+DES_TEST ( des_test4_11,
+	   0x0110010101010101, 0x0000000000000000, 0xdfdd3cc64dae1642 );
+DES_TEST ( des_test4_12,
+	   0x0108010101010101, 0x0000000000000000, 0x178c83ce2b399d94 );
+DES_TEST ( des_test4_13,
+	   0x0104010101010101, 0x0000000000000000, 0x50f636324a9b7f80 );
+DES_TEST ( des_test4_14,
+	   0x0102010101010101, 0x0000000000000000, 0xa8468ee3bc18f06d );
+DES_TEST ( des_test4_15,
+	   0x0101800101010101, 0x0000000000000000, 0xa2dc9e92fd3cde92 );
+DES_TEST ( des_test4_16,
+	   0x0101400101010101, 0x0000000000000000, 0xcac09f797d031287 );
+DES_TEST ( des_test4_17,
+	   0x0101200101010101, 0x0000000000000000, 0x90ba680b22aeb525 );
+DES_TEST ( des_test4_18,
+	   0x0101100101010101, 0x0000000000000000, 0xce7a24f350e280b6 );
+DES_TEST ( des_test4_19,
+	   0x0101080101010101, 0x0000000000000000, 0x882bff0aa01a0b87 );
+DES_TEST ( des_test4_20,
+	   0x0101040101010101, 0x0000000000000000, 0x25610288924511c2 );
+DES_TEST ( des_test4_21,
+	   0x0101020101010101, 0x0000000000000000, 0xc71516c29c75d170 );
+DES_TEST ( des_test4_22,
+	   0x0101018001010101, 0x0000000000000000, 0x5199c29a52c9f059 );
+DES_TEST ( des_test4_23,
+	   0x0101014001010101, 0x0000000000000000, 0xc22f0a294a71f29f );
+DES_TEST ( des_test4_24,
+	   0x0101012001010101, 0x0000000000000000, 0xee371483714c02ea );
+DES_TEST ( des_test4_25,
+	   0x0101011001010101, 0x0000000000000000, 0xa81fbd448f9e522f );
+DES_TEST ( des_test4_26,
+	   0x0101010801010101, 0x0000000000000000, 0x4f644c92e192dfed );
+DES_TEST ( des_test4_27,
+	   0x0101010401010101, 0x0000000000000000, 0x1afa9a66a6df92ae );
+DES_TEST ( des_test4_28,
+	   0x0101010201010101, 0x0000000000000000, 0xb3c1cc715cb879d8 );
+DES_TEST ( des_test4_29,
+	   0x0101010180010101, 0x0000000000000000, 0x19d032e64ab0bd8b );
+DES_TEST ( des_test4_30,
+	   0x0101010140010101, 0x0000000000000000, 0x3cfaa7a7dc8720dc );
+DES_TEST ( des_test4_31,
+	   0x0101010120010101, 0x0000000000000000, 0xb7265f7f447ac6f3 );
+DES_TEST ( des_test4_32,
+	   0x0101010110010101, 0x0000000000000000, 0x9db73b3c0d163f54 );
+DES_TEST ( des_test4_33,
+	   0x0101010108010101, 0x0000000000000000, 0x8181b65babf4a975 );
+DES_TEST ( des_test4_34,
+	   0x0101010104010101, 0x0000000000000000, 0x93c9b64042eaa240 );
+DES_TEST ( des_test4_35,
+	   0x0101010102010101, 0x0000000000000000, 0x5570530829705592 );
+DES_TEST ( des_test4_36,
+	   0x0101010101800101, 0x0000000000000000, 0x8638809e878787a0 );
+DES_TEST ( des_test4_37,
+	   0x0101010101400101, 0x0000000000000000, 0x41b9a79af79ac208 );
+DES_TEST ( des_test4_38,
+	   0x0101010101200101, 0x0000000000000000, 0x7a9be42f2009a892 );
+DES_TEST ( des_test4_39,
+	   0x0101010101100101, 0x0000000000000000, 0x29038d56ba6d2745 );
+DES_TEST ( des_test4_40,
+	   0x0101010101080101, 0x0000000000000000, 0x5495c6abf1e5df51 );
+DES_TEST ( des_test4_41,
+	   0x0101010101040101, 0x0000000000000000, 0xae13dbd561488933 );
+DES_TEST ( des_test4_42,
+	   0x0101010101020101, 0x0000000000000000, 0x024d1ffa8904e389 );
+DES_TEST ( des_test4_43,
+	   0x0101010101018001, 0x0000000000000000, 0xd1399712f99bf02e );
+DES_TEST ( des_test4_44,
+	   0x0101010101014001, 0x0000000000000000, 0x14c1d7c1cffec79e );
+DES_TEST ( des_test4_45,
+	   0x0101010101012001, 0x0000000000000000, 0x1de5279dae3bed6f );
+DES_TEST ( des_test4_46,
+	   0x0101010101011001, 0x0000000000000000, 0xe941a33f85501303 );
+DES_TEST ( des_test4_47,
+	   0x0101010101010801, 0x0000000000000000, 0xda99dbbc9a03f379 );
+DES_TEST ( des_test4_48,
+	   0x0101010101010401, 0x0000000000000000, 0xb7fc92f91d8e92e9 );
+DES_TEST ( des_test4_49,
+	   0x0101010101010201, 0x0000000000000000, 0xae8e5caa3ca04e85 );
+DES_TEST ( des_test4_50,
+	   0x0101010101010180, 0x0000000000000000, 0x9cc62df43b6eed74 );
+DES_TEST ( des_test4_51,
+	   0x0101010101010140, 0x0000000000000000, 0xd863dbb5c59a91a0 );
+DES_TEST ( des_test4_52,
+	   0x0101010101010120, 0x0000000000000000, 0xa1ab2190545b91d7 );
+DES_TEST ( des_test4_53,
+	   0x0101010101010110, 0x0000000000000000, 0x0875041e64c570f7 );
+DES_TEST ( des_test4_54,
+	   0x0101010101010108, 0x0000000000000000, 0x5a594528bebef1cc );
+DES_TEST ( des_test4_55,
+	   0x0101010101010104, 0x0000000000000000, 0xfcdb3291de21f0c0 );
+DES_TEST ( des_test4_56,
+	   0x0101010101010102, 0x0000000000000000, 0x869efd7f9f265a09 );
+
+/* Test 5: S-box tests
+ *
+ * "Set Data and Key equal to the inputs defined in the Substitution
+ * Table test"
+ *
+ * Appendix B, page 33 ("19 key data pairs which exercise every S-box entry")
+ */
+DES_TEST ( des_test5_1,
+	   0x7ca110454a1a6e57, 0x01a1d6d039776742, 0x690f5b0d9a26939b );
+DES_TEST ( des_test5_2,
+	   0x0131d9619dc1376e, 0x5cd54ca83def57da, 0x7a389d10354bd271 );
+DES_TEST ( des_test5_3,
+	   0x07a1133e4a0b2686, 0x0248d43806f67172, 0x868ebb51cab4599a );
+DES_TEST ( des_test5_4,
+	   0x3849674c2602319e, 0x51454b582ddf440a, 0x7178876e01f19b2a );
+DES_TEST ( des_test5_5,
+	   0x04b915ba43feb5b6, 0x42fd443059577fa2, 0xaf37fb421f8c4095 );
+DES_TEST ( des_test5_6,
+	   0x0113b970fd34f2ce, 0x059b5e0851cf143a, 0x86a560f10ec6d85b );
+DES_TEST ( des_test5_7,
+	   0x0170f175468fb5e6, 0x0756d8e0774761d2, 0x0cd3da020021dc09 );
+DES_TEST ( des_test5_8,
+	   0x43297fad38e373fe, 0x762514b829bf486a, 0xea676b2cb7db2b7a );
+DES_TEST ( des_test5_9,
+	   0x07a7137045da2a16, 0x3bdd119049372802, 0xdfd64a815caf1a0f );
+DES_TEST ( des_test5_10,
+	   0x04689104c2fd3b2f, 0x26955f6835af609a, 0x5c513c9c4886c088 );
+DES_TEST ( des_test5_11,
+	   0x37d06bb516cb7546, 0x164d5e404f275232, 0x0a2aeeae3ff4ab77 );
+DES_TEST ( des_test5_12,
+	   0x1f08260d1ac2465e, 0x6b056e18759f5cca, 0xef1bf03e5dfa575a );
+DES_TEST ( des_test5_13,
+	   0x584023641aba6176, 0x004bd6ef09176062, 0x88bf0db6d70dee56 );
+DES_TEST ( des_test5_14,
+	   0x025816164629b007, 0x480d39006ee762f2, 0xa1f9915541020b56 );
+DES_TEST ( des_test5_15,
+	   0x49793ebc79b3258f, 0x437540c8698f3cfa, 0x6fbf1cafcffd0556 );
+DES_TEST ( des_test5_16,
+	   0x4fb05e1515ab73a7, 0x072d43a077075292, 0x2f22e49bab7ca1ac );
+DES_TEST ( des_test5_17,
+	   0x49e95d6d4ca229bf, 0x02fe55778117f12a, 0x5a6b612cc26cce4a );
+DES_TEST ( des_test5_18,
+	   0x018310dc409b26d6, 0x1d9d5c5018f728c2, 0x5f4c038ed12b2e41 );
+DES_TEST ( des_test5_19,
+	   0x1c587f1c13924fef, 0x305532286d6f295a, 0x63fac0d034d9f793 );
+
+/* Unofficial tests
+ *
+ * The official tests are all exactly one block in length.  Add some
+ * multi-block tests (generated in Python).
+ */
+CIPHER_TEST ( des_unofficial_ecb, &des_ecb_algorithm,
+	      KEY ( 0x6e, 0x6f, 0x70, 0x61, 0x72, 0x69, 0x74, 0x79 ),
+	      IV(), ADDITIONAL(),
+	      PLAINTEXT ( 0x53, 0x6f, 0x20, 0x63, 0x75, 0x74, 0x65, 0x20,
+			  0x74, 0x6f, 0x20, 0x73, 0x65, 0x65, 0x20, 0x61,
+			  0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69,
+			  0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x77,
+			  0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x20, 0x6f,
+			  0x6e, 0x20, 0x61, 0x6e, 0x20, 0x61, 0x63, 0x74,
+			  0x75, 0x61, 0x6c, 0x20, 0x74, 0x79, 0x70, 0x65,
+			  0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x21, 0x21 ),
+	      CIPHERTEXT ( 0x1a, 0x02, 0x17, 0xcb, 0x93, 0xa3, 0xd2, 0xf2,
+			   0xf9, 0x45, 0x71, 0x1c, 0x33, 0xb1, 0x5c, 0xa4,
+			   0x8b, 0x6b, 0x11, 0x7a, 0x7c, 0x86, 0x7c, 0x7f,
+			   0x9f, 0x56, 0x61, 0x46, 0x7f, 0xa6, 0xae, 0xf1,
+			   0x49, 0xf7, 0x53, 0xe0, 0xbc, 0x15, 0x6a, 0x30,
+			   0xe7, 0xf8, 0xf3, 0x29, 0x11, 0xd8, 0x7d, 0x04,
+			   0x62, 0x5a, 0xaa, 0xa1, 0x89, 0x61, 0x4c, 0xf6,
+			   0x5a, 0x47, 0x3b, 0xc6, 0x04, 0x15, 0xce, 0xf6 ),
+	      AUTH() );
+CIPHER_TEST ( des_unofficial_cbc, &des_cbc_algorithm,
+	      KEY ( 0x6e, 0x6f, 0x70, 0x61, 0x72, 0x69, 0x74, 0x79 ),
+	      IV ( 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 ),
+	      ADDITIONAL(),
+	      PLAINTEXT ( 0x53, 0x6f, 0x20, 0x63, 0x75, 0x74, 0x65, 0x20,
+			  0x74, 0x6f, 0x20, 0x73, 0x65, 0x65, 0x20, 0x61,
+			  0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69,
+			  0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x77,
+			  0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x20, 0x6f,
+			  0x6e, 0x20, 0x61, 0x6e, 0x20, 0x61, 0x63, 0x74,
+			  0x75, 0x61, 0x6c, 0x20, 0x74, 0x79, 0x70, 0x65,
+			  0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x21, 0x21 ),
+	      CIPHERTEXT ( 0x4c, 0x5f, 0x62, 0xfc, 0xf4, 0x93, 0x09, 0xb5,
+			   0x1d, 0x52, 0x25, 0xec, 0xc7, 0x42, 0x3c, 0x29,
+			   0x33, 0x67, 0xf5, 0xe9, 0xd6, 0x3c, 0x27, 0x5b,
+			   0x49, 0x69, 0xc5, 0xa9, 0x08, 0xa3, 0x14, 0x66,
+			   0x3c, 0x95, 0x33, 0x30, 0xcf, 0x3c, 0x7c, 0xaf,
+			   0xa3, 0xe4, 0xf8, 0x2e, 0xc3, 0x55, 0x57, 0x81,
+			   0x33, 0xd9, 0x90, 0xe2, 0x99, 0xdc, 0x32, 0x10,
+			   0x13, 0x21, 0xb6, 0xc1, 0x6b, 0x0f, 0x22, 0xa9 ),
+	      AUTH() );
+
+/**
+ * Perform DES self-test
+ *
+ */
+static void des_test_exec ( void ) {
+
+	/* Sample round outputs (page 9) */
+	cipher_ok ( &des_round_sample );
+
+	/* Test 1: Initial permutation and expansion tests */
+	cipher_ok ( &des_test1_1 );
+	cipher_ok ( &des_test1_2 );
+	cipher_ok ( &des_test1_3 );
+	cipher_ok ( &des_test1_4 );
+	cipher_ok ( &des_test1_5 );
+	cipher_ok ( &des_test1_6 );
+	cipher_ok ( &des_test1_7 );
+	cipher_ok ( &des_test1_8 );
+	cipher_ok ( &des_test1_9 );
+	cipher_ok ( &des_test1_10 );
+	cipher_ok ( &des_test1_11 );
+	cipher_ok ( &des_test1_12 );
+	cipher_ok ( &des_test1_13 );
+	cipher_ok ( &des_test1_14 );
+	cipher_ok ( &des_test1_15 );
+	cipher_ok ( &des_test1_16 );
+	cipher_ok ( &des_test1_17 );
+	cipher_ok ( &des_test1_18 );
+	cipher_ok ( &des_test1_19 );
+	cipher_ok ( &des_test1_20 );
+	cipher_ok ( &des_test1_21 );
+	cipher_ok ( &des_test1_22 );
+	cipher_ok ( &des_test1_23 );
+	cipher_ok ( &des_test1_24 );
+	cipher_ok ( &des_test1_25 );
+	cipher_ok ( &des_test1_26 );
+	cipher_ok ( &des_test1_27 );
+	cipher_ok ( &des_test1_28 );
+	cipher_ok ( &des_test1_29 );
+	cipher_ok ( &des_test1_30 );
+	cipher_ok ( &des_test1_31 );
+	cipher_ok ( &des_test1_32 );
+	cipher_ok ( &des_test1_33 );
+	cipher_ok ( &des_test1_34 );
+	cipher_ok ( &des_test1_35 );
+	cipher_ok ( &des_test1_36 );
+	cipher_ok ( &des_test1_37 );
+	cipher_ok ( &des_test1_38 );
+	cipher_ok ( &des_test1_39 );
+	cipher_ok ( &des_test1_40 );
+	cipher_ok ( &des_test1_41 );
+	cipher_ok ( &des_test1_42 );
+	cipher_ok ( &des_test1_43 );
+	cipher_ok ( &des_test1_44 );
+	cipher_ok ( &des_test1_45 );
+	cipher_ok ( &des_test1_46 );
+	cipher_ok ( &des_test1_47 );
+	cipher_ok ( &des_test1_48 );
+	cipher_ok ( &des_test1_49 );
+	cipher_ok ( &des_test1_50 );
+	cipher_ok ( &des_test1_51 );
+	cipher_ok ( &des_test1_52 );
+	cipher_ok ( &des_test1_53 );
+	cipher_ok ( &des_test1_54 );
+	cipher_ok ( &des_test1_55 );
+	cipher_ok ( &des_test1_56 );
+	cipher_ok ( &des_test1_57 );
+	cipher_ok ( &des_test1_58 );
+	cipher_ok ( &des_test1_59 );
+	cipher_ok ( &des_test1_60 );
+	cipher_ok ( &des_test1_61 );
+	cipher_ok ( &des_test1_62 );
+	cipher_ok ( &des_test1_63 );
+	cipher_ok ( &des_test1_64 );
+
+	/* Test 2: Inverse permutation and expansion tests */
+	cipher_ok ( &des_test2_1 );
+	cipher_ok ( &des_test2_2 );
+	cipher_ok ( &des_test2_3 );
+	cipher_ok ( &des_test2_4 );
+	cipher_ok ( &des_test2_5 );
+	cipher_ok ( &des_test2_6 );
+	cipher_ok ( &des_test2_7 );
+	cipher_ok ( &des_test2_8 );
+	cipher_ok ( &des_test2_9 );
+	cipher_ok ( &des_test2_10 );
+	cipher_ok ( &des_test2_11 );
+	cipher_ok ( &des_test2_12 );
+	cipher_ok ( &des_test2_13 );
+	cipher_ok ( &des_test2_14 );
+	cipher_ok ( &des_test2_15 );
+	cipher_ok ( &des_test2_16 );
+	cipher_ok ( &des_test2_17 );
+	cipher_ok ( &des_test2_18 );
+	cipher_ok ( &des_test2_19 );
+	cipher_ok ( &des_test2_20 );
+	cipher_ok ( &des_test2_21 );
+	cipher_ok ( &des_test2_22 );
+	cipher_ok ( &des_test2_23 );
+	cipher_ok ( &des_test2_24 );
+	cipher_ok ( &des_test2_25 );
+	cipher_ok ( &des_test2_26 );
+	cipher_ok ( &des_test2_27 );
+	cipher_ok ( &des_test2_28 );
+	cipher_ok ( &des_test2_29 );
+	cipher_ok ( &des_test2_30 );
+	cipher_ok ( &des_test2_31 );
+	cipher_ok ( &des_test2_32 );
+	cipher_ok ( &des_test2_33 );
+	cipher_ok ( &des_test2_34 );
+	cipher_ok ( &des_test2_35 );
+	cipher_ok ( &des_test2_36 );
+	cipher_ok ( &des_test2_37 );
+	cipher_ok ( &des_test2_38 );
+	cipher_ok ( &des_test2_39 );
+	cipher_ok ( &des_test2_40 );
+	cipher_ok ( &des_test2_41 );
+	cipher_ok ( &des_test2_42 );
+	cipher_ok ( &des_test2_43 );
+	cipher_ok ( &des_test2_44 );
+	cipher_ok ( &des_test2_45 );
+	cipher_ok ( &des_test2_46 );
+	cipher_ok ( &des_test2_47 );
+	cipher_ok ( &des_test2_48 );
+	cipher_ok ( &des_test2_49 );
+	cipher_ok ( &des_test2_50 );
+	cipher_ok ( &des_test2_51 );
+	cipher_ok ( &des_test2_52 );
+	cipher_ok ( &des_test2_53 );
+	cipher_ok ( &des_test2_54 );
+	cipher_ok ( &des_test2_55 );
+	cipher_ok ( &des_test2_56 );
+	cipher_ok ( &des_test2_57 );
+	cipher_ok ( &des_test2_58 );
+	cipher_ok ( &des_test2_59 );
+	cipher_ok ( &des_test2_60 );
+	cipher_ok ( &des_test2_61 );
+	cipher_ok ( &des_test2_62 );
+	cipher_ok ( &des_test2_63 );
+	cipher_ok ( &des_test2_64 );
+
+	/* Test 3: Data permutation tests */
+	cipher_ok ( &des_test3_1 );
+	cipher_ok ( &des_test3_2 );
+	cipher_ok ( &des_test3_3 );
+	cipher_ok ( &des_test3_4 );
+	cipher_ok ( &des_test3_5 );
+	cipher_ok ( &des_test3_6 );
+	cipher_ok ( &des_test3_7 );
+	cipher_ok ( &des_test3_8 );
+	cipher_ok ( &des_test3_9 );
+	cipher_ok ( &des_test3_10 );
+	cipher_ok ( &des_test3_11 );
+	cipher_ok ( &des_test3_12 );
+	cipher_ok ( &des_test3_13 );
+	cipher_ok ( &des_test3_14 );
+	cipher_ok ( &des_test3_15 );
+	cipher_ok ( &des_test3_16 );
+	cipher_ok ( &des_test3_17 );
+	cipher_ok ( &des_test3_18 );
+	cipher_ok ( &des_test3_19 );
+	cipher_ok ( &des_test3_20 );
+	cipher_ok ( &des_test3_21 );
+	cipher_ok ( &des_test3_22 );
+	cipher_ok ( &des_test3_23 );
+	cipher_ok ( &des_test3_24 );
+	cipher_ok ( &des_test3_25 );
+	cipher_ok ( &des_test3_26 );
+	cipher_ok ( &des_test3_27 );
+	cipher_ok ( &des_test3_28 );
+	cipher_ok ( &des_test3_29 );
+	cipher_ok ( &des_test3_30 );
+	cipher_ok ( &des_test3_31 );
+	cipher_ok ( &des_test3_32 );
+
+	/* Test 4: Key permutation tests */
+	cipher_ok ( &des_test4_1 );
+	cipher_ok ( &des_test4_2 );
+	cipher_ok ( &des_test4_3 );
+	cipher_ok ( &des_test4_4 );
+	cipher_ok ( &des_test4_5 );
+	cipher_ok ( &des_test4_6 );
+	cipher_ok ( &des_test4_7 );
+	cipher_ok ( &des_test4_8 );
+	cipher_ok ( &des_test4_9 );
+	cipher_ok ( &des_test4_10 );
+	cipher_ok ( &des_test4_11 );
+	cipher_ok ( &des_test4_12 );
+	cipher_ok ( &des_test4_13 );
+	cipher_ok ( &des_test4_14 );
+	cipher_ok ( &des_test4_15 );
+	cipher_ok ( &des_test4_16 );
+	cipher_ok ( &des_test4_17 );
+	cipher_ok ( &des_test4_18 );
+	cipher_ok ( &des_test4_19 );
+	cipher_ok ( &des_test4_20 );
+	cipher_ok ( &des_test4_21 );
+	cipher_ok ( &des_test4_22 );
+	cipher_ok ( &des_test4_23 );
+	cipher_ok ( &des_test4_24 );
+	cipher_ok ( &des_test4_25 );
+	cipher_ok ( &des_test4_26 );
+	cipher_ok ( &des_test4_27 );
+	cipher_ok ( &des_test4_28 );
+	cipher_ok ( &des_test4_29 );
+	cipher_ok ( &des_test4_30 );
+	cipher_ok ( &des_test4_31 );
+	cipher_ok ( &des_test4_32 );
+	cipher_ok ( &des_test4_33 );
+	cipher_ok ( &des_test4_34 );
+	cipher_ok ( &des_test4_35 );
+	cipher_ok ( &des_test4_36 );
+	cipher_ok ( &des_test4_37 );
+	cipher_ok ( &des_test4_38 );
+	cipher_ok ( &des_test4_39 );
+	cipher_ok ( &des_test4_40 );
+	cipher_ok ( &des_test4_41 );
+	cipher_ok ( &des_test4_42 );
+	cipher_ok ( &des_test4_43 );
+	cipher_ok ( &des_test4_44 );
+	cipher_ok ( &des_test4_45 );
+	cipher_ok ( &des_test4_46 );
+	cipher_ok ( &des_test4_47 );
+	cipher_ok ( &des_test4_48 );
+	cipher_ok ( &des_test4_49 );
+	cipher_ok ( &des_test4_50 );
+	cipher_ok ( &des_test4_51 );
+	cipher_ok ( &des_test4_52 );
+	cipher_ok ( &des_test4_53 );
+	cipher_ok ( &des_test4_54 );
+	cipher_ok ( &des_test4_55 );
+	cipher_ok ( &des_test4_56 );
+
+	/* Test 5: S-box tests */
+	cipher_ok ( &des_test5_1 );
+	cipher_ok ( &des_test5_2 );
+	cipher_ok ( &des_test5_3 );
+	cipher_ok ( &des_test5_4 );
+	cipher_ok ( &des_test5_5 );
+	cipher_ok ( &des_test5_6 );
+	cipher_ok ( &des_test5_7 );
+	cipher_ok ( &des_test5_8 );
+	cipher_ok ( &des_test5_9 );
+	cipher_ok ( &des_test5_10 );
+	cipher_ok ( &des_test5_11 );
+	cipher_ok ( &des_test5_12 );
+	cipher_ok ( &des_test5_13 );
+	cipher_ok ( &des_test5_14 );
+	cipher_ok ( &des_test5_15 );
+	cipher_ok ( &des_test5_16 );
+	cipher_ok ( &des_test5_17 );
+	cipher_ok ( &des_test5_18 );
+	cipher_ok ( &des_test5_19 );
+
+	/* Multi-block tests */
+	cipher_ok ( &des_unofficial_ecb );
+	cipher_ok ( &des_unofficial_cbc );
+
+	/* Speed tests */
+	DBG ( "DES-ECB encryption required %ld cycles per byte\n",
+	      cipher_cost_encrypt ( &des_ecb_algorithm, 8 ) );
+	DBG ( "DES-ECB decryption required %ld cycles per byte\n",
+	      cipher_cost_decrypt ( &des_ecb_algorithm, 8 ) );
+	DBG ( "DES-CBC encryption required %ld cycles per byte\n",
+	      cipher_cost_encrypt ( &des_cbc_algorithm, 8 ) );
+	DBG ( "DES-CBC decryption required %ld cycles per byte\n",
+	      cipher_cost_decrypt ( &des_cbc_algorithm, 8 ) );
+}
+
+/** DES self-test */
+struct self_test des_test __self_test = {
+	.name = "des",
+	.exec = des_test_exec,
+};
diff --git a/src/tests/tests.c b/src/tests/tests.c
index 41c199c0b..282d2eb65 100644
--- a/src/tests/tests.c
+++ b/src/tests/tests.c
@@ -82,3 +82,4 @@ REQUIRE_OBJECT ( dhe_test );
 REQUIRE_OBJECT ( gcm_test );
 REQUIRE_OBJECT ( nap_test );
 REQUIRE_OBJECT ( x25519_test );
+REQUIRE_OBJECT ( des_test );

From a846c4ccfc7db212dff792e081991df17268b4d5 Mon Sep 17 00:00:00 2001
From: Joseph Wong <joseph.wong@broadcom.com>
Date: Wed, 7 Feb 2024 14:14:25 -0800
Subject: [PATCH 071/237] [bnxt] Add support for BCM957608

Add support for BCM957608 device.  Add support for additional link
speeds supported by BCM957608.

Signed-off-by: Joseph Wong <joseph.wong@broadcom.com>
---
 src/drivers/net/bnxt/bnxt.c     | 375 +++++++++++++++++++++++++-------
 src/drivers/net/bnxt/bnxt.h     | 211 +++++++++++++++++-
 src/drivers/net/bnxt/bnxt_dbg.h |   6 +-
 src/drivers/net/bnxt/bnxt_hsi.h | 229 ++++++++++++++++++-
 4 files changed, 720 insertions(+), 101 deletions(-)

diff --git a/src/drivers/net/bnxt/bnxt.c b/src/drivers/net/bnxt/bnxt.c
index 5dd217971..a99f29ec0 100644
--- a/src/drivers/net/bnxt/bnxt.c
+++ b/src/drivers/net/bnxt/bnxt.c
@@ -76,6 +76,7 @@ static struct pci_device_id bnxt_nics[] = {
 	PCI_ROM( 0x14e4, 0x1801, "14e4-1801", "14e4-1801", 0 ),
 	PCI_ROM( 0x14e4, 0x1804, "14e4-1804", "14e4-1804", 0 ),
 	PCI_ROM( 0x14e4, 0x1752, "14e4-1752", "14e4-1752", 0 ),
+	PCI_ROM( 0x14e4, 0x1760, "14e4-1760", "14e4-1760", 0 ),
 	PCI_ROM( 0x14e4, 0x1800, "14e4-1800", "14e4-1800", 0 ),
 	PCI_ROM( 0x14e4, 0x1803, "14e4-1803", "14e4-1803", 0 ),
 	PCI_ROM( 0x14e4, 0x1806, "14e4-1806", "14e4-1806", BNXT_FLAG_PCI_VF ),
@@ -181,7 +182,7 @@ static void bnxt_set_link ( struct bnxt *bp )
 		netdev_link_down ( bp->dev );
 }
 
-static void thor_db ( struct bnxt *bp, u32 idx, u32 xid, u32 flag )
+static void dev_p5_db ( struct bnxt *bp, u32 idx, u32 xid, u32 flag )
 {
 	void *off;
 	u64  val;
@@ -196,10 +197,28 @@ static void thor_db ( struct bnxt *bp, u32 idx, u32 xid, u32 flag )
 	write64 ( val, off );
 }
 
+static void dev_p7_db ( struct bnxt *bp, u32 idx, u32 xid, u32 flag, u32 epoch, u32 toggle )
+{
+	void *off;
+	u64  val;
+
+	off = ( void * ) ( bp->bar1 );
+
+	val = ( ( u64 )DBC_MSG_XID ( xid, flag ) << 32 ) |
+	        ( u64 )DBC_MSG_IDX ( idx ) |
+	        ( u64 )DBC_MSG_EPCH ( epoch ) |
+	        ( u64 )DBC_MSG_TOGGLE ( toggle );
+	write64 ( val, off );
+}
+
 static void bnxt_db_nq ( struct bnxt *bp )
 {
-	if ( bp->thor )
-		thor_db ( bp, ( u32 )bp->nq.cons_id,
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) )
+		dev_p7_db ( bp, ( u32 )bp->nq.cons_id,
+			 ( u32 )bp->nq_ring_id, DBC_DBC_TYPE_NQ_ARM,
+                         ( u32 )bp->nq.epoch, 0 );
+	else if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5 ) )
+		dev_p5_db ( bp, ( u32 )bp->nq.cons_id,
 			 ( u32 )bp->nq_ring_id, DBC_DBC_TYPE_NQ_ARM );
 	else
 		write32 ( CMPL_DOORBELL_KEY_CMPL, ( bp->bar1 + 0 ) );
@@ -207,8 +226,12 @@ static void bnxt_db_nq ( struct bnxt *bp )
 
 static void bnxt_db_cq ( struct bnxt *bp )
 {
-	if ( bp->thor )
-		thor_db ( bp, ( u32 )bp->cq.cons_id,
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) )
+		dev_p7_db ( bp, ( u32 )bp->cq.cons_id,
+			 ( u32 )bp->cq_ring_id, DBC_DBC_TYPE_CQ_ARMALL,
+                         ( u32 )bp->cq.epoch, ( u32 )bp->nq.toggle );
+	else if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5 ) )
+		dev_p5_db ( bp, ( u32 )bp->cq.cons_id,
 			 ( u32 )bp->cq_ring_id, DBC_DBC_TYPE_CQ_ARMALL );
 	else
 		write32 ( CQ_DOORBELL_KEY_IDX ( bp->cq.cons_id ),
@@ -217,16 +240,22 @@ static void bnxt_db_cq ( struct bnxt *bp )
 
 static void bnxt_db_rx ( struct bnxt *bp, u32 idx )
 {
-	if ( bp->thor )
-		thor_db ( bp, idx, ( u32 )bp->rx_ring_id, DBC_DBC_TYPE_SRQ );
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) )
+		dev_p7_db ( bp, idx, ( u32 )bp->rx_ring_id, DBC_DBC_TYPE_SRQ,
+                         ( u32 )bp->rx.epoch, 0 );
+	else if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5 ) )
+		dev_p5_db ( bp, idx, ( u32 )bp->rx_ring_id, DBC_DBC_TYPE_SRQ );
 	else
 		write32 ( RX_DOORBELL_KEY_RX | idx, ( bp->bar1 + 0 ) );
 }
 
 static void bnxt_db_tx ( struct bnxt *bp, u32 idx )
 {
-	if ( bp->thor )
-		thor_db ( bp, idx, ( u32 )bp->tx_ring_id, DBC_DBC_TYPE_SQ );
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) )
+		dev_p7_db ( bp, idx, ( u32 )bp->tx_ring_id, DBC_DBC_TYPE_SQ,
+                         ( u32 )bp->tx.epoch, 0 );
+	else if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5 ) )
+		dev_p5_db ( bp, idx, ( u32 )bp->tx_ring_id, DBC_DBC_TYPE_SQ );
 	else
 		write32 ( ( u32 ) ( TX_DOORBELL_KEY_TX | idx ),
 			( bp->bar1 + 0 ) );
@@ -253,6 +282,31 @@ static u16 bnxt_get_pkt_vlan ( char *src )
 	return 0;
 }
 
+static u16 bnxt_get_rx_vlan ( struct rx_pkt_cmpl *rx_cmp, struct rx_pkt_cmpl_hi *rx_cmp_hi )
+{
+	struct rx_pkt_v3_cmpl *rx_cmp_v3 = ( struct rx_pkt_v3_cmpl * )rx_cmp;
+	struct rx_pkt_v3_cmpl_hi *rx_cmp_hi_v3 = ( struct rx_pkt_v3_cmpl_hi * )rx_cmp_hi;
+	u16 rx_vlan;
+
+	/* Get VLAN ID from RX completion ring */
+	if ( ( rx_cmp_v3->flags_type & RX_PKT_V3_CMPL_TYPE_MASK ) ==
+	     RX_PKT_V3_CMPL_TYPE_RX_L2_V3 ) {
+		if ( rx_cmp_hi_v3->flags2 & RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_ACT_REC_PTR )
+			rx_vlan = ( rx_cmp_hi_v3->metadata0 &
+				RX_PKT_V3_CMPL_HI_METADATA0_VID_MASK );
+		else
+			rx_vlan = 0;
+	} else {
+	        if ( rx_cmp_hi->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN )
+			rx_vlan = ( rx_cmp_hi->metadata &
+				RX_PKT_CMPL_METADATA_VID_MASK );
+	        else
+			rx_vlan = 0;
+	}
+
+	return rx_vlan;
+}
+
 int bnxt_vlan_drop ( struct bnxt *bp, u16 rx_vlan )
 {
 	if ( rx_vlan ) {
@@ -382,6 +436,9 @@ int bnxt_post_rx_buffers ( struct bnxt *bp )
 			}
 		}
 		cons_id = NEXT_IDX ( cons_id, bp->rx.ring_cnt );
+		/* If the ring has wrapped, flip the epoch bit */
+		if ( iob_idx > cons_id )
+		        bp->rx.epoch ^= 1;
 		bp->rx.iob_cnt++;
 	}
 
@@ -396,14 +453,21 @@ int bnxt_post_rx_buffers ( struct bnxt *bp )
 }
 
 u8 bnxt_rx_drop ( struct bnxt *bp, struct io_buffer *iob,
+	struct rx_pkt_cmpl *rx_cmp,
 	struct rx_pkt_cmpl_hi *rx_cmp_hi, u16 rx_len )
 {
+	struct rx_pkt_v3_cmpl *rx_cmp_v3 = ( struct rx_pkt_v3_cmpl * )rx_cmp;
+	struct rx_pkt_v3_cmpl_hi  *rx_cmp_hi_v3 = ( struct rx_pkt_v3_cmpl_hi * )rx_cmp_hi;
 	u8  *rx_buf = ( u8 * )iob->data;
 	u16 err_flags, rx_vlan;
 	u8  ignore_chksum_err = 0;
 	int i;
 
-	err_flags = rx_cmp_hi->errors_v2 >> RX_PKT_CMPL_ERRORS_BUFFER_ERROR_SFT;
+	if ( ( rx_cmp_v3->flags_type & RX_PKT_V3_CMPL_TYPE_MASK ) ==
+	     RX_PKT_V3_CMPL_TYPE_RX_L2_V3 ) {
+	        err_flags = rx_cmp_hi_v3->errors_v2 >> RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_SFT;
+	} else
+	        err_flags = rx_cmp_hi->errors_v2 >> RX_PKT_CMPL_ERRORS_BUFFER_ERROR_SFT;
 	if ( rx_cmp_hi->errors_v2 == 0x20 || rx_cmp_hi->errors_v2 == 0x21 )
 		ignore_chksum_err = 1;
 
@@ -423,13 +487,7 @@ u8 bnxt_rx_drop ( struct bnxt *bp, struct io_buffer *iob,
 		return 2;
 	}
 
-	/* Get VLAN ID from RX completion ring */
-	if ( rx_cmp_hi->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN )
-		rx_vlan = ( rx_cmp_hi->metadata &
-				RX_PKT_CMPL_METADATA_VID_MASK );
-	else
-		rx_vlan = 0;
-
+	rx_vlan = bnxt_get_rx_vlan ( rx_cmp, rx_cmp_hi );
 	dbg_rx_vlan ( bp, rx_cmp_hi->metadata, rx_cmp_hi->flags2, rx_vlan );
 	if ( bnxt_vlan_drop ( bp, rx_vlan ) ) {
 		bp->rx.drop_vlan++;
@@ -449,10 +507,11 @@ static void bnxt_adv_cq_index ( struct bnxt *bp, u16 cnt )
 	u16 cons_id;
 
 	cons_id = bp->cq.cons_id + cnt;
-	if ( cons_id >= MAX_CQ_DESC_CNT ) {
+	if ( cons_id >= bp->cq.ring_cnt) {
 		/* Toggle completion bit when the ring wraps. */
 		bp->cq.completion_bit ^= 1;
-		cons_id = cons_id - MAX_CQ_DESC_CNT;
+		bp->cq.epoch ^= 1;
+		cons_id = cons_id - bp->cq.ring_cnt;
 	}
 	bp->cq.cons_id = cons_id;
 }
@@ -466,7 +525,7 @@ void bnxt_rx_process ( struct net_device *dev, struct bnxt *bp,
 
 	dump_rx_bd ( rx_cmp, rx_cmp_hi, desc_idx );
 	assert ( !iob );
-	drop = bnxt_rx_drop ( bp, iob, rx_cmp_hi, rx_cmp->len );
+	drop = bnxt_rx_drop ( bp, iob, rx_cmp, rx_cmp_hi, rx_cmp->len );
 	dbg_rxp ( iob->data, rx_cmp->len, drop );
 	if ( drop )
 		netdev_rx_err ( dev, iob, -EINVAL );
@@ -531,12 +590,17 @@ void bnxt_mm_nic ( struct bnxt *bp )
 	memset ( bp->nq.bd_virt, 0, NQ_RING_BUFFER_SIZE );
 	bp->nq.cons_id		= 0;
 	bp->nq.completion_bit	= 0x1;
+	bp->nq.epoch            = 0;
+	bp->nq.toggle           = 0;
 	bp->cq.cons_id		= 0;
 	bp->cq.completion_bit	= 0x1;
+	bp->cq.epoch            = 0;
 	bp->tx.prod_id		= 0;
 	bp->tx.cons_id		= 0;
+	bp->tx.epoch            = 0;
 	bp->rx.cons_id		= 0;
 	bp->rx.iob_cnt		= 0;
+	bp->rx.epoch            = 0;
 
 	bp->link_status		= STATUS_LINK_DOWN;
 	bp->wait_link_timeout	= LINK_DEFAULT_TIMEOUT;
@@ -724,8 +788,14 @@ static int bnxt_hwrm_ver_get ( struct bnxt *bp )
 	bp->hwrm_max_ext_req_len = resp->max_ext_req_len;
 	if ( ( bp->chip_num == CHIP_NUM_57508 ) ||
 	     ( bp->chip_num == CHIP_NUM_57504 ) ||
-	     ( bp->chip_num == CHIP_NUM_57502 ) )
-		bp->thor = 1;
+	     ( bp->chip_num == CHIP_NUM_57502 ) ) {
+		FLAG_SET ( bp->flags, BNXT_FLAG_IS_CHIP_P5 );
+		FLAG_SET ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS );
+	}
+	if ( bp->chip_num == CHIP_NUM_57608 ) {
+		FLAG_SET ( bp->flags, BNXT_FLAG_IS_CHIP_P7 );
+		FLAG_SET ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS );
+	}
 	dbg_fw_ver ( resp, bp->hwrm_cmd_timeout );
 	return STATUS_SUCCESS;
 }
@@ -917,6 +987,30 @@ static int bnxt_hwrm_func_qcfg_req ( struct bnxt *bp )
 	return STATUS_SUCCESS;
 }
 
+static int bnxt_hwrm_port_phy_qcaps_req ( struct bnxt *bp )
+{
+	u16 cmd_len = ( u16 )sizeof ( struct hwrm_port_phy_qcaps_input );
+	struct hwrm_port_phy_qcaps_input *req;
+	struct hwrm_port_phy_qcaps_output *resp;
+	int rc;
+
+	DBGP ( "%s\n", __func__ );
+
+	req = ( struct hwrm_port_phy_qcaps_input * )bp->hwrm_addr_req;
+	resp = ( struct hwrm_port_phy_qcaps_output * )bp->hwrm_addr_resp;
+	hwrm_init ( bp, ( void * )req, ( u16 )HWRM_PORT_PHY_QCAPS, cmd_len );
+	rc = wait_resp ( bp, bp->hwrm_cmd_timeout, cmd_len, __func__ );
+	if ( rc ) {
+		DBGP ( "-s %s (  ): Failed\n", __func__ );
+		return STATUS_FAILURE;
+	}
+
+	if ( resp->flags2 & PORT_PHY_QCAPS_RESP_FLAGS2_SPEEDS2_SUPPORTED )
+                FLAG_SET ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 );
+
+	return STATUS_SUCCESS;
+}
+
 static int bnxt_hwrm_func_reset_req ( struct bnxt *bp )
 {
 	u16 cmd_len = ( u16 )sizeof ( struct hwrm_func_reset_input );
@@ -944,7 +1038,7 @@ static int bnxt_hwrm_func_cfg_req ( struct bnxt *bp )
 	hwrm_init ( bp, ( void * )req, ( u16 )HWRM_FUNC_CFG, cmd_len );
 	req->fid = ( u16 )HWRM_NA_SIGNATURE;
 	bnxt_hwrm_assign_resources ( bp );
-	if ( bp->thor ) {
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) {
 		req->enables |= ( FUNC_CFG_REQ_ENABLES_NUM_MSIX |
 				  FUNC_CFG_REQ_ENABLES_NUM_VNICS |
 				  FUNC_CFG_REQ_ENABLES_EVB_MODE );
@@ -1011,7 +1105,7 @@ static int bnxt_hwrm_set_async_event ( struct bnxt *bp )
 	u16 idx;
 
 	DBGP ( "%s\n", __func__ );
-	if ( bp->thor )
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) )
 		idx = bp->nq_ring_id;
 	else
 		idx = bp->cq_ring_id;
@@ -1162,6 +1256,10 @@ static int bnxt_hwrm_port_phy_qcfg ( struct bnxt *bp, u16 idx )
 	if ( idx & SUPPORT_SPEEDS )
 		bp->support_speeds = resp->support_speeds;
 
+	if ( idx & SUPPORT_SPEEDS2 )
+                if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) )
+		        bp->auto_link_speeds2_mask = resp->auto_link_speeds2;
+
 	if ( idx & DETECT_MEDIA )
 		bp->media_detect = resp->module_status;
 
@@ -1201,22 +1299,24 @@ static int bnxt_get_link_speed ( struct bnxt *bp )
 	u32 *ptr32 = ( u32 * )bp->hwrm_addr_dma;
 
 	DBGP ( "%s\n", __func__ );
-	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
-		( u16 )LINK_SPEED_DRV_NUM,
-		1, ( u16 )bp->port_idx ) != STATUS_SUCCESS )
-		return STATUS_FAILURE;
-	bp->link_set = SET_LINK ( *ptr32, SPEED_DRV_MASK, SPEED_DRV_SHIFT );
+	if ( ! ( FLAG_TEST (bp->flags, BNXT_FLAG_IS_CHIP_P7 ) ) ) {
+	        test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
+		        ( u16 )LINK_SPEED_DRV_NUM,
+		        1, ( u16 )bp->port_idx ) != STATUS_SUCCESS )
+		        return STATUS_FAILURE;
+	        bp->link_set = SET_LINK ( *ptr32, SPEED_DRV_MASK, SPEED_DRV_SHIFT );
+	        test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
+		        ( u16 )D3_LINK_SPEED_FW_NUM, 1,
+		        ( u16 )bp->port_idx ) != STATUS_SUCCESS )
+		        return STATUS_FAILURE;
+	        bp->link_set |= SET_LINK ( *ptr32, D3_SPEED_FW_MASK,
+				D3_SPEED_FW_SHIFT );
+	}
 	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
 		( u16 )LINK_SPEED_FW_NUM,
 		1, ( u16 )bp->port_idx ) != STATUS_SUCCESS )
 		return STATUS_FAILURE;
 	bp->link_set |= SET_LINK ( *ptr32, SPEED_FW_MASK, SPEED_FW_SHIFT );
-	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
-		 ( u16 )D3_LINK_SPEED_FW_NUM, 1,
-		 ( u16 )bp->port_idx ) != STATUS_SUCCESS )
-		return STATUS_FAILURE;
-	bp->link_set |= SET_LINK ( *ptr32, D3_SPEED_FW_MASK,
-			D3_SPEED_FW_SHIFT );
 	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 1,
 		 ( u16 )PORT_CFG_LINK_SETTINGS_MEDIA_AUTO_DETECT_NUM,
 		1, ( u16 )bp->port_idx ) != STATUS_SUCCESS )
@@ -1224,32 +1324,51 @@ static int bnxt_get_link_speed ( struct bnxt *bp )
 	bp->link_set |= SET_LINK ( *ptr32,
 		MEDIA_AUTO_DETECT_MASK, MEDIA_AUTO_DETECT_SHIFT );
 
-	switch ( bp->link_set & LINK_SPEED_DRV_MASK ) {
-	case LINK_SPEED_DRV_1G:
+	/* Use LINK_SPEED_FW_xxx which is valid for CHIP_P7 and earlier devices */
+	switch ( bp->link_set & LINK_SPEED_FW_MASK ) {
+	case LINK_SPEED_FW_1G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_1000MBPS );
 		break;
-	case LINK_SPEED_DRV_2_5G:
+	case LINK_SPEED_FW_2_5G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_2500MBPS );
 		break;
-	case LINK_SPEED_DRV_10G:
+	case LINK_SPEED_FW_10G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_10GBPS );
 		break;
-	case LINK_SPEED_DRV_25G:
+	case LINK_SPEED_FW_25G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_25GBPS );
 		break;
-	case LINK_SPEED_DRV_40G:
+	case LINK_SPEED_FW_40G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_40GBPS );
 		break;
-	case LINK_SPEED_DRV_50G:
+	case LINK_SPEED_FW_50G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_50GBPS );
 		break;
-	case LINK_SPEED_DRV_100G:
+	case LINK_SPEED_FW_50G_PAM4:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_50PAM4GBPS );
+		break;
+	case LINK_SPEED_FW_100G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_100GBPS );
 		break;
-	case LINK_SPEED_DRV_200G:
+	case LINK_SPEED_FW_100G_PAM4:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_100PAM4GBPS );
+		break;
+	case LINK_SPEED_FW_100G_PAM4_112:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_100PAM4_112GBPS );
+		break;
+	case LINK_SPEED_FW_200G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_200GBPS );
 		break;
-	case LINK_SPEED_DRV_AUTONEG:
+	case LINK_SPEED_FW_200G_PAM4_112:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_200PAM4_112GBPS );
+		break;
+	case LINK_SPEED_FW_400G_PAM4:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_400PAM4GBPS );
+		break;
+	case LINK_SPEED_FW_400G_PAM4_112:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_400PAM4_112GBPS );
+		break;
+	case LINK_SPEED_FW_AUTONEG:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_AUTONEG );
 		break;
 	default:
@@ -1268,27 +1387,29 @@ static int bnxt_get_vlan ( struct bnxt *bp )
 	if ( bp->vf )
 		return STATUS_SUCCESS;
 
-	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 1,
-		 ( u16 )FUNC_CFG_PRE_BOOT_MBA_VLAN_NUM, 1,
-		 ( u16 )bp->ordinal_value ) != STATUS_SUCCESS )
-		return STATUS_FAILURE;
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) ) ) {
+		test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 1,
+			 ( u16 )FUNC_CFG_PRE_BOOT_MBA_VLAN_NUM, 1,
+			 ( u16 )bp->ordinal_value ) != STATUS_SUCCESS )
+			return STATUS_FAILURE;
 
-	bp->mba_cfg2 = SET_MBA ( *ptr32, VLAN_MASK, VLAN_SHIFT );
-	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 16,
-		 ( u16 )FUNC_CFG_PRE_BOOT_MBA_VLAN_VALUE_NUM, 1,
-		 ( u16 )bp->ordinal_value ) != STATUS_SUCCESS )
-		return STATUS_FAILURE;
+		bp->mba_cfg2 = SET_MBA ( *ptr32, VLAN_MASK, VLAN_SHIFT );
+		test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 16,
+			 ( u16 )FUNC_CFG_PRE_BOOT_MBA_VLAN_VALUE_NUM, 1,
+			 ( u16 )bp->ordinal_value ) != STATUS_SUCCESS )
+			return STATUS_FAILURE;
 
-	bp->mba_cfg2 |= SET_MBA ( *ptr32, VLAN_VALUE_MASK, VLAN_VALUE_SHIFT );
-	if ( bp->mba_cfg2 & FUNC_CFG_PRE_BOOT_MBA_VLAN_ENABLED )
-		bp->vlan_id = bp->mba_cfg2 & VLAN_VALUE_MASK;
-	else
-		bp->vlan_id = 0;
+		bp->mba_cfg2 |= SET_MBA ( *ptr32, VLAN_VALUE_MASK, VLAN_VALUE_SHIFT );
+		if ( bp->mba_cfg2 & FUNC_CFG_PRE_BOOT_MBA_VLAN_ENABLED )
+			bp->vlan_id = bp->mba_cfg2 & VLAN_VALUE_MASK;
+		else
+			bp->vlan_id = 0;
 
-	if ( bp->mba_cfg2 & FUNC_CFG_PRE_BOOT_MBA_VLAN_ENABLED )
-		DBGP ( "VLAN MBA Enabled ( %d )\n",
-			( bp->mba_cfg2 & VLAN_VALUE_MASK ) );
+		if ( bp->mba_cfg2 & FUNC_CFG_PRE_BOOT_MBA_VLAN_ENABLED )
+			DBGP ( "VLAN MBA Enabled ( %d )\n",
+				( bp->mba_cfg2 & VLAN_VALUE_MASK ) );
 
+	}
 	return STATUS_SUCCESS;
 }
 
@@ -1298,7 +1419,7 @@ static int bnxt_hwrm_backing_store_qcfg ( struct bnxt *bp )
 	struct hwrm_func_backing_store_qcfg_input *req;
 
 	DBGP ( "%s\n", __func__ );
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 
 	req = ( struct hwrm_func_backing_store_qcfg_input * )bp->hwrm_addr_req;
@@ -1313,7 +1434,7 @@ static int bnxt_hwrm_backing_store_cfg ( struct bnxt *bp )
 	struct hwrm_func_backing_store_cfg_input *req;
 
 	DBGP ( "%s\n", __func__ );
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 
 	req = ( struct hwrm_func_backing_store_cfg_input * )bp->hwrm_addr_req;
@@ -1332,7 +1453,7 @@ static int bnxt_hwrm_queue_qportcfg ( struct bnxt *bp )
 	int rc;
 
 	DBGP ( "%s\n", __func__ );
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 
 	req = ( struct hwrm_queue_qportcfg_input * )bp->hwrm_addr_req;
@@ -1372,7 +1493,10 @@ static int bnxt_hwrm_port_phy_cfg ( struct bnxt *bp )
 	u32 flags;
 	u32 enables = 0;
 	u16 force_link_speed = 0;
+	u16 force_link_speeds2 = 0;
+	u16 force_pam4_link_speed = 0;
 	u16 auto_link_speed_mask = 0;
+	u16 auto_link_speeds2_mask = 0;
 	u8  auto_mode = 0;
 	u8  auto_pause = 0;
 	u8  auto_duplex = 0;
@@ -1387,34 +1511,111 @@ static int bnxt_hwrm_port_phy_cfg ( struct bnxt *bp )
 		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_1GB;
 		break;
 	case MEDIUM_SPEED_10GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_10GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB;
+		}
 		break;
 	case MEDIUM_SPEED_25GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_25GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB;
+		}
 		break;
 	case MEDIUM_SPEED_40GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_40GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB;
+		}
 		break;
 	case MEDIUM_SPEED_50GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB;
+		}
+		break;
+	case MEDIUM_SPEED_50PAM4GBPS:
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB_PAM4_56;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_pam4_link_speed = PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_50GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED;
+		}
 		break;
 	case MEDIUM_SPEED_100GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB;
+		}
+		break;
+	case MEDIUM_SPEED_100PAM4GBPS:
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_56;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_pam4_link_speed = PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_100GB;
+                        enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED;
+		}
+		break;
+	case MEDIUM_SPEED_100PAM4_112GBPS:
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_112;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		}
 		break;
 	case MEDIUM_SPEED_200GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_200GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_56;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+			force_pam4_link_speed = PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED;
+		}
 		break;
+	case MEDIUM_SPEED_200PAM4_112GBPS:
+                if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+                        force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_112;
+                        enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+                }
+                break;
+	case MEDIUM_SPEED_400PAM4GBPS:
+                if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+                        force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_56;
+                        enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+                }
+		break;
+	case MEDIUM_SPEED_400PAM4_112GBPS:
+                if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+                        force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112;
+                        enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+                }
+                break;
 	default:
 		auto_mode = PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
 		flags &= ~PORT_PHY_CFG_REQ_FLAGS_FORCE;
 		enables |= PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE |
-			PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK |
 			PORT_PHY_CFG_REQ_ENABLES_AUTO_DUPLEX |
 			PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE;
+		if ( FLAG_TEST (bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) )
+			enables |= PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK;
+		else
+			enables |= PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK;
 		auto_pause = PORT_PHY_CFG_REQ_AUTO_PAUSE_TX |
 				PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
 		auto_duplex = PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH;
 		auto_link_speed_mask = bp->support_speeds;
+		auto_link_speeds2_mask = bp->auto_link_speeds2_mask;
 		break;
 	}
 
@@ -1423,10 +1624,13 @@ static int bnxt_hwrm_port_phy_cfg ( struct bnxt *bp )
 	req->enables = enables;
 	req->port_id = bp->port_idx;
 	req->force_link_speed = force_link_speed;
+	req->force_pam4_link_speed = force_pam4_link_speed;
+	req->force_link_speeds2 = force_link_speeds2;
 	req->auto_mode = auto_mode;
 	req->auto_duplex = auto_duplex;
 	req->auto_pause = auto_pause;
 	req->auto_link_speed_mask = auto_link_speed_mask;
+	req->auto_link_speeds2_mask = auto_link_speeds2_mask;
 
 	return wait_resp ( bp, bp->hwrm_cmd_timeout, cmd_len, __func__ );
 }
@@ -1565,7 +1769,7 @@ static int bnxt_hwrm_ring_alloc_grp ( struct bnxt *bp )
 	int rc;
 
 	DBGP ( "%s\n", __func__ );
-	if ( bp->thor )
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) )
 		return STATUS_SUCCESS;
 
 	req = ( struct hwrm_ring_grp_alloc_input * )bp->hwrm_addr_req;
@@ -1616,7 +1820,7 @@ static int bnxt_hwrm_ring_alloc ( struct bnxt *bp, u8 type )
 	switch ( type ) {
 	case RING_ALLOC_REQ_RING_TYPE_NQ:
 		req->page_size  = LM_PAGE_BITS ( 12 );
-		req->int_mode   = BNXT_CQ_INTR_MODE ( bp->vf );
+		req->int_mode   = BNXT_CQ_INTR_MODE ( ( (FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7) ) || bp->vf ) );
 		req->length 	= ( u32 )bp->nq.ring_cnt;
 		req->logical_id = 0xFFFF; // Required value for Thor FW?
 		req->page_tbl_addr = virt_to_bus ( bp->nq.bd_virt );
@@ -1626,7 +1830,7 @@ static int bnxt_hwrm_ring_alloc ( struct bnxt *bp, u8 type )
 		req->int_mode  = BNXT_CQ_INTR_MODE ( bp->vf );
 		req->length    = ( u32 )bp->cq.ring_cnt;
 		req->page_tbl_addr = virt_to_bus ( bp->cq.bd_virt );
-		if ( !bp->thor )
+	        if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 			break;
 		req->enables = RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID;
 		req->nq_ring_id = bp->nq_ring_id;
@@ -1648,7 +1852,7 @@ static int bnxt_hwrm_ring_alloc ( struct bnxt *bp, u8 type )
 		req->stat_ctx_id = ( u32 )STAT_CTX_ID;
 		req->cmpl_ring_id  = bp->cq_ring_id;
 		req->page_tbl_addr = virt_to_bus ( bp->rx.bd_virt );
-		if ( !bp->thor )
+	        if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 			break;
 		req->queue_id    = ( u16 )RX_RING_QID;
 		req->rx_buf_size = MAX_ETHERNET_PACKET_BUFFER_SIZE;
@@ -1744,7 +1948,7 @@ static int bnxt_hwrm_ring_free_rx ( struct bnxt *bp )
 
 static int bnxt_hwrm_ring_alloc_nq ( struct bnxt *bp )
 {
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 	return bnxt_hwrm_ring_alloc ( bp, RING_ALLOC_REQ_RING_TYPE_NQ );
 }
@@ -1753,7 +1957,7 @@ static int bnxt_hwrm_ring_free_nq ( struct bnxt *bp )
 {
 	int ret = STATUS_SUCCESS;
 
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 
 	DBGP ( "%s\n", __func__ );
@@ -1824,7 +2028,7 @@ static int bnxt_hwrm_vnic_cfg ( struct bnxt *bp )
 	req->enables = VNIC_CFG_REQ_ENABLES_MRU;
 	req->mru	 = bp->mtu;
 
-	if ( bp->thor ) {
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) {
 		req->enables |= ( VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
 				VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID );
 		req->default_rx_ring_id   = bp->rx_ring_id;
@@ -1878,6 +2082,7 @@ hwrm_func_t bring_up_chip[] = {
 	bnxt_hwrm_backing_store_cfg,	/* HWRM_FUNC_BACKING_STORE_CFG  */
 	bnxt_hwrm_backing_store_qcfg,	/* HWRM_FUNC_BACKING_STORE_QCFG	*/
 	bnxt_hwrm_func_resource_qcaps,	/* HWRM_FUNC_RESOURCE_QCAPS	*/
+	bnxt_hwrm_port_phy_qcaps_req,	/* HWRM_PORT_PHY_QCAPS	*/
 	bnxt_hwrm_func_qcfg_req,	/* HWRM_FUNC_QCFG		*/
 	bnxt_get_vlan,			/* HWRM_NVM_GET_VARIABLE - vlan	*/
 	bnxt_hwrm_port_mac_cfg,		/* HWRM_PORT_MAC_CFG		*/
@@ -1970,6 +2175,9 @@ static int bnxt_tx ( struct net_device *dev, struct io_buffer *iob )
 	bp->tx.iob[entry] = iob;
 	bnxt_set_txq ( bp, entry, mapping, len );
 	entry = NEXT_IDX ( entry, bp->tx.ring_cnt );
+	/* If the ring has wrapped, toggle the epoch bit */
+	if ( bp->tx.prod_id > entry )
+		bp->tx.epoch ^= 1;
 	dump_tx_pkt ( ( u8 * )iob->data, len, bp->tx.prod_id );
 	/* Packets are ready, update Tx producer idx local and on card. */
 	bnxt_db_tx ( bp, ( u32 )entry );
@@ -1988,6 +2196,7 @@ static void bnxt_adv_nq_index ( struct bnxt *bp, u16 cnt )
 	if ( cons_id >= bp->nq.ring_cnt ) {
 		/* Toggle completion bit when the ring wraps. */
 		bp->nq.completion_bit ^= 1;
+		bp->nq.epoch ^= 1;
 		cons_id = cons_id - bp->nq.ring_cnt;
 	}
 	bp->nq.cons_id = cons_id;
@@ -2028,7 +2237,7 @@ static void bnxt_service_cq ( struct net_device *dev )
 
 		cq_type = cmp->type & CMPL_BASE_TYPE_MASK;
 		dump_evt ( ( u8 * )cmp, cq_type, bp->cq.cons_id, 0 );
-		dump_cq ( cmp, bp->cq.cons_id );
+		dump_cq ( cmp, bp->cq.cons_id, bp->nq.toggle );
 
 		switch ( cq_type ) {
 		case CMPL_BASE_TYPE_TX_L2:
@@ -2039,6 +2248,7 @@ static void bnxt_service_cq ( struct net_device *dev )
 			bnxt_adv_cq_index ( bp, 1 );
 			break;
 		case CMPL_BASE_TYPE_RX_L2:
+		case CMPL_BASE_TYPE_RX_L2_V3:
 			done = bnxt_rx_complete ( dev,
 				( struct rx_pkt_cmpl * )cmp );
 			break;
@@ -2065,7 +2275,7 @@ static void bnxt_service_nq ( struct net_device *dev )
 	int done = SERVICE_NEXT_NQ_BD;
 	u32 nq_type;
 
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return;
 
 	while ( done == SERVICE_NEXT_NQ_BD ) {
@@ -2074,6 +2284,7 @@ static void bnxt_service_nq ( struct net_device *dev )
 		if ( ( nqp->v & NQ_CN_V ) ^ bp->nq.completion_bit )
 			break;
 		nq_type = ( nqp->type & NQ_CN_TYPE_MASK );
+		bp->nq.toggle = ( ( nqp->type & NQ_CN_TOGGLE_MASK ) >> NQ_CN_TOGGLE_SFT );
 		dump_evt ( ( u8 * )nqp, nq_type, bp->nq.cons_id, 1 );
 		dump_nq ( nqp, bp->nq.cons_id );
 
@@ -2098,8 +2309,8 @@ static void bnxt_service_nq ( struct net_device *dev )
 static void bnxt_poll ( struct net_device *dev )
 {
 	mb (  );
-	bnxt_service_cq ( dev );
 	bnxt_service_nq ( dev );
+	bnxt_service_cq ( dev );
 }
 
 static void bnxt_close ( struct net_device *dev )
diff --git a/src/drivers/net/bnxt/bnxt.h b/src/drivers/net/bnxt/bnxt.h
index cf2dea8bc..8c8a33282 100644
--- a/src/drivers/net/bnxt/bnxt.h
+++ b/src/drivers/net/bnxt/bnxt.h
@@ -52,6 +52,10 @@ union dma_addr64_t {
 #define BNXT_FLAG_NPAR_MODE                     0x0010
 #define BNXT_FLAG_ATOMICS_ENABLE                0x0020
 #define BNXT_FLAG_PCI_VF                        0x0040
+#define BNXT_FLAG_LINK_SPEEDS2                  0x0080
+#define BNXT_FLAG_IS_CHIP_P5                    0x0100
+#define BNXT_FLAG_IS_CHIP_P5_PLUS               0x0200
+#define BNXT_FLAG_IS_CHIP_P7                    0x0400
 /*******************************************************************************
  * Status codes.
  ******************************************************************************/
@@ -106,6 +110,12 @@ union dma_addr64_t {
 #define MEDIUM_SPEED_50GBPS                     0x0a00L
 #define MEDIUM_SPEED_100GBPS                    0x0b00L
 #define MEDIUM_SPEED_200GBPS                    0x0c00L
+#define MEDIUM_SPEED_50PAM4GBPS                 0x0d00L
+#define MEDIUM_SPEED_100PAM4GBPS                0x0e00L
+#define MEDIUM_SPEED_100PAM4_112GBPS            0x0f00L
+#define MEDIUM_SPEED_200PAM4_112GBPS            0x1000L
+#define MEDIUM_SPEED_400PAM4GBPS                0x2000L
+#define MEDIUM_SPEED_400PAM4_112GBPS            0x3000L
 #define MEDIUM_SPEED_AUTONEG_1G_FALLBACK        0x8000L /* Serdes */
 #define MEDIUM_SPEED_AUTONEG_2_5G_FALLBACK      0x8100L /* Serdes */
 #define MEDIUM_SPEED_HARDWARE_DEFAULT           0xff00L /* Serdes nvram def.*/
@@ -168,9 +178,9 @@ union dma_addr64_t {
 	RX_MASK_ACCEPT_MULTICAST)
 #define MAX_NQ_DESC_CNT                         64
 #define NQ_RING_BUFFER_SIZE (MAX_NQ_DESC_CNT * sizeof(struct cmpl_base))
-#define TX_RING_QID (bp->thor ? (u16)bp->queue_id : ((u16)bp->port_idx * 10))
-#define RX_RING_QID (bp->thor ? bp->queue_id : 0)
-#define STAT_CTX_ID ((bp->vf || bp->thor) ? bp->stat_ctx_id : 0)
+#define TX_RING_QID (FLAG_TEST(bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS) ? (u16)bp->queue_id : ((u16)bp->port_idx * 10))
+#define RX_RING_QID (FLAG_TEST(bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS) ? bp->queue_id : 0)
+#define STAT_CTX_ID ((bp->vf || FLAG_TEST(bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS)) ? bp->stat_ctx_id : 0)
 #define TX_AVAIL(r)                      (r - 1)
 #define TX_IN_USE(a, b, c) ((a - b) & (c - 1))
 #define NO_MORE_NQ_BD_TO_SERVICE         1
@@ -189,13 +199,19 @@ union dma_addr64_t {
 	((idx) << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK)
 #define DBC_MSG_XID(xid, flg)  (\
 	(((xid) << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | \
-	DBC_DBC_PATH_L2 | (flg))
+	DBC_DBC_PATH_L2 | (FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) ? DBC_DBC_VALID : 0) | (flg))
+#define DBC_MSG_EPCH(idx)      (\
+        ((idx) << DBC_DBC_EPOCH_SFT))
+#define DBC_MSG_TOGGLE(idx)    (\
+        ((idx) << DBC_DBC_TOGGLE_SFT) & DBC_DBC_TOGGLE_MASK)
 #define PHY_STATUS         0x0001
 #define PHY_SPEED          0x0002
 #define DETECT_MEDIA       0x0004
 #define SUPPORT_SPEEDS     0x0008
+#define SUPPORT_SPEEDS2    0x0010
 #define QCFG_PHY_ALL   (\
-	SUPPORT_SPEEDS | DETECT_MEDIA | PHY_SPEED | PHY_STATUS)
+	SUPPORT_SPEEDS | SUPPORT_SPEEDS2 | \
+        DETECT_MEDIA | PHY_SPEED | PHY_STATUS)
 #define str_mbps           "Mbps"
 #define str_gbps           "Gbps"
 /*
@@ -287,6 +303,18 @@ union dma_addr64_t {
 #define NS_LINK_SPEED_FW_100G                                   (0x6)
 #define LINK_SPEED_FW_200G                                      (0x7L << 7)
 #define NS_LINK_SPEED_FW_200G                                   (0x7)
+#define LINK_SPEED_FW_50G_PAM4                                  (0x8L << 7)
+#define NS_LINK_SPEED_FW_50G_PAM4                               (0x8)
+#define LINK_SPEED_FW_100G_PAM4                                 (0x9L << 7)
+#define NS_LINK_SPEED_FW_100G_PAM4                              (0x9)
+#define LINK_SPEED_FW_100G_PAM4_112                             (0xAL << 7)
+#define NS_LINK_SPEED_FW_100G_PAM4_112                          (0xA)
+#define LINK_SPEED_FW_200G_PAM4_112                             (0xBL << 7)
+#define NS_LINK_SPEED_FW_200G_PAM4_112                          (0xB)
+#define LINK_SPEED_FW_400G_PAM4                                 (0xCL << 7)
+#define NS_LINK_SPEED_FW_400G_PAM4                              (0xC)
+#define LINK_SPEED_FW_400G_PAM4_112                             (0xDL << 7)
+#define NS_LINK_SPEED_FW_400G_PAM4_112                          (0xD)
 #define LINK_SPEED_FW_2_5G                                      (0xEL << 7)
 #define NS_LINK_SPEED_FW_2_5G                                   (0xE)
 #define LINK_SPEED_FW_100M                                      (0xFL << 7)
@@ -387,6 +415,10 @@ struct dbc_dbc {
 	__le32  index;
 	#define DBC_DBC_INDEX_MASK          0xffffffUL
 	#define DBC_DBC_INDEX_SFT           0
+	#define DBC_DBC_EPOCH               0x1000000UL
+	#define DBC_DBC_EPOCH_SFT           24
+	#define DBC_DBC_TOGGLE_MASK         0x6000000UL
+	#define DBC_DBC_TOGGLE_SFT          25
 	__le32  type_path_xid;
 	#define DBC_DBC_XID_MASK            0xfffffUL
 	#define DBC_DBC_XID_SFT             0
@@ -396,6 +428,7 @@ struct dbc_dbc {
 	#define DBC_DBC_PATH_L2             (0x1UL << 24)
 	#define DBC_DBC_PATH_ENGINE         (0x2UL << 24)
 	#define DBC_DBC_PATH_LAST           DBC_DBC_PATH_ENGINE
+	#define DBC_DBC_VALID               0x4000000UL
 	#define DBC_DBC_DEBUG_TRACE         0x8000000UL
 	#define DBC_DBC_TYPE_MASK           0xf0000000UL
 	#define DBC_DBC_TYPE_SFT            28
@@ -481,6 +514,8 @@ struct tx_info {
 	u16              ring_cnt;
 	u32              cnt;   /* Tx statistics. */
 	u32              cnt_req;
+	u8               epoch;
+	u8               res[3];
 };
 
 struct cmpl_base {
@@ -492,6 +527,7 @@ struct cmpl_base {
 #define CMPL_BASE_TYPE_RX_AGG            0x12UL
 #define CMPL_BASE_TYPE_RX_TPA_START      0x13UL
 #define CMPL_BASE_TYPE_RX_TPA_END        0x15UL
+#define CMPL_BASE_TYPE_RX_L2_V3          0x17UL
 #define CMPL_BASE_TYPE_STAT_EJECT        0x1aUL
 #define CMPL_BASE_TYPE_HWRM_DONE         0x20UL
 #define CMPL_BASE_TYPE_HWRM_FWD_REQ      0x22UL
@@ -517,7 +553,8 @@ struct cmp_info {
 	u16       cons_id;
 	u16       ring_cnt;
 	u8        completion_bit;
-	u8        res[3];
+	u8        epoch;
+	u8        res[2];
 };
 
 /* Completion Queue Notification */
@@ -533,6 +570,8 @@ struct nq_base {
  */
 #define NQ_CN_TYPE_MASK           0x3fUL
 #define NQ_CN_TYPE_SFT            0
+#define NQ_CN_TOGGLE_MASK         0xc0UL
+#define NQ_CN_TOGGLE_SFT          6
 /* CQ Notification */
 	    #define NQ_CN_TYPE_CQ_NOTIFICATION  0x30UL
 	    #define NQ_CN_TYPE_LAST            NQ_CN_TYPE_CQ_NOTIFICATION
@@ -561,7 +600,9 @@ struct nq_info {
 	u16       cons_id;
 	u16       ring_cnt;
 	u8        completion_bit;
-	u8        res[3];
+	u8        epoch;
+	u8        toggle;
+	u8        res[1];
 };
 
 struct rx_pkt_cmpl {
@@ -675,6 +716,156 @@ struct rx_pkt_cmpl_hi {
 #define RX_PKT_CMPL_REORDER_SFT  0
 };
 
+struct rx_pkt_v3_cmpl {
+	u16	flags_type;
+	#define RX_PKT_V3_CMPL_TYPE_MASK                      0x3fUL
+	#define RX_PKT_V3_CMPL_TYPE_SFT                       0
+	/*
+	 * RX L2 V3 completion:
+	 * Completion of and L2 RX packet. Length = 32B
+	 * This is the new version of the RX_L2 completion used in Thor2
+	 * and later chips.
+	 */
+	#define RX_PKT_V3_CMPL_TYPE_RX_L2_V3                    0x17UL
+	#define RX_PKT_V3_CMPL_TYPE_LAST                       RX_PKT_V3_CMPL_TYPE_RX_L2_V3
+	#define RX_PKT_V3_CMPL_FLAGS_MASK                     0xffc0UL
+	#define RX_PKT_V3_CMPL_FLAGS_SFT                      6
+	#define RX_PKT_V3_CMPL_FLAGS_ERROR                     0x40UL
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_MASK            0x380UL
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_SFT             7
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_NORMAL            (0x0UL << 7)
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_JUMBO             (0x1UL << 7)
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_HDS               (0x2UL << 7)
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_TRUNCATION        (0x3UL << 7)
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_LAST             RX_PKT_V3_CMPL_FLAGS_PLACEMENT_TRUNCATION
+	#define RX_PKT_V3_CMPL_FLAGS_RSS_VALID                 0x400UL
+	#define RX_PKT_V3_CMPL_FLAGS_PKT_METADATA_PRESENT      0x800UL
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_MASK                0xf000UL
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_SFT                 12
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_NOT_KNOWN             (0x0UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_IP                    (0x1UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_TCP                   (0x2UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_UDP                   (0x3UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_FCOE                  (0x4UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_ROCE                  (0x5UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_ICMP                  (0x7UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_PTP_WO_TIMESTAMP      (0x8UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP       (0x9UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_LAST                 RX_PKT_V3_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP
+	u16	len;
+	u32	opaque;
+	u16	rss_hash_type_agg_bufs_v1;
+	#define RX_PKT_V3_CMPL_V1                   0x1UL
+	#define RX_PKT_V3_CMPL_AGG_BUFS_MASK        0x3eUL
+	#define RX_PKT_V3_CMPL_AGG_BUFS_SFT         1
+	#define RX_PKT_V3_CMPL_UNUSED1              0x40UL
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_MASK   0xff80UL
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_SFT    7
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_0   (0x0UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_1   (0x1UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_3   (0x3UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_4   (0x4UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_5   (0x5UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_6   (0x6UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_7   (0x7UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_8   (0x8UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_9   (0x9UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_10  (0xaUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_11  (0xbUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_12  (0xcUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_13  (0xdUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_14  (0xeUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_LAST    RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_14
+	u16	metadata1_payload_offset;
+	#define RX_PKT_V3_CMPL_PAYLOAD_OFFSET_MASK        0x1ffUL
+	#define RX_PKT_V3_CMPL_PAYLOAD_OFFSET_SFT         0
+	#define RX_PKT_V3_CMPL_METADATA1_MASK             0xf000UL
+	#define RX_PKT_V3_CMPL_METADATA1_SFT              12
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_MASK     0x7000UL
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_SFT      12
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID88A8   (0x0UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID8100   (0x1UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID9100   (0x2UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID9200   (0x3UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID9300   (0x4UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPIDCFG    (0x5UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_LAST      RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPIDCFG
+	#define RX_PKT_V3_CMPL_METADATA1_VALID             0x8000UL
+	u32	rss_hash;
+};
+
+struct rx_pkt_v3_cmpl_hi {
+	u32	flags2;
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_IP_CS_CALC                 0x1UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_L4_CS_CALC                 0x2UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_CS_CALC               0x4UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_L4_CS_CALC               0x8UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_MASK           0xf0UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_SFT            4
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_NONE             (0x0UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_ACT_REC_PTR      (0x1UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_TUNNEL_ID        (0x2UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_CHDR_DATA        (0x3UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_HDR_OFFSET       (0x4UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_LAST            RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_HDR_OFFSET
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_IP_TYPE                    0x100UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_COMPLETE_CHECKSUM_CALC     0x200UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE                  0x400UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE_IPV4               (0x0UL << 10)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE_IPV6               (0x1UL << 10)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE_LAST              RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE_IPV6
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_COMPLETE_CHECKSUM_MASK     0xffff0000UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_COMPLETE_CHECKSUM_SFT      16
+	u32	metadata2;
+	u16	errors_v2;
+	#define RX_PKT_V3_CMPL_HI_V2                                       0x1UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_MASK                              0xfffeUL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_SFT                               1
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_MASK                  0xeUL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_SFT                   1
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_NO_BUFFER               (0x0UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_DID_NOT_FIT             (0x1UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_NOT_ON_CHIP             (0x2UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_BAD_FORMAT              (0x3UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_FLUSH                   (0x5UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_LAST                   RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_FLUSH
+	#define RX_PKT_V3_CMPL_HI_ERRORS_IP_CS_ERROR                        0x10UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_L4_CS_ERROR                        0x20UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_IP_CS_ERROR                      0x40UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_L4_CS_ERROR                      0x80UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_CRC_ERROR                          0x100UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_MASK                   0xe00UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_SFT                    9
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_NO_ERROR                 (0x0UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION         (0x1UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN         (0x2UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR         (0x3UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR        (0x4UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL             (0x5UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_TOTAL_ERROR            (0x6UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_LAST                    RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_TOTAL_ERROR
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_MASK                     0xf000UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_SFT                      12
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_NO_ERROR                   (0x0UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L3_BAD_VERSION             (0x1UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN             (0x2UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L3_BAD_TTL                 (0x3UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_IP_TOTAL_ERROR             (0x4UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR            (0x5UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN             (0x6UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL   (0x7UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN             (0x8UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_LAST                      RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN
+	u16	metadata0;
+	#define RX_PKT_V3_CMPL_HI_METADATA0_VID_MASK 0xfffUL
+	#define RX_PKT_V3_CMPL_HI_METADATA0_VID_SFT 0
+	#define RX_PKT_V3_CMPL_HI_METADATA0_DE      0x1000UL
+	/* When meta_format=1, this value is the VLAN PRI. */
+	#define RX_PKT_V3_CMPL_HI_METADATA0_PRI_MASK 0xe000UL
+	#define RX_PKT_V3_CMPL_HI_METADATA0_PRI_SFT 13
+	u32	timestamp;
+};
+
 struct rx_prod_pkt_bd {
 	u16  flags_type;
 #define RX_PROD_PKT_BD_TYPE_MASK          0x3fUL
@@ -705,6 +896,8 @@ struct rx_info {
 	u32               drop_err;
 	u32               drop_lb;
 	u32               drop_vlan;
+	u8                epoch;
+	u8                res[3];
 };
 
 #define VALID_DRIVER_REG          0x0001
@@ -750,7 +943,6 @@ struct bnxt {
 	struct nq_info            nq; /* completion info. */
 	u16                       nq_ring_id;
 	u8                        queue_id;
-	u8                        thor;
 	u16                       last_resp_code;
 	u16                       seq_id;
 	u32                       flag_hwrm;
@@ -792,6 +984,7 @@ struct bnxt {
 	u32                       mba_cfg2;
 	u32                       medium;
 	u16                       support_speeds;
+	u16                       auto_link_speeds2_mask;
 	u32                       link_set;
 	u8                        media_detect;
 	u8                        rsvd;
@@ -871,3 +1064,5 @@ struct bnxt {
 #define CHIP_NUM_57508       0x1750
 #define CHIP_NUM_57504       0x1751
 #define CHIP_NUM_57502       0x1752
+
+#define CHIP_NUM_57608       0x1760
diff --git a/src/drivers/net/bnxt/bnxt_dbg.h b/src/drivers/net/bnxt/bnxt_dbg.h
index 188978ad6..145402818 100644
--- a/src/drivers/net/bnxt/bnxt_dbg.h
+++ b/src/drivers/net/bnxt/bnxt_dbg.h
@@ -475,7 +475,7 @@ void dbg_rx_stat(struct bnxt *bp)
 #endif
 
 #if defined(DEBUG_CQ)
-static void dump_cq(struct cmpl_base *cmp, u16 cid)
+static void dump_cq(struct cmpl_base *cmp, u16 cid, u8 toggle)
 {
 	dbg_prn("- CQ Type ");
 	switch (cmp->type & CMPL_BASE_TYPE_MASK) {
@@ -495,7 +495,7 @@ static void dump_cq(struct cmpl_base *cmp, u16 cid)
 		dbg_prn("%04x", (u16)(cmp->type & CMPL_BASE_TYPE_MASK));
 		break;
 	}
-	dbg_prn(" cid %d", cid);
+	dbg_prn(" cid %d, tog %d", cid, toggle);
 #if defined(DEBUG_CQ_DUMP)
 	dump_mem((u8 *)cmp, (u32)sizeof(struct cmpl_base), DISP_U8);
 #else
@@ -513,7 +513,7 @@ static void dump_nq(struct nq_base *nqp, u16 cid)
 #endif
 }
 #else
-#define dump_cq(cq, id)
+#define dump_cq(cq, id, toggle)
 #define dump_nq(nq, id)
 #endif
 
diff --git a/src/drivers/net/bnxt/bnxt_hsi.h b/src/drivers/net/bnxt/bnxt_hsi.h
index 086acb8b3..dbcffd909 100644
--- a/src/drivers/net/bnxt/bnxt_hsi.h
+++ b/src/drivers/net/bnxt/bnxt_hsi.h
@@ -2929,7 +2929,7 @@ struct hwrm_func_drv_if_change_output {
 	u8	valid;
 };
 
-/* hwrm_port_phy_cfg_input (size:448b/56B) */
+/* hwrm_port_phy_cfg_input (size:512b/64B) */
 struct hwrm_port_phy_cfg_input {
 	__le16	req_type;
 	__le16	cmpl_ring;
@@ -2952,6 +2952,15 @@ struct hwrm_port_phy_cfg_input {
 	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_ENABLE      0x1000UL
 	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_DISABLE     0x2000UL
 	#define PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN           0x4000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_1XN_ENABLE     0x8000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_1XN_DISABLE    0x10000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_IEEE_ENABLE    0x20000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_IEEE_DISABLE   0x40000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_1XN_ENABLE     0x80000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_1XN_DISABLE    0x100000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_IEEE_ENABLE    0x200000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_IEEE_DISABLE   0x400000UL
+
 	__le32	enables;
 	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE                0x1UL
 	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_DUPLEX              0x2UL
@@ -2964,6 +2973,10 @@ struct hwrm_port_phy_cfg_input {
 	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE              0x100UL
 	#define PORT_PHY_CFG_REQ_ENABLES_EEE_LINK_SPEED_MASK      0x200UL
 	#define PORT_PHY_CFG_REQ_ENABLES_TX_LPI_TIMER             0x400UL
+	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED         0x800UL
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK     0x1000UL
+	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2            0x2000UL
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK        0x4000UL
 	__le16	port_id;
 	__le16	force_link_speed;
 	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100MB 0x1UL
@@ -3049,11 +3062,48 @@ struct hwrm_port_phy_cfg_input {
 	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD3     0x10UL
 	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD4     0x20UL
 	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_10GB      0x40UL
-	u8	unused_2[2];
+	__le16	force_pam4_link_speed;
+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB 0x7d0UL
+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_LAST PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB
 	__le32	tx_lpi_timer;
 	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_MASK 0xffffffUL
 	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_SFT 0
-	__le32	unused_3;
+	__le16	auto_link_pam4_speed_mask;
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_50G      0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_100G     0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_200G     0x4UL
+	__le16	force_link_speeds2;
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_1GB            0xaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_10GB           0x64UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_25GB           0xfaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_40GB           0x190UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB           0x1f4UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB          0x3e8UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB_PAM4_56   0x1f5UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_56  0x3e9UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_56  0x7d1UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_56  0xfa1UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_112 0x3eaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_112 0x7d2UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112 0xfa2UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_LAST          PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112
+	__le16	auto_link_speeds2_mask;
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_1GB                0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_10GB               0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_25GB               0x4UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_40GB               0x8UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_50GB               0x10UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB              0x20UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_400GB_PAM4_112     0x1000UL
+	u8	unused_2[6];
 };
 
 /* hwrm_port_phy_cfg_output (size:128b/16B) */
@@ -3087,7 +3137,7 @@ struct hwrm_port_phy_qcfg_input {
 	u8	unused_0[6];
 };
 
-/* hwrm_port_phy_qcfg_output (size:768b/96B) */
+/* hwrm_port_phy_qcfg_output (size:832b/104B) */
 struct hwrm_port_phy_qcfg_output {
 	__le16	error_code;
 	__le16	req_type;
@@ -3098,7 +3148,23 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_LINK_SIGNAL  0x1UL
 	#define PORT_PHY_QCFG_RESP_LINK_LINK    0x2UL
 	#define PORT_PHY_QCFG_RESP_LINK_LAST   PORT_PHY_QCFG_RESP_LINK_LINK
-	u8	unused_0;
+	u8	active_fec_signal_mode;
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK     0xfUL
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_SFT      0
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ      0x0UL
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4     0x1UL
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112 0x2UL
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_LAST     HWRM_PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK                   0xf0UL
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_SFT                    4
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE        (0x0UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE    (0x1UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE    (0x2UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE   (0x3UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE  (0x4UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE   (0x5UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE  (0x6UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_LAST                   PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE
 	__le16	link_speed;
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100MB 0x1UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_1GB   0xaUL
@@ -3111,6 +3177,7 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_50GB  0x1f4UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100GB 0x3e8UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_200GB 0x7d0UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_400GB 0xfa0UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10MB  0xffffUL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_LINK_SPEED_10MB
 	u8	duplex_cfg;
@@ -3249,7 +3316,31 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASESR4     0x1dUL
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASELR4     0x1eUL
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER4     0x1fUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_LAST            PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER4
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASECR       0x20UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASESR       0x21UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASELR       0x22UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASEER       0x23UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR2     0x24UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR2     0x25UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR2     0x26UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER2     0x27UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR      0x28UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR      0x29UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR      0x2aUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER      0x2bUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASECR2     0x2cUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASESR2     0x2dUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASELR2     0x2eUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER2     0x2fUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASECR8     0x30UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASESR8     0x31UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASELR8     0x32UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER8     0x33UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASECR4     0x34UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASESR4     0x35UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASELR4     0x36UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER4     0x37UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_LAST            PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER4
 	u8	media_type;
 	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_UNKNOWN 0x0UL
 	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP      0x1UL
@@ -3330,15 +3421,90 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_ENABLED       0x10UL
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_SUPPORTED     0x20UL
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_ENABLED       0x40UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_1XN_SUPPORTED    0x80UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_1XN_ENABLED      0x100UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_IEEE_SUPPORTED   0x200UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_IEEE_ENABLED     0x400UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_1XN_SUPPORTED    0x800UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_1XN_ENABLED      0x1000UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_IEEE_SUPPORTED   0x2000UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_IEEE_ENABLED     0x4000UL
 	u8	duplex_state;
 	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_HALF 0x0UL
 	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL 0x1UL
 	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_LAST PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL
 	u8	option_flags;
 	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_MEDIA_AUTO_DETECT     0x1UL
+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN     0x2UL
+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_SPEEDS2_SUPPORTED     0x4UL
 	char	phy_vendor_name[16];
 	char	phy_vendor_partnumber[16];
-	u8	unused_2[7];
+	__le16	support_pam4_speeds;
+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_50G      0x1UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_100G     0x2UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_200G     0x4UL
+	__le16	force_pam4_link_speed;
+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_200GB 0x7d0UL
+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_200GB
+	__le16	auto_pam4_link_speed_mask;
+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_50G      0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_100G     0x2UL
+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_200G     0x4UL
+	u8	link_partner_pam4_adv_speeds;
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_50GB      0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_100GB     0x2UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_200GB     0x4UL
+	u8	link_down_reason;
+	#define PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_RF     0x1UL
+	__le16	support_speeds2;
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_1GB                0x1UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_10GB               0x2UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_25GB               0x4UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_40GB               0x8UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_50GB               0x10UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB              0x20UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_400GB_PAM4_112     0x1000UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_800GB_PAM4_112     0x2000UL
+	__le16	force_link_speeds2;
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_1GB            0xaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_10GB           0x64UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_25GB           0xfaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_40GB           0x190UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_50GB           0x1f4UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB          0x3e8UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_50GB_PAM4_56   0x1f5UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB_PAM4_56  0x3e9UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_200GB_PAM4_56  0x7d1UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_400GB_PAM4_56  0xfa1UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB_PAM4_112 0x3eaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_200GB_PAM4_112 0x7d2UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_400GB_PAM4_112 0xfa2UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_800GB_PAM4_112 0x1f42UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_LAST          PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_800GB_PAM4_112
+	__le16	auto_link_speeds2;
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_1GB                0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_10GB               0x2UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_25GB               0x4UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_40GB               0x8UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_50GB               0x10UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB              0x20UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_400GB_PAM4_112     0x1000UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_800GB_PAM4_112     0x2000UL
+	u8	active_lanes;
 	u8	valid;
 };
 
@@ -3888,7 +4054,7 @@ struct hwrm_port_phy_qcaps_input {
 	u8	unused_0[6];
 };
 
-/* hwrm_port_phy_qcaps_output (size:192b/24B) */
+/* hwrm_port_phy_qcaps_output (size:320b/40B) */
 struct hwrm_port_phy_qcaps_output {
 	__le16	error_code;
 	__le16	req_type;
@@ -3954,6 +4120,53 @@ struct hwrm_port_phy_qcaps_output {
 	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_SFT 0
 	#define PORT_PHY_QCAPS_RESP_VALID_MASK            0xff000000UL
 	#define PORT_PHY_QCAPS_RESP_VALID_SFT             24
+	__le16	supported_pam4_speeds_auto_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_50G      0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_100G     0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_200G     0x4UL
+	__le16	supported_pam4_speeds_force_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_50G      0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_100G     0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_200G     0x4UL
+	__le16	flags2;
+	#define PORT_PHY_QCAPS_RESP_FLAGS2_PAUSE_UNSUPPORTED       0x1UL
+	#define PORT_PHY_QCAPS_RESP_FLAGS2_PFC_UNSUPPORTED         0x2UL
+	#define PORT_PHY_QCAPS_RESP_FLAGS2_BANK_ADDR_SUPPORTED     0x4UL
+	#define PORT_PHY_QCAPS_RESP_FLAGS2_SPEEDS2_SUPPORTED       0x8UL
+	u8	internal_port_cnt;
+	u8	unused_0;
+	__le16	supported_speeds2_force_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_1GB                0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_10GB               0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_25GB               0x4UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_40GB               0x8UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_50GB               0x10UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB              0x20UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_400GB_PAM4_112     0x1000UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_800GB_PAM4_112     0x2000UL
+	__le16	supported_speeds2_auto_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_1GB                0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_10GB               0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_25GB               0x4UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_40GB               0x8UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_50GB               0x10UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB              0x20UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_400GB_PAM4_112     0x1000UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_800GB_PAM4_112     0x2000UL
+	u8	unused_1[3];
+	u8	valid;
 };
 
 /* hwrm_port_phy_i2c_write_input (size:832b/104B) */

From 0f5abd8b117ea7da0747fe36b66c050706f6584f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 8 Feb 2024 16:39:35 +0000
Subject: [PATCH 072/237] [libc] Allow build_assert() failures to be ignored
 via NO_WERROR=1

We build with -Werror by default so that any warning is treated as an
error and aborts the build.  The build system allows NO_WERROR=1 to be
used to override this behaviour, in order to allow builds to succeed
when spurious warnings occur (e.g. when using a newer compiler that
includes checks for which the codebase is not yet prepared).

Some versions of gcc (observed with gcc 4.8.5 in CentOS 7) will report
spurious build_assert() failures: the compilation will fail due to an
allegedly unelided call to the build assertion's external function
declared with __attribute__((error)) even though the compiler does
manage to successfully elide the call (as verified by the fact that
there are no unresolvable symbol references in the compiler output).

Change build_assert() to declare __attribute__((warning)) instead of
__attribute__((error)) on its extern function.  This will still abort
a normal build if the assertion fails, but may be overridden using
NO_WERROR=1 if necessary to work around a spurious assertion failure.

Note that if the build assertion has genuinely failed (i.e. if the
compiler has genuinely not been able to elide the call) then the
object will still contain an unresolvable symbol reference that will
cause the link to fail (which matches the behaviour of the old
linker_assert() mechanism).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/assert.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/include/assert.h b/src/include/assert.h
index b3a9b1fe0..01a287855 100644
--- a/src/include/assert.h
+++ b/src/include/assert.h
@@ -76,7 +76,7 @@ assert_printf ( const char *fmt, ... ) asm ( "printf" );
 #define build_assert( condition )					     \
 	do { 								     \
 		if ( ! (condition) ) {					     \
-			extern void __attribute__ (( error (		     \
+			extern void __attribute__ (( warning (		     \
 				"build_assert(" #condition ") failed"	     \
 			) )) _C2 ( build_assert_, __LINE__ ) ( void );	     \
 			_C2 ( build_assert_, __LINE__ ) ();		     \

From 94b39fbe9298160b034c93ca06deb39a907e3b3f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sat, 10 Feb 2024 14:41:29 +0000
Subject: [PATCH 073/237] [build] Fix build failures with older versions of gcc

Some versions of gcc (observed with gcc 4.8.5 in CentOS 7) will report
spurious build_assert() failures for some assertions about structure
layouts.  There is no clear pattern as to what causes these spurious
failures, and the build assertion does succeed in that no unresolvable
symbol reference is generated in the compiled code.

Adjust the assertions to work around these apparent compiler issues.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/gcm.c       | 14 ++++++++------
 src/include/ipxe/gcm.h |  3 ++-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/crypto/gcm.c b/src/crypto/gcm.c
index c21aad14c..a32890d59 100644
--- a/src/crypto/gcm.c
+++ b/src/crypto/gcm.c
@@ -469,13 +469,15 @@ int gcm_setkey ( struct gcm_context *context, const void *key, size_t keylen,
  * @v ivlen		Initialisation vector length
  */
 void gcm_setiv ( struct gcm_context *context, const void *iv, size_t ivlen ) {
-	union gcm_block *check = ( ( void * ) context );
 
-	/* Sanity checks */
-	build_assert ( &context->hash == check );
-	build_assert ( &context->len == check + 1 );
-	build_assert ( &context->ctr == check + 2 );
-	build_assert ( &context->key == check + 3 );
+	/* Sanity check: ensure that memset()s will clear expected state */
+	build_assert ( &context->hash < &context->ctr );
+	build_assert ( &context->len < &context->ctr );
+	build_assert ( &context->ctr < &context->key );
+	build_assert ( ( ( void * ) &context->raw_cipher ) >
+		       ( ( void * ) &context->key ) );
+	build_assert ( ( ( void * ) context->raw_ctx ) >
+		       ( ( void * ) &context->key ) );
 
 	/* Reset non-key state */
 	memset ( context, 0, offsetof ( typeof ( *context ), key ) );
diff --git a/src/include/ipxe/gcm.h b/src/include/ipxe/gcm.h
index 9653a0a1a..4864445d2 100644
--- a/src/include/ipxe/gcm.h
+++ b/src/include/ipxe/gcm.h
@@ -89,7 +89,8 @@ static int _gcm_name ## _setkey ( void *ctx, const void *key,		\
 				  size_t keylen ) {			\
 	struct _gcm_name ## _context *context = ctx;			\
 	build_assert ( _blocksize == sizeof ( context->gcm.key ) );	\
-	build_assert ( ( ( void * ) &context->gcm ) == ctx );		\
+	build_assert ( ( ( void * ) &context->gcm ) ==			\
+		       ( ( void * ) context ) );			\
 	build_assert ( ( ( void * ) &context->raw ) ==			\
 		       ( ( void * ) context->gcm.raw_ctx ) );		\
 	return gcm_setkey ( &context->gcm, key, keylen, &_raw_cipher );	\

From 88b291d647686ac686a5614fdd7be92b48747360 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 14 Feb 2024 16:01:43 +0000
Subject: [PATCH 074/237] [list] Add list_is_head_entry()

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/list.h | 11 +++++++++++
 src/tests/list_test.c   | 16 ++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/include/ipxe/list.h b/src/include/ipxe/list.h
index 8de254984..53edfc9f7 100644
--- a/src/include/ipxe/list.h
+++ b/src/include/ipxe/list.h
@@ -398,6 +398,17 @@ extern void extern_list_splice_tail_init ( struct list_head *list,
 #define list_is_last_entry( entry, head, member )		\
 	( (head)->prev == &(entry)->member )
 
+/**
+ * Test if entry is the list head
+ *
+ * @v entry		List entry
+ * @v head		List head
+ * @v member		Name of list field within iterator's type
+ * @ret is_head		Entry is the list head
+ */
+#define list_is_head_entry( entry, head, member )		\
+	( (head) == &(entry)->member )
+
 /**
  * Iterate over a list
  *
diff --git a/src/tests/list_test.c b/src/tests/list_test.c
index d5b5c65db..f18c63f2d 100644
--- a/src/tests/list_test.c
+++ b/src/tests/list_test.c
@@ -440,6 +440,22 @@ static void list_test_exec ( void ) {
 	ok ( list_is_first_entry ( &list_tests[3], list, list ) );
 	ok ( list_is_last_entry ( &list_tests[3], list, list ) );
 
+	/* Test list_is_head_entry() */
+	INIT_LIST_HEAD ( list );
+	list_add_tail ( &list_tests[1].list, list );
+	list_add_tail ( &list_tests[6].list, list );
+	list_add_tail ( &list_tests[8].list, list );
+	ok ( list_is_head_entry ( list_entry ( list, typeof ( *pos ), list ),
+				  list, list ) );
+	ok ( ! list_is_head_entry ( &list_tests[1], list, list ) );
+	ok ( ! list_is_head_entry ( &list_tests[6], list, list ) );
+	ok ( ! list_is_head_entry ( &list_tests[8], list, list ) );
+	list_for_each_entry ( pos, list, list ) {
+		ok ( list_contains_entry ( pos, list, list ) );
+		ok ( ! list_is_head_entry ( pos, list, list ) );
+	}
+	ok ( list_is_head_entry ( pos, list, list ) );
+
 	/* Test list_for_each() */
 	INIT_LIST_HEAD ( list );
 	list_add_tail ( &list_tests[6].list, list );

From e10dfe5dc7a5985333c85d6b196196b5cce9303a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 13 Feb 2024 16:26:34 +0000
Subject: [PATCH 075/237] [list] Add list_for_each_entry_safe_continue()

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/list.h | 16 ++++++++++++++++
 src/tests/list_test.c   | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/src/include/ipxe/list.h b/src/include/ipxe/list.h
index 53edfc9f7..2f02e71f0 100644
--- a/src/include/ipxe/list.h
+++ b/src/include/ipxe/list.h
@@ -489,6 +489,22 @@ extern void extern_list_splice_tail_init ( struct list_head *list,
 	      &pos->member != (head);					      \
 	      pos = list_entry ( pos->member.prev, typeof ( *pos ), member ) )
 
+/**
+ * Iterate over subsequent entries in a list, safe against deletion
+ *
+ * @v pos		Iterator
+ * @v tmp		Temporary value (of same type as iterator)
+ * @v head		List head
+ * @v member		Name of list field within iterator's type
+ */
+#define list_for_each_entry_safe_continue( pos, tmp, head, member )	      \
+	for ( list_check ( (head) ),					      \
+	      pos = list_entry ( pos->member.next, typeof ( *pos ), member ), \
+	      tmp = list_entry ( pos->member.next, typeof ( *tmp ), member ); \
+	      &pos->member != (head);					      \
+	      pos = tmp,						      \
+	      tmp = list_entry ( tmp->member.next, typeof ( *tmp ), member ) )
+
 /**
  * Test if list contains a specified entry
  *
diff --git a/src/tests/list_test.c b/src/tests/list_test.c
index f18c63f2d..c24e80828 100644
--- a/src/tests/list_test.c
+++ b/src/tests/list_test.c
@@ -518,6 +518,38 @@ static void list_test_exec ( void ) {
 	list_iterate_entry_ok ( list_for_each_entry_continue_reverse, "",
 				pos, list, list );
 
+	/* Test list_for_each_entry_safe_continue() */
+	INIT_LIST_HEAD ( list );
+	list_add_tail ( &list_tests[9].list, list );
+	list_add_tail ( &list_tests[4].list, list );
+	list_add_tail ( &list_tests[2].list, list );
+	list_add_tail ( &list_tests[5].list, list );
+	list_add_tail ( &list_tests[7].list, list );
+	{
+		char *expecteds[] = { "94257", "9457", "947", "94" };
+		char **expected = expecteds;
+		pos = &list_tests[4];
+		list_for_each_entry_safe_continue ( pos, tmp, list, list ) {
+			list_contents_ok ( list, *expected );
+			list_del ( &pos->list );
+			expected++;
+			list_contents_ok ( list, *expected );
+		}
+	}
+	list_contents_ok ( list, "94" );
+	{
+		char *expecteds[] = { "94", "4", "" };
+		char **expected = expecteds;
+		ok ( pos == list_entry ( list, struct list_test, list ) );
+		list_for_each_entry_safe_continue ( pos, tmp, list, list ) {
+			list_contents_ok ( list, *expected );
+			list_del ( &pos->list );
+			expected++;
+			list_contents_ok ( list, *expected );
+		}
+	}
+	ok ( list_empty ( list ) );
+
 	/* Test list_contains() and list_contains_entry() */
 	INIT_LIST_HEAD ( list );
 	INIT_LIST_HEAD ( &list_tests[3].list );

From 3e721e0c0836588b64deb6e1c1befd08f0f02e71 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 13 Feb 2024 16:27:31 +0000
Subject: [PATCH 076/237] [crypto] Add x509_truncate() to truncate a
 certificate chain

Downloading a cross-signed certificate chain to partially replace
(rather than simply extend) an existing chain will require the ability
to discard all certificates after a specified link in the chain.

Extract the relevant logic from x509_free_chain() and expose it
separately as x509_truncate().

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/x509.c       | 32 +++++++++++++++++++++++---------
 src/include/ipxe/x509.h |  1 +
 src/tests/x509_test.c   | 13 +++++++++++++
 3 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/src/crypto/x509.c b/src/crypto/x509.c
index 1f017eb03..92318093e 100644
--- a/src/crypto/x509.c
+++ b/src/crypto/x509.c
@@ -1603,19 +1603,12 @@ int x509_check_name ( struct x509_certificate *cert, const char *name ) {
 static void x509_free_chain ( struct refcnt *refcnt ) {
 	struct x509_chain *chain =
 		container_of ( refcnt, struct x509_chain, refcnt );
-	struct x509_link *link;
-	struct x509_link *tmp;
 
 	DBGC2 ( chain, "X509 chain %p freed\n", chain );
 
-	/* Free each link in the chain */
-	list_for_each_entry_safe ( link, tmp, &chain->links, list ) {
-		x509_put ( link->cert );
-		list_del ( &link->list );
-		free ( link );
-	}
-
 	/* Free chain */
+	x509_truncate ( chain, NULL );
+	assert ( list_empty ( &chain->links ) );
 	free ( chain );
 }
 
@@ -1696,6 +1689,27 @@ int x509_append_raw ( struct x509_chain *chain, const void *data,
 	return rc;
 }
 
+/**
+ * Truncate X.509 certificate chain
+ *
+ * @v chain		X.509 certificate chain
+ * @v link		Link after which to truncate chain, or NULL
+ */
+void x509_truncate ( struct x509_chain *chain, struct x509_link *link ) {
+	struct x509_link *tmp;
+
+	/* Truncate entire chain if no link is specified */
+	if ( ! link )
+		link = list_entry ( &chain->links, struct x509_link, list );
+
+	/* Free each link in the chain */
+	list_for_each_entry_safe_continue ( link, tmp, &chain->links, list ) {
+		x509_put ( link->cert );
+		list_del ( &link->list );
+		free ( link );
+	}
+}
+
 /**
  * Identify X.509 certificate by subject
  *
diff --git a/src/include/ipxe/x509.h b/src/include/ipxe/x509.h
index c703c8f10..5cad4597d 100644
--- a/src/include/ipxe/x509.h
+++ b/src/include/ipxe/x509.h
@@ -391,6 +391,7 @@ extern int x509_append ( struct x509_chain *chain,
 			 struct x509_certificate *cert );
 extern int x509_append_raw ( struct x509_chain *chain, const void *data,
 			     size_t len );
+extern void x509_truncate ( struct x509_chain *chain, struct x509_link *link );
 extern int x509_auto_append ( struct x509_chain *chain,
 			      struct x509_chain *certs );
 extern int x509_validate_chain ( struct x509_chain *chain, time_t time,
diff --git a/src/tests/x509_test.c b/src/tests/x509_test.c
index b6cba575c..bc9032041 100644
--- a/src/tests/x509_test.c
+++ b/src/tests/x509_test.c
@@ -984,6 +984,7 @@ static void x509_validate_chain_fail_okx ( struct x509_test_chain *chn,
  *
  */
 static void x509_test_exec ( void ) {
+	struct x509_link *link;
 
 	/* Parse all certificates */
 	x509_certificate_ok ( &root_crt );
@@ -1089,6 +1090,18 @@ static void x509_test_exec ( void ) {
 	x509_validate_chain_fail_ok ( &useless_chain, test_ca_expired,
 				      &empty_store, &test_root );
 
+	/* Check chain truncation */
+	link = list_last_entry ( &server_chain.chain->links,
+				 struct x509_link, list );
+	ok ( link->cert == root_crt.cert );
+	link = list_prev_entry ( link, &server_chain.chain->links, list );
+	ok ( link->cert == intermediate_crt.cert );
+	x509_validate_chain_ok ( &server_chain, test_time,
+				 &empty_store, &test_root );
+	x509_truncate ( server_chain.chain, link );
+	x509_validate_chain_fail_ok ( &server_chain, test_time,
+				      &empty_store, &test_root );
+
 	/* Sanity check */
 	assert ( list_empty ( &empty_store.links ) );
 

From 943d75b557a8bf857d651e8116a7368b9d284e41 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 15 Feb 2024 12:43:51 +0000
Subject: [PATCH 077/237] [crypto] Add x509_is_self_signed() helper function

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/x509.h | 10 ++++++++++
 src/net/validator.c     |  2 +-
 src/tests/x509_test.c   |  4 ++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/include/ipxe/x509.h b/src/include/ipxe/x509.h
index 5cad4597d..d2ba49fbb 100644
--- a/src/include/ipxe/x509.h
+++ b/src/include/ipxe/x509.h
@@ -374,6 +374,16 @@ x509_root_put ( struct x509_root *root ) {
 	ref_put ( &root->refcnt );
 }
 
+/**
+ * Check if X.509 certificate is self-signed
+ *
+ * @v cert		X.509 certificate
+ * @ret is_self_signed	X.509 certificate is self-signed
+ */
+static inline int x509_is_self_signed ( struct x509_certificate *cert ) {
+	return ( asn1_compare ( &cert->issuer.raw, &cert->subject.raw ) == 0 );
+}
+
 extern const char * x509_name ( struct x509_certificate *cert );
 extern int x509_parse ( struct x509_certificate *cert,
 			const struct asn1_cursor *raw );
diff --git a/src/net/validator.c b/src/net/validator.c
index 693d4464b..333c60798 100644
--- a/src/net/validator.c
+++ b/src/net/validator.c
@@ -595,7 +595,7 @@ static void validator_step ( struct validator *validator ) {
 	 * nothing more to do.
 	 */
 	last = x509_last ( validator->chain );
-	if ( asn1_compare ( &last->issuer.raw, &last->subject.raw ) == 0 ) {
+	if ( x509_is_self_signed ( last ) ) {
 		validator_finished ( validator, rc );
 		return;
 	}
diff --git a/src/tests/x509_test.c b/src/tests/x509_test.c
index bc9032041..50eb4d787 100644
--- a/src/tests/x509_test.c
+++ b/src/tests/x509_test.c
@@ -1102,6 +1102,10 @@ static void x509_test_exec ( void ) {
 	x509_validate_chain_fail_ok ( &server_chain, test_time,
 				      &empty_store, &test_root );
 
+	/* Check self-signedess */
+	ok ( x509_is_self_signed ( root_crt.cert ) );
+	ok ( ! x509_is_self_signed ( intermediate_crt.cert ) );
+
 	/* Sanity check */
 	assert ( list_empty ( &empty_store.links ) );
 

From 929f06a76de37612015882af592997a7da15a82d Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 14 Feb 2024 16:02:43 +0000
Subject: [PATCH 078/237] [crypto] Allow for multiple cross-signed certificate
 download attempts

Certificates issued by Let's Encrypt have two options for their chain
of trust: the chain can either terminate in the self-signed ISRG Root
X1 root certificate, or in an intermediate ISRG Root X1 certificate
that is signed in turn by the self-signed DST Root CA X3 root
certificate.  This is a historical artifact: when Let's Encrypt first
launched as a project, the chain ending in DST Root CA X3 was used
since existing clients would not have recognised the ISRG Root X1
certificate as a trusted root certificate.

The DST Root CA X3 certificate expired in September 2021, and so is no
longer trusted by clients (such as iPXE) that validate the expiry
times of all certificates in the certificate chain.

In order to maintain usability of certificates on older Android
devices, the default certificate chain provided by Let's Encrypt still
terminates in DST Root CA X3, even though that certificate has now
expired.  On newer devices which include ISRG Root X1 as a trusted
root certificate, the intermediate version of ISRG Root X1 in the
certificate chain is ignored and validation is performed as though the
chain had terminated in the self-signed ISRG Root X1 root certificate.
On older Android devices which do not include ISRG Root X1 as a
trusted root certificate, the validation succeeds since Android
chooses to ignore expiry times for root certificates and so continues
to trust the DST Root CA X3 root certificate.

This backwards compatibility hack unfortunately breaks the cross-
signing mechanism used by iPXE, which assumes that the certificate
chain will always terminate in a non-expired root certificate.

Generalise the validator's cross-signed certificate download mechanism
to walk up the certificate chain in the event of a failure, attempting
to find a replacement cross-signed certificate chain starting from the
next level up.  This allows the validator to step over the expired
(and hence invalidatable) DST Root CA X3 certificate, and instead
download the cross-signed version of the ISRG Root X1 certificate.

This generalisation also gives us the ability to handle servers that
provide a full certificate chain including their root certificate:
iPXE will step over the untrusted public root certificate and attempt
to find a cross-signed version of it instead.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/x509.h |  22 +++
 src/net/validator.c     | 329 ++++++++++++++++++++++++++++------------
 2 files changed, 250 insertions(+), 101 deletions(-)

diff --git a/src/include/ipxe/x509.h b/src/include/ipxe/x509.h
index d2ba49fbb..87323cec0 100644
--- a/src/include/ipxe/x509.h
+++ b/src/include/ipxe/x509.h
@@ -171,6 +171,28 @@ struct x509_link {
 	struct list_head list;
 	/** Certificate */
 	struct x509_certificate *cert;
+	/** Flags */
+	unsigned int flags;
+};
+
+/** X.509 certficate chain link flags */
+enum x509_link_flags {
+	/** Cross-signed certificate download has been attempted
+	 *
+	 * This indicates that a cross-signature download attempt has
+	 * been made to find a cross-signed issuer for this link's
+	 * certificate.
+	 */
+	X509_LINK_FL_CROSSED = 0x0001,
+	/** OCSP has been attempted
+	 *
+	 * This indicates that an OCSP attempt has been made using
+	 * this link's certificate as an issuer.  (We record the flag
+	 * on the issuer rather than on the issued certificate, since
+	 * we want to retry OCSP if an issuer is replaced with a
+	 * downloaded cross-signed certificate.)
+	 */
+	X509_LINK_FL_OCSPED = 0x0002,
 };
 
 /** An X.509 certificate chain */
diff --git a/src/net/validator.c b/src/net/validator.c
index 333c60798..69c0df333 100644
--- a/src/net/validator.c
+++ b/src/net/validator.c
@@ -57,8 +57,7 @@ struct validator_action {
 	/** Name */
 	const char *name;
 	/** Action to take upon completed transfer */
-	int ( * done ) ( struct validator *validator, const void *data,
-			 size_t len );
+	void ( * done ) ( struct validator *validator, int rc );
 };
 
 /** A certificate validator */
@@ -72,6 +71,40 @@ struct validator {
 
 	/** Process */
 	struct process process;
+	/** Most relevant status code
+	 *
+	 * The cross-signed certificate mechanism may attempt several
+	 * downloads as it works its way up the provided partial chain
+	 * to locate a suitable cross-signed certificate with which to
+	 * complete the chain.
+	 *
+	 * Some of these download or validation attempts may fail for
+	 * uninteresting reasons (i.e. because a cross-signed
+	 * certificate has never existed for that link in the chain).
+	 *
+	 * We must therefore keep track of the most relevant error
+	 * that has occurred, in order to be able to report a
+	 * meaningful overall status to the user.
+	 *
+	 * As a concrete example: consider the case of an expired OCSP
+	 * signer for an intermediate certificate.  This will cause
+	 * OCSP validation to fail for that intermediate certificate,
+	 * and this is the error that should eventually be reported to
+	 * the user.  We do not want to instead report the
+	 * uninteresting fact that no cross-signed certificate was
+	 * found for the remaining links in the chain, nor do we want
+	 * to report just a generic "OCSP required" error.
+	 *
+	 * We record the most relevant status code whenever a
+	 * definitely relevant error occurs, and clear it whenever we
+	 * successfully make forward progress (e.g. by completing
+	 * OCSP, or by adding new cross-signed certificates).
+	 *
+	 * When we subsequently attempt to validate the chain, we
+	 * report the most relevant error status code (if recorded),
+	 * otherwise we report the validation error itself.
+	 */
+	int rc;
 
 	/** Root of trust (or NULL to use default) */
 	struct x509_root *root;
@@ -84,13 +117,15 @@ struct validator {
 
 	/** Current action */
 	const struct validator_action *action;
-	/** Current certificate
+	/** Current certificate (for progress reporting)
 	 *
 	 * This will always be present within the certificate chain
 	 * and so this pointer does not hold a reference to the
 	 * certificate.
 	 */
 	struct x509_certificate *cert;
+	/** Current link within certificate chain */
+	struct x509_link *link;
 };
 
 /**
@@ -196,17 +231,36 @@ static const char crosscert_default[] = CROSSCERT;
  * Append cross-signing certificates to certificate chain
  *
  * @v validator		Certificate validator
- * @v data		Raw cross-signing certificate data
- * @v len		Length of raw data
+ * @v rc		Completion status code
  * @ret rc		Return status code
  */
-static int validator_append ( struct validator *validator,
-			      const void *data, size_t len ) {
+static void validator_append ( struct validator *validator, int rc ) {
 	struct asn1_cursor cursor;
 	struct x509_chain *certs;
 	struct x509_certificate *cert;
-	struct x509_certificate *last;
-	int rc;
+	struct x509_link *link;
+	struct x509_link *prev;
+
+	/* Check for errors */
+	if ( rc != 0 ) {
+		DBGC ( validator, "VALIDATOR %p \"%s\" could not download ",
+		       validator, validator_name ( validator ) );
+		DBGC ( validator, "\"%s\" cross-signature: %s\n",
+		       x509_name ( validator->cert ), strerror ( rc ) );
+		/* If the overall validation is going to fail, then we
+		 * will end up attempting multiple downloads for
+		 * non-existent cross-signed certificates as we work
+		 * our way up the certificate chain.  Do not record
+		 * these as relevant errors, since we want to
+		 * eventually report whichever much more relevant
+		 * error occurred previously.
+		 */
+		goto err_irrelevant;
+	}
+	DBGC ( validator, "VALIDATOR %p \"%s\" downloaded ",
+	       validator, validator_name ( validator ) );
+	DBGC ( validator, "\"%s\" cross-signature\n",
+	       x509_name ( validator->cert ) );
 
 	/* Allocate certificate list */
 	certs = x509_alloc_chain();
@@ -216,8 +270,8 @@ static int validator_append ( struct validator *validator,
 	}
 
 	/* Initialise cursor */
-	cursor.data = data;
-	cursor.len = len;
+	cursor.data = validator->buffer.data;
+	cursor.len = validator->buffer.len;
 
 	/* Enter certificateSet */
 	if ( ( rc = asn1_enter ( &cursor, ASN1_SET ) ) != 0 ) {
@@ -230,14 +284,14 @@ static int validator_append ( struct validator *validator,
 	/* Add each certificate to list */
 	while ( cursor.len ) {
 
-		/* Add certificate to chain */
+		/* Add certificate to list */
 		if ( ( rc = x509_append_raw ( certs, cursor.data,
 					      cursor.len ) ) != 0 ) {
 			DBGC ( validator, "VALIDATOR %p \"%s\" could not "
 			       "append certificate: %s\n", validator,
 			       validator_name ( validator ), strerror ( rc) );
 			DBGC_HDA ( validator, 0, cursor.data, cursor.len );
-			return rc;
+			goto err_append_raw;
 		}
 		cert = x509_last ( certs );
 		DBGC ( validator, "VALIDATOR %p \"%s\" found certificate ",
@@ -248,8 +302,12 @@ static int validator_append ( struct validator *validator,
 		asn1_skip_any ( &cursor );
 	}
 
+	/* Truncate existing certificate chain at current link */
+	link = validator->link;
+	assert ( link->flags & X509_LINK_FL_CROSSED );
+	x509_truncate ( validator->chain, link );
+
 	/* Append certificates to chain */
-	last = x509_last ( validator->chain );
 	if ( ( rc = x509_auto_append ( validator->chain, certs ) ) != 0 ) {
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not append "
 		       "certificates: %s\n", validator,
@@ -257,26 +315,31 @@ static int validator_append ( struct validator *validator,
 		goto err_auto_append;
 	}
 
-	/* Check that at least one certificate has been added */
-	if ( last == x509_last ( validator->chain ) ) {
-		DBGC ( validator, "VALIDATOR %p \"%s\" failed to append any "
-		       "applicable certificates\n", validator,
-		       validator_name ( validator ) );
-		rc = -EACCES;
-		goto err_no_progress;
+	/* Record that a cross-signed certificate download has already
+	 * been performed for all but the last of the appended
+	 * certificates.  (It may be necessary to perform a further
+	 * download to complete the chain, if this download did not
+	 * extend all the way to a root of trust.)
+	 */
+	prev = NULL;
+	list_for_each_entry_continue ( link, &validator->chain->links, list ) {
+		if ( prev )
+			prev->flags |= X509_LINK_FL_CROSSED;
+		prev = link;
 	}
 
-	/* Drop reference to certificate list */
-	x509_chain_put ( certs );
+	/* Success */
+	rc = 0;
 
-	return 0;
-
- err_no_progress:
  err_auto_append:
+ err_append_raw:
  err_certificateset:
 	x509_chain_put ( certs );
  err_alloc_certs:
-	return rc;
+	validator->rc = rc;
+ err_irrelevant:
+	/* Do not record irrelevant errors */
+	return;
 }
 
 /** Cross-signing certificate download validator action */
@@ -289,11 +352,12 @@ static const struct validator_action validator_crosscert = {
  * Start download of cross-signing certificate
  *
  * @v validator		Certificate validator
- * @v cert		X.509 certificate
+ * @v link		Link in certificate chain
  * @ret rc		Return status code
  */
 static int validator_start_download ( struct validator *validator,
-				      struct x509_certificate *cert ) {
+				      struct x509_link *link ) {
+	struct x509_certificate *cert = link->cert;
 	const struct asn1_cursor *issuer = &cert->issuer.raw;
 	const char *crosscert;
 	char *crosscert_copy;
@@ -336,6 +400,7 @@ static int validator_start_download ( struct validator *validator,
 	/* Set completion handler */
 	validator->action = &validator_crosscert;
 	validator->cert = cert;
+	validator->link = link;
 
 	/* Open URI */
 	if ( ( rc = xfer_open_uri_string ( &validator->xfer,
@@ -346,14 +411,20 @@ static int validator_start_download ( struct validator *validator,
 		goto err_open_uri_string;
 	}
 
-	/* Success */
-	rc = 0;
+	/* Free temporary allocations */
+	free ( uri_string );
+	free ( crosscert_copy );
 
+	/* Success */
+	return 0;
+
+	intf_restart ( &validator->xfer, rc );
  err_open_uri_string:
 	free ( uri_string );
  err_alloc_uri_string:
  err_check_uri_string:
 	free ( crosscert_copy );
+	validator->rc = rc;
 	return rc;
 }
 
@@ -367,21 +438,27 @@ static int validator_start_download ( struct validator *validator,
  * Validate OCSP response
  *
  * @v validator		Certificate validator
- * @v data		Raw OCSP response
- * @v len		Length of raw data
- * @ret rc		Return status code
+ * @v rc		Completion status code
  */
-static int validator_ocsp_validate ( struct validator *validator,
-				     const void *data, size_t len ) {
+static void validator_ocsp_validate ( struct validator *validator, int rc ) {
+	const void *data = validator->buffer.data;
+	size_t len = validator->buffer.len;
 	time_t now;
-	int rc;
+
+	/* Check for errors */
+	if ( rc != 0 ) {
+		DBGC ( validator, "VALIDATOR %p \"%s\" could not fetch OCSP "
+		       "response: %s\n", validator,
+		       validator_name ( validator ), strerror ( rc ) );
+		goto err_status;
+	}
 
 	/* Record OCSP response */
 	if ( ( rc = ocsp_response ( validator->ocsp, data, len ) ) != 0 ) {
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not record OCSP "
 		       "response: %s\n", validator,
-		       validator_name ( validator ),strerror ( rc ) );
-		return rc;
+		       validator_name ( validator ), strerror ( rc ) );
+		goto err_response;
 	}
 
 	/* Validate OCSP response */
@@ -390,14 +467,20 @@ static int validator_ocsp_validate ( struct validator *validator,
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not validate "
 		       "OCSP response: %s\n", validator,
 		       validator_name ( validator ), strerror ( rc ) );
-		return rc;
+		goto err_validate;
 	}
 
-	/* Drop reference to OCSP check */
+	/* Success */
+	DBGC ( validator, "VALIDATOR %p \"%s\" checked ",
+	       validator, validator_name ( validator ) );
+	DBGC ( validator, "\"%s\" via OCSP\n", x509_name ( validator->cert ) );
+
+ err_validate:
+ err_response:
+ err_status:
 	ocsp_put ( validator->ocsp );
 	validator->ocsp = NULL;
-
-	return 0;
+	validator->rc = rc;
 }
 
 /** OCSP validator action */
@@ -426,7 +509,7 @@ static int validator_start_ocsp ( struct validator *validator,
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not create OCSP "
 		       "check: %s\n", validator, validator_name ( validator ),
 		       strerror ( rc ) );
-		return rc;
+		goto err_check;
 	}
 
 	/* Set completion handler */
@@ -444,10 +527,18 @@ static int validator_start_ocsp ( struct validator *validator,
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not open %s: "
 		       "%s\n", validator, validator_name ( validator ),
 		       uri_string, strerror ( rc ) );
-		return rc;
+		goto err_open;
 	}
 
 	return 0;
+
+	intf_restart ( &validator->xfer, rc );
+ err_open:
+	ocsp_put ( validator->ocsp );
+	validator->ocsp = NULL;
+ err_check:
+	validator->rc = rc;
+	return rc;
 }
 
 /****************************************************************************
@@ -466,34 +557,18 @@ static void validator_xfer_close ( struct validator *validator, int rc ) {
 
 	/* Close data transfer interface */
 	intf_restart ( &validator->xfer, rc );
-
-	/* Check for errors */
-	if ( rc != 0 ) {
-		DBGC ( validator, "VALIDATOR %p \"%s\" transfer failed: %s\n",
-		       validator, validator_name ( validator ),
-		       strerror ( rc ) );
-		goto err_transfer;
-	}
 	DBGC2 ( validator, "VALIDATOR %p \"%s\" transfer complete\n",
 		validator, validator_name ( validator ) );
 
 	/* Process completed download */
 	assert ( validator->action != NULL );
-	if ( ( rc = validator->action->done ( validator, validator->buffer.data,
-					      validator->buffer.len ) ) != 0 )
-		goto err_append;
+	validator->action->done ( validator, rc );
 
 	/* Free downloaded data */
 	xferbuf_free ( &validator->buffer );
 
 	/* Resume validation process */
 	process_add ( &validator->process );
-
-	return;
-
- err_append:
- err_transfer:
-	validator_finished ( validator, rc );
 }
 
 /**
@@ -515,7 +590,7 @@ static int validator_xfer_deliver ( struct validator *validator,
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not receive "
 		       "data: %s\n", validator, validator_name ( validator ),
 		       strerror ( rc ) );
-		validator_finished ( validator, rc );
+		validator_xfer_close ( validator, rc );
 		return rc;
 	}
 
@@ -544,10 +619,10 @@ static struct interface_descriptor validator_xfer_desc =
  * @v validator		Certificate validator
  */
 static void validator_step ( struct validator *validator ) {
+	struct x509_chain *chain = validator->chain;
 	struct x509_link *link;
+	struct x509_link *prev;
 	struct x509_certificate *cert;
-	struct x509_certificate *issuer = NULL;
-	struct x509_certificate *last;
 	time_t now;
 	int rc;
 
@@ -556,57 +631,109 @@ static void validator_step ( struct validator *validator ) {
 	 * previously.
 	 */
 	now = time ( NULL );
-	if ( ( rc = x509_validate_chain ( validator->chain, now, NULL,
+	if ( ( rc = x509_validate_chain ( chain, now, NULL,
 					  validator->root ) ) == 0 ) {
 		DBGC ( validator, "VALIDATOR %p \"%s\" validated\n",
 		       validator, validator_name ( validator ) );
 		validator_finished ( validator, 0 );
 		return;
 	}
+	DBGC ( validator, "VALIDATOR %p \"%s\" not yet valid: %s\n",
+	       validator, validator_name ( validator ), strerror ( rc ) );
 
-	/* If there is a certificate that could be validated using
-	 * OCSP, try it.
+	/* Record as the most relevant error, if no more relevant
+	 * error has already been recorded.
 	 */
-	list_for_each_entry ( link, &validator->chain->links, list ) {
-		cert = issuer;
-		issuer = link->cert;
-		if ( ! cert )
-			continue;
-		if ( ! x509_is_valid ( issuer, validator->root ) )
-			continue;
-		/* The issuer is valid, but this certificate is not
-		 * yet valid.  If OCSP is applicable, start it.
-		 */
-		if ( ocsp_required ( cert ) ) {
-			/* Start OCSP */
-			if ( ( rc = validator_start_ocsp ( validator, cert,
-							   issuer ) ) != 0 ) {
-				validator_finished ( validator, rc );
-				return;
-			}
+	if ( validator->rc == 0 )
+		validator->rc = rc;
+
+	/* Find the first valid link in the chain, if any
+	 *
+	 * There is no point in attempting OCSP or cross-signed
+	 * certificate downloads for certificates after the first
+	 * valid link in the chain, since they cannot make a
+	 * difference to the overall validation of the chain.
+	 */
+	prev = NULL;
+	list_for_each_entry ( link, &chain->links, list ) {
+
+		/* Dump link information (for debugging) */
+		DBGC ( validator, "VALIDATOR %p \"%s\" has link ",
+		       validator, validator_name ( validator ) );
+		DBGC ( validator, "\"%s\"%s%s%s%s%s\n",
+		       x509_name ( link->cert ),
+		       ( ocsp_required ( link->cert ) ? " [NEEDOCSP]" : "" ),
+		       ( ( link->flags & X509_LINK_FL_OCSPED ) ?
+			 " [OCSPED]" : "" ),
+		       ( ( link->flags & X509_LINK_FL_CROSSED ) ?
+			 " [CROSSED]" : "" ),
+		       ( x509_is_self_signed ( link->cert ) ? " [SELF]" : "" ),
+		       ( x509_is_valid ( link->cert, validator->root ) ?
+			 " [VALID]" : "" ) );
+
+		/* Stop at first valid link */
+		if ( x509_is_valid ( link->cert, validator->root ) )
+			break;
+		prev = link;
+	}
+
+	/* If this link is the issuer for a certificate that is
+	 * pending an OCSP check attempt, then start OCSP to validate
+	 * that certificate.
+	 *
+	 * If OCSP is not required for the issued certificate, or has
+	 * already been attempted, or if we were unable to start OCSP
+	 * for any reason, then proceed to attempting a cross-signed
+	 * certificate download (which may end up replacing this
+	 * issuer anyway).
+	 */
+	if ( ( ! list_is_head_entry ( link, &chain->links, list ) ) &&
+	     ( ! ( link->flags & X509_LINK_FL_OCSPED ) ) &&
+	     ( prev != NULL ) && ocsp_required ( prev->cert ) ) {
+
+		/* Mark OCSP as attempted with this issuer */
+		link->flags |= X509_LINK_FL_OCSPED;
+
+		/* Start OCSP */
+		if ( ( rc = validator_start_ocsp ( validator, prev->cert,
+						   link->cert ) ) == 0 ) {
+			/* Sleep until OCSP is complete */
 			return;
 		}
-		/* Otherwise, this is a permanent failure */
-		validator_finished ( validator, rc );
-		return;
 	}
 
-	/* If chain ends with a self-issued certificate, then there is
-	 * nothing more to do.
+	/* Work back up the chain (starting from the already
+	 * identified first valid link, if any) to find a not-yet
+	 * valid certificate for which we could attempt to download a
+	 * cross-signed certificate chain.
 	 */
-	last = x509_last ( validator->chain );
-	if ( x509_is_self_signed ( last ) ) {
-		validator_finished ( validator, rc );
-		return;
+	list_for_each_entry_continue_reverse ( link, &chain->links, list ) {
+		cert = link->cert;
+
+		/* Sanity check */
+		assert ( ! x509_is_valid ( cert, validator->root ) );
+
+		/* Skip self-signed certificates (cannot be cross-signed) */
+		if ( x509_is_self_signed ( cert ) )
+			continue;
+
+		/* Skip previously attempted cross-signed downloads */
+		if ( link->flags & X509_LINK_FL_CROSSED )
+			continue;
+
+		/* Mark cross-signed certificate download as attempted */
+		link->flags |= X509_LINK_FL_CROSSED;
+
+		/* Start cross-signed certificate download */
+		if ( ( rc = validator_start_download ( validator,
+						       link ) ) == 0 ) {
+			/* Sleep until download is complete */
+			return;
+		}
 	}
 
-	/* Otherwise, try to download a suitable cross-signing
-	 * certificate.
-	 */
-	if ( ( rc = validator_start_download ( validator, last ) ) != 0 ) {
-		validator_finished ( validator, rc );
-		return;
-	}
+	/* Nothing more to try: fail the validation */
+	validator_finished ( validator, validator->rc );
 }
 
 /** Certificate validator process descriptor */

From 075292cc2dcde1ad2580d37ac019f29a0eaa01ef Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 21 Feb 2024 16:45:50 +0000
Subject: [PATCH 079/237] [crypto] Add implementation of MS-CHAPv2
 authentication

Add an implementation of the authentication portions of the MS-CHAPv2
algorithm as defined in RFC 2759, along with the single test vector
provided therein.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/mschapv2.c       | 363 ++++++++++++++++++++++++++++++++++++
 src/include/ipxe/mschapv2.h |  59 ++++++
 src/tests/mschapv2_test.c   | 144 ++++++++++++++
 src/tests/tests.c           |   1 +
 4 files changed, 567 insertions(+)
 create mode 100644 src/crypto/mschapv2.c
 create mode 100644 src/include/ipxe/mschapv2.h
 create mode 100644 src/tests/mschapv2_test.c

diff --git a/src/crypto/mschapv2.c b/src/crypto/mschapv2.c
new file mode 100644
index 000000000..ac55fec17
--- /dev/null
+++ b/src/crypto/mschapv2.c
@@ -0,0 +1,363 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * MS-CHAPv2 authentication
+ *
+ * The algorithms used for MS-CHAPv2 authentication are defined in
+ * RFC 2759 section 8.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <byteswap.h>
+#include <ipxe/md4.h>
+#include <ipxe/sha1.h>
+#include <ipxe/des.h>
+#include <ipxe/mschapv2.h>
+
+/**
+ * MS-CHAPv2 context block
+ *
+ * For no particularly discernible reason, MS-CHAPv2 uses two
+ * different digest algorithms and one block cipher.  The uses do not
+ * overlap, so share the context storage between these to reduce stack
+ * usage.
+ */
+union mschapv2_context {
+	/** SHA-1 digest context */
+	uint8_t sha1[SHA1_CTX_SIZE];
+	/** MD4 digest context */
+	uint8_t md4[MD4_CTX_SIZE];
+	/** DES cipher context */
+	uint8_t des[DES_CTX_SIZE];
+};
+
+/**
+ * MS-CHAPv2 challenge hash
+ *
+ * MS-CHAPv2 calculates the SHA-1 digest of the peer challenge, the
+ * authenticator challenge, and the username, and then uses only the
+ * first 8 bytes of the result (as a DES plaintext block).
+ */
+union mschapv2_challenge_hash {
+	/** SHA-1 digest */
+	uint8_t sha1[SHA1_DIGEST_SIZE];
+	/** DES plaintext block */
+	uint8_t des[DES_BLOCKSIZE];
+};
+
+/**
+ * MS-CHAPv2 password hash
+ *
+ * MS-CHAPv2 calculates the MD4 digest of an unspecified two-byte
+ * little-endian Unicode encoding (presumably either UCS-2LE or
+ * UTF-16LE) of the password.
+ *
+ * For constructing the challenge response, the MD4 digest is then
+ * zero-padded to 21 bytes and used as three separate 56-bit DES keys.
+ *
+ * For constructing the authenticator response, the MD4 digest is then
+ * used as an input to a SHA-1 digest along with the NT response and a
+ * magic constant.
+ */
+union mschapv2_password_hash {
+	/** MD4 digest */
+	uint8_t md4[MD4_DIGEST_SIZE];
+	/** SHA-1 digest */
+	uint8_t sha1[SHA1_DIGEST_SIZE];
+	/** DES keys */
+	uint8_t des[3][DES_BLOCKSIZE];
+	/** DES key expansion */
+	uint8_t expand[ 3 * DES_BLOCKSIZE ];
+};
+
+/** MS-CHAPv2 magic constant 1 */
+static const char mschapv2_magic1[39] =
+	"Magic server to client signing constant";
+
+/** MS-CHAPv2 magic constant 2 */
+static const char mschapv2_magic2[41] =
+	"Pad to make it do more than one iteration";
+
+/**
+ * Calculate MS-CHAPv2 challenge hash
+ *
+ * @v ctx		Context block
+ * @v challenge		Authenticator challenge
+ * @v peer		Peer challenge
+ * @v username		User name (or NULL to use empty string)
+ * @v chash		Challenge hash to fill in
+ *
+ * This is the ChallengeHash() function as documented in RFC 2759
+ * section 8.2.
+ */
+static void
+mschapv2_challenge_hash ( union mschapv2_context *ctx,
+			  const struct mschapv2_challenge *challenge,
+			  const struct mschapv2_challenge *peer,
+			  const char *username,
+			  union mschapv2_challenge_hash *chash ) {
+	struct digest_algorithm *sha1 = &sha1_algorithm;
+
+	/* Calculate SHA-1 hash of challenges and username */
+	digest_init ( sha1, ctx->sha1 );
+	digest_update ( sha1, ctx->sha1, peer, sizeof ( *peer ) );
+	digest_update ( sha1, ctx->sha1, challenge, sizeof ( *challenge ) );
+	if ( username ) {
+		digest_update ( sha1, ctx->sha1, username,
+				strlen ( username ) );
+	}
+	digest_final ( sha1, ctx->sha1, chash->sha1 );
+	DBGC ( ctx, "MSCHAPv2 authenticator challenge:\n" );
+	DBGC_HDA ( ctx, 0, challenge, sizeof ( *challenge ) );
+	DBGC ( ctx, "MSCHAPv2 peer challenge:\n" );
+	DBGC_HDA ( ctx, 0, peer, sizeof ( *peer ) );
+	DBGC ( ctx, "MSCHAPv2 challenge hash:\n" );
+	DBGC_HDA ( ctx, 0, chash->des, sizeof ( chash->des ) );
+}
+
+/**
+ * Calculate MS-CHAPv2 password hash
+ *
+ * @v ctx		Context block
+ * @v password		Password (or NULL to use empty string)
+ * @v phash		Password hash to fill in
+ *
+ * This is the NtPasswordHash() function as documented in RFC 2759
+ * section 8.3.
+ */
+static void mschapv2_password_hash ( union mschapv2_context *ctx,
+				     const char *password,
+				     union mschapv2_password_hash *phash ) {
+	struct digest_algorithm *md4 = &md4_algorithm;
+	uint16_t wc;
+	uint8_t c;
+
+	/* Construct zero-padded MD4 hash of encoded password */
+	memset ( phash, 0, sizeof ( *phash ) );
+	digest_init ( md4, ctx->md4 );
+	if ( password ) {
+		while ( ( c = *(password++) ) ) {
+			wc = cpu_to_le16 ( c );
+			digest_update ( md4, ctx->md4, &wc, sizeof ( wc ) );
+		}
+	}
+	digest_final ( md4, ctx->md4, phash->md4 );
+	DBGC ( ctx, "MSCHAPv2 password hash:\n" );
+	DBGC_HDA ( ctx, 0, phash->md4, sizeof ( phash->md4 ) );
+}
+
+/**
+ * Hash the MS-CHAPv2 password hash
+ *
+ * @v ctx		Context block
+ * @v phash		Password hash to be rehashed
+ *
+ * This is the HashNtPasswordHash() function as documented in RFC 2759
+ * section 8.4.
+ */
+static void mschapv2_hash_hash ( union mschapv2_context *ctx,
+				 union mschapv2_password_hash *phash ) {
+	struct digest_algorithm *md4 = &md4_algorithm;
+
+	/* Calculate MD4 hash of existing MD4 hash */
+	digest_init ( md4, ctx->md4 );
+	digest_update ( md4, ctx->md4, phash->md4, sizeof ( phash->md4 ) );
+	digest_final ( md4, ctx->md4, phash->md4 );
+	DBGC ( ctx, "MSCHAPv2 password hash hash:\n" );
+	DBGC_HDA ( ctx, 0, phash->md4, sizeof ( phash->md4 ) );
+}
+
+/**
+ * Expand MS-CHAPv2 password hash by inserting DES dummy parity bits
+ *
+ * @v ctx		Context block
+ * @v phash		Password hash to expand
+ *
+ * This is part of the DesEncrypt() function as documented in RFC 2759
+ * section 8.6.
+ */
+static void mschapv2_expand_hash ( union mschapv2_context *ctx,
+				   union mschapv2_password_hash *phash ) {
+	uint8_t *dst;
+	uint8_t *src;
+	unsigned int i;
+
+	/* Expand password hash by inserting (unused) DES parity bits */
+	for ( i = ( sizeof ( phash->expand ) - 1 ) ; i > 0 ; i-- ) {
+		dst = &phash->expand[i];
+		src = ( dst - ( i / 8 ) );
+		*dst = ( ( ( src[-1] << 8 ) | src[0] ) >> ( i % 8 ) );
+	}
+	DBGC ( ctx, "MSCHAPv2 expanded password hash:\n" );
+	DBGC_HDA ( ctx, 0, phash->expand, sizeof ( phash->expand ) );
+}
+
+/**
+ * Calculate MS-CHAPv2 challenge response
+ *
+ * @v ctx		Context block
+ * @v chash		Challenge hash
+ * @v phash		Password hash (after expansion)
+ * @v nt		NT response to fill in
+ *
+ * This is the ChallengeResponse() function as documented in RFC 2759
+ * section 8.5.
+ */
+static void
+mschapv2_challenge_response ( union mschapv2_context *ctx,
+			      const union mschapv2_challenge_hash *chash,
+			      const union mschapv2_password_hash *phash,
+			      struct mschapv2_nt_response *nt ) {
+	struct cipher_algorithm *des = &des_algorithm;
+	unsigned int i;
+	int rc;
+
+	/* Construct response.  The design of the algorithm here is
+	 * interesting, suggesting that an intern at Microsoft had
+	 * heard the phrase "Triple DES" and hazarded a blind guess at
+	 * what it might mean.
+	 */
+	for ( i = 0 ; i < ( sizeof ( phash->des ) /
+			    sizeof ( phash->des[0] ) ) ; i++ ) {
+		rc = cipher_setkey ( des, ctx->des, phash->des[i],
+				     sizeof ( phash->des[i] ) );
+		assert ( rc == 0 ); /* no failure mode exists */
+		cipher_encrypt ( des, ctx->des, chash->des, nt->block[i],
+				 sizeof ( chash->des ) );
+	}
+	DBGC ( ctx, "MSCHAPv2 NT response:\n" );
+	DBGC_HDA ( ctx, 0, nt, sizeof ( *nt ) );
+}
+
+/**
+ * Calculate MS-CHAPv2 challenge response
+ *
+ * @v username		User name (or NULL to use empty string)
+ * @v password		Password (or NULL to use empty string)
+ * @v challenge		Authenticator challenge
+ * @v peer		Peer challenge
+ * @v response		Challenge response to fill in
+ *
+ * This is essentially the GenerateNTResponse() function as documented
+ * in RFC 2759 section 8.1.
+ */
+void mschapv2_response ( const char *username, const char *password,
+			 const struct mschapv2_challenge *challenge,
+			 const struct mschapv2_challenge *peer,
+			 struct mschapv2_response *response ) {
+	union mschapv2_context ctx;
+	union mschapv2_challenge_hash chash;
+	union mschapv2_password_hash phash;
+
+	/* Zero reserved fields */
+	memset ( response, 0, sizeof ( *response ) );
+
+	/* Copy peer challenge to response */
+	memcpy ( &response->peer, peer, sizeof ( response->peer ) );
+
+	/* Construct challenge hash */
+	mschapv2_challenge_hash ( &ctx, challenge, peer, username, &chash );
+
+	/* Construct expanded password hash */
+	mschapv2_password_hash ( &ctx, password, &phash );
+	mschapv2_expand_hash ( &ctx, &phash );
+
+	/* Construct NT response */
+	mschapv2_challenge_response ( &ctx, &chash, &phash, &response->nt );
+	DBGC ( &ctx, "MSCHAPv2 challenge response:\n" );
+	DBGC_HDA ( &ctx, 0, response, sizeof ( *response ) );
+}
+
+/**
+ * Calculate MS-CHAPv2 authenticator response
+ *
+ * @v username		User name (or NULL to use empty string)
+ * @v password		Password (or NULL to use empty string)
+ * @v challenge		Authenticator challenge
+ * @v response		Challenge response
+ * @v auth		Authenticator response to fill in
+ *
+ * This is essentially the GenerateAuthenticatorResponse() function as
+ * documented in RFC 2759 section 8.7.
+ */
+void mschapv2_auth ( const char *username, const char *password,
+		     const struct mschapv2_challenge *challenge,
+		     const struct mschapv2_response *response,
+		     struct mschapv2_auth *auth ) {
+	struct digest_algorithm *sha1 = &sha1_algorithm;
+	union mschapv2_context ctx;
+	union mschapv2_challenge_hash chash;
+	union mschapv2_password_hash phash;
+	char tmp[3];
+	char *wtf;
+	unsigned int i;
+
+	/* Construct hash of password hash */
+	mschapv2_password_hash ( &ctx, password, &phash );
+	mschapv2_hash_hash ( &ctx, &phash );
+
+	/* Construct unnamed intermediate hash */
+	digest_init ( sha1, ctx.sha1 );
+	digest_update ( sha1, ctx.sha1, phash.md4, sizeof ( phash.md4 ) );
+	digest_update ( sha1, ctx.sha1, &response->nt,
+			sizeof ( response->nt ) );
+	digest_update ( sha1, ctx.sha1, mschapv2_magic1,
+			sizeof ( mschapv2_magic1 ) );
+	digest_final ( sha1, ctx.sha1, phash.sha1 );
+	DBGC ( &ctx, "MSCHAPv2 NT response:\n" );
+	DBGC_HDA ( &ctx, 0, &response->nt, sizeof ( response->nt ) );
+	DBGC ( &ctx, "MSCHAPv2 unnamed intermediate hash:\n" );
+	DBGC_HDA ( &ctx, 0, phash.sha1, sizeof ( phash.sha1 ) );
+
+	/* Construct challenge hash */
+	mschapv2_challenge_hash ( &ctx, challenge, &response->peer,
+				  username, &chash );
+
+	/* Construct authenticator response hash */
+	digest_init ( sha1, ctx.sha1 );
+	digest_update ( sha1, ctx.sha1, phash.sha1, sizeof ( phash.sha1 ) );
+	digest_update ( sha1, ctx.sha1, chash.des, sizeof ( chash.des ) );
+	digest_update ( sha1, ctx.sha1, mschapv2_magic2,
+			sizeof ( mschapv2_magic2 ) );
+	digest_final ( sha1, ctx.sha1, phash.sha1 );
+	DBGC ( &ctx, "MSCHAPv2 authenticator response hash:\n" );
+	DBGC_HDA ( &ctx, 0, phash.sha1, sizeof ( phash.sha1 ) );
+
+	/* Encode authenticator response hash */
+	wtf = auth->wtf;
+	*(wtf++) = 'S';
+	*(wtf++) = '=';
+	DBGC ( &ctx, "MSCHAPv2 authenticator response: S=" );
+	for ( i = 0 ; i < sizeof ( phash.sha1 ) ; i++ ) {
+		snprintf ( tmp, sizeof ( tmp ), "%02X", phash.sha1[i] );
+		*(wtf++) = tmp[0];
+		*(wtf++) = tmp[1];
+		DBGC ( &ctx, "%s", tmp );
+	}
+	DBGC ( &ctx, "\n" );
+}
diff --git a/src/include/ipxe/mschapv2.h b/src/include/ipxe/mschapv2.h
new file mode 100644
index 000000000..59cf37eee
--- /dev/null
+++ b/src/include/ipxe/mschapv2.h
@@ -0,0 +1,59 @@
+#ifndef _IPXE_MSCHAPV2_H
+#define _IPXE_MSCHAPV2_H
+
+/** @file
+ *
+ * MS-CHAPv2 authentication
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/** An MS-CHAPv2 challenge */
+struct mschapv2_challenge {
+	/** Raw bytes */
+	uint8_t byte[16];
+} __attribute__ (( packed ));
+
+/** An MS-CHAPv2 NT response */
+struct mschapv2_nt_response {
+	/** DES-encrypted blocks */
+	uint8_t block[3][8];
+} __attribute__ (( packed ));
+
+/** An MS-CHAPv2 challenge response */
+struct mschapv2_response {
+	/** Peer challenge */
+	struct mschapv2_challenge peer;
+	/** Reserved, must be zero */
+	uint8_t reserved[8];
+	/** NT response */
+	struct mschapv2_nt_response nt;
+	/** Flags, must be zero */
+	uint8_t flags;
+} __attribute__ (( packed ));
+
+/** An MS-CHAPv2 authenticator response */
+struct mschapv2_auth {
+	/** Authenticator response string
+	 *
+	 * This is an unterminated 42-byte string of the form
+	 * "S=<auth_string>" where <auth_string> is the upper-cased
+	 * hexadecimal encoding of the actual authenticator response
+	 * value.  Joy.
+	 */
+	char wtf[42];
+} __attribute__ (( packed ));
+
+extern void mschapv2_response ( const char *username, const char *password,
+				const struct mschapv2_challenge *challenge,
+				const struct mschapv2_challenge *peer,
+				struct mschapv2_response *response );
+extern void mschapv2_auth ( const char *username, const char *password,
+			    const struct mschapv2_challenge *challenge,
+			    const struct mschapv2_response *response,
+			    struct mschapv2_auth *auth );
+
+#endif /* _IPXE_MSCHAPV2_H */
diff --git a/src/tests/mschapv2_test.c b/src/tests/mschapv2_test.c
new file mode 100644
index 000000000..3d10ed184
--- /dev/null
+++ b/src/tests/mschapv2_test.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * MS-CHAPv2 authentication self-tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdlib.h>
+#include <string.h>
+#include <ipxe/mschapv2.h>
+#include <ipxe/test.h>
+
+/** An MS-CHAPv2 test */
+struct mschapv2_test {
+	/** Username */
+	const char *username;
+	/** Password */
+	const char *password;
+	/** Authenticator challenge */
+	const struct mschapv2_challenge *challenge;
+	/** Peer challenge */
+	const struct mschapv2_challenge *peer;
+	/** Expected challenge response */
+	const struct mschapv2_response *response;
+	/** Expected authenticator response */
+	const struct mschapv2_auth *auth;
+};
+
+/** Define inline data */
+#define DATA(...) { __VA_ARGS__ }
+
+/** Define an MS-CHAPv2 test */
+#define MSCHAPV2_TEST( name, USERNAME, PASSWORD, CHALLENGE, PEER,	\
+		       RESPONSE, AUTH )					\
+	static const struct mschapv2_challenge name ## _challenge = {	\
+		.byte = CHALLENGE,					\
+	};								\
+	static const struct mschapv2_challenge name ## _peer = {	\
+		.byte = PEER,						\
+	};								\
+	static const union {						\
+		struct mschapv2_response response;			\
+		uint8_t byte[ sizeof ( struct mschapv2_response ) ];	\
+	} name ## _response = {						\
+		.byte = RESPONSE,					\
+	};								\
+	static const union {						\
+		struct mschapv2_auth auth;				\
+		uint8_t byte[ sizeof ( struct mschapv2_auth ) ];	\
+	} name ## _auth = {						\
+		.byte = AUTH,						\
+	};								\
+	static struct mschapv2_test name = {				\
+		.username = USERNAME,					\
+		.password = PASSWORD,					\
+		.challenge = &name ## _challenge,			\
+		.peer = &name ## _peer,					\
+		.response = &name ## _response.response,		\
+		.auth = &name ## _auth.auth,				\
+	};
+
+/** RFC 2759 section 9.2 test case */
+MSCHAPV2_TEST ( rfc2759_test,
+		"User", "clientPass",
+		DATA ( 0x5b, 0x5d, 0x7c, 0x7d, 0x7b, 0x3f, 0x2f, 0x3e,
+		       0x3c, 0x2c, 0x60, 0x21, 0x32, 0x26, 0x26, 0x28 ),
+		DATA ( 0x21, 0x40, 0x23, 0x24, 0x25, 0x5e, 0x26, 0x2a,
+		       0x28, 0x29, 0x5f, 0x2b, 0x3a, 0x33, 0x7c, 0x7e ),
+		DATA ( 0x21, 0x40, 0x23, 0x24, 0x25, 0x5e, 0x26, 0x2a,
+		       0x28, 0x29, 0x5f, 0x2b, 0x3a, 0x33, 0x7c, 0x7e,
+		       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		       0x82, 0x30, 0x9e, 0xcd, 0x8d, 0x70, 0x8b, 0x5e,
+		       0xa0, 0x8f, 0xaa, 0x39, 0x81, 0xcd, 0x83, 0x54,
+		       0x42, 0x33, 0x11, 0x4a, 0x3d, 0x85, 0xd6, 0xdf,
+		       0x00 ),
+		"S=407A5589115FD0D6209F510FE9C04566932CDA56" );
+
+/**
+ * Report an MS-CHAPv2 test result
+ *
+ * @v test		Authentication test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void mschapv2_okx ( struct mschapv2_test *test,
+			   const char *file, unsigned int line ) {
+	struct mschapv2_response response;
+	struct mschapv2_auth auth;
+
+	/* Compute challenge response */
+	mschapv2_response ( test->username, test->password, test->challenge,
+			    test->peer, &response );
+	okx ( memcmp ( &response, test->response, sizeof ( response ) ) == 0,
+	      file, line );
+
+	/* Compute authenticator response */
+	mschapv2_auth ( test->username, test->password, test->challenge,
+			test->response, &auth );
+	okx ( memcmp ( &auth, test->auth, sizeof ( auth ) ) == 0, file, line );
+}
+#define mschapv2_ok( test )				\
+	mschapv2_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Perform MS-CHAPv2 self-test
+ *
+ */
+static void mschapv2_test_exec ( void ) {
+
+	mschapv2_ok ( &rfc2759_test );
+}
+
+/** MS-CHAPv2 self-test */
+struct self_test mschapv2_test __self_test = {
+	.name = "mschapv2",
+	.exec = mschapv2_test_exec,
+};
diff --git a/src/tests/tests.c b/src/tests/tests.c
index 282d2eb65..90cec9489 100644
--- a/src/tests/tests.c
+++ b/src/tests/tests.c
@@ -83,3 +83,4 @@ REQUIRE_OBJECT ( gcm_test );
 REQUIRE_OBJECT ( nap_test );
 REQUIRE_OBJECT ( x25519_test );
 REQUIRE_OBJECT ( des_test );
+REQUIRE_OBJECT ( mschapv2_test );

From 582132fe3f72377f592aa127d920355afa887384 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 22 Feb 2024 12:55:59 +0000
Subject: [PATCH 080/237] [crypto] Force inlining of trivial wrapper functions

Inspection of the generated assembly shows that gcc will often emit
standalone implementations of frequently invoked functions such as
digest_update(), which contain no logic and exist only as syntactic
sugar.

Force inlining of these functions to reduce the overall binary size.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/crypto.h | 99 ++++++++++++++++++++++-----------------
 1 file changed, 55 insertions(+), 44 deletions(-)

diff --git a/src/include/ipxe/crypto.h b/src/include/ipxe/crypto.h
index 9ee1c40cb..a6f437655 100644
--- a/src/include/ipxe/crypto.h
+++ b/src/include/ipxe/crypto.h
@@ -212,34 +212,37 @@ struct elliptic_curve {
 			     void *result );
 };
 
-static inline void digest_init ( struct digest_algorithm *digest,
-				 void *ctx ) {
+static inline __attribute__ (( always_inline )) void
+digest_init ( struct digest_algorithm *digest, void *ctx ) {
 	digest->init ( ctx );
 }
 
-static inline void digest_update ( struct digest_algorithm *digest,
-				   void *ctx, const void *data, size_t len ) {
+static inline __attribute__ (( always_inline )) void
+digest_update ( struct digest_algorithm *digest, void *ctx,
+		const void *data, size_t len ) {
 	digest->update ( ctx, data, len );
 }
 
-static inline void digest_final ( struct digest_algorithm *digest,
-				  void *ctx, void *out ) {
+static inline __attribute__ (( always_inline )) void
+digest_final ( struct digest_algorithm *digest, void *ctx, void *out ) {
 	digest->final ( ctx, out );
 }
 
-static inline int cipher_setkey ( struct cipher_algorithm *cipher,
-				  void *ctx, const void *key, size_t keylen ) {
+static inline __attribute__ (( always_inline )) int
+cipher_setkey ( struct cipher_algorithm *cipher, void *ctx,
+		const void *key, size_t keylen ) {
 	return cipher->setkey ( ctx, key, keylen );
 }
 
-static inline void cipher_setiv ( struct cipher_algorithm *cipher,
-				  void *ctx, const void *iv, size_t ivlen ) {
+static inline __attribute__ (( always_inline )) void
+cipher_setiv ( struct cipher_algorithm *cipher, void *ctx,
+	       const void *iv, size_t ivlen ) {
 	cipher->setiv ( ctx, iv, ivlen );
 }
 
-static inline void cipher_encrypt ( struct cipher_algorithm *cipher,
-				    void *ctx, const void *src, void *dst,
-				    size_t len ) {
+static inline __attribute__ (( always_inline )) void
+cipher_encrypt ( struct cipher_algorithm *cipher, void *ctx,
+		 const void *src, void *dst, size_t len ) {
 	cipher->encrypt ( ctx, src, dst, len );
 }
 #define cipher_encrypt( cipher, ctx, src, dst, len ) do {		\
@@ -247,9 +250,9 @@ static inline void cipher_encrypt ( struct cipher_algorithm *cipher,
 	cipher_encrypt ( (cipher), (ctx), (src), (dst), (len) );	\
 	} while ( 0 )
 
-static inline void cipher_decrypt ( struct cipher_algorithm *cipher,
-				    void *ctx, const void *src, void *dst,
-				    size_t len ) {
+static inline __attribute__ (( always_inline )) void
+cipher_decrypt ( struct cipher_algorithm *cipher, void *ctx,
+		 const void *src, void *dst, size_t len ) {
 	cipher->decrypt ( ctx, src, dst, len );
 }
 #define cipher_decrypt( cipher, ctx, src, dst, len ) do {		\
@@ -257,71 +260,79 @@ static inline void cipher_decrypt ( struct cipher_algorithm *cipher,
 	cipher_decrypt ( (cipher), (ctx), (src), (dst), (len) );	\
 	} while ( 0 )
 
-static inline void cipher_auth ( struct cipher_algorithm *cipher, void *ctx,
-				 void *auth ) {
+static inline __attribute__ (( always_inline )) void
+cipher_auth ( struct cipher_algorithm *cipher, void *ctx, void *auth ) {
 	cipher->auth ( ctx, auth );
 }
 
-static inline int is_stream_cipher ( struct cipher_algorithm *cipher ) {
+static inline __attribute__ (( always_inline )) int
+is_stream_cipher ( struct cipher_algorithm *cipher ) {
 	return ( cipher->blocksize == 1 );
 }
 
-static inline int is_block_cipher ( struct cipher_algorithm *cipher ) {
+static inline __attribute__ (( always_inline )) int
+is_block_cipher ( struct cipher_algorithm *cipher ) {
 	return ( cipher->blocksize > 1 );
 }
 
-static inline int is_auth_cipher ( struct cipher_algorithm *cipher ) {
+static inline __attribute__ (( always_inline )) int
+is_auth_cipher ( struct cipher_algorithm *cipher ) {
 	return cipher->authsize;
 }
 
-static inline int pubkey_init ( struct pubkey_algorithm *pubkey, void *ctx,
-				const void *key, size_t key_len ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_init ( struct pubkey_algorithm *pubkey, void *ctx,
+	      const void *key, size_t key_len ) {
 	return pubkey->init ( ctx, key, key_len );
 }
 
-static inline size_t pubkey_max_len ( struct pubkey_algorithm *pubkey,
-				      void *ctx ) {
+static inline __attribute__ (( always_inline )) size_t
+pubkey_max_len ( struct pubkey_algorithm *pubkey, void *ctx ) {
 	return pubkey->max_len ( ctx );
 }
 
-static inline int pubkey_encrypt ( struct pubkey_algorithm *pubkey, void *ctx,
-				   const void *data, size_t len, void *out ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_encrypt ( struct pubkey_algorithm *pubkey, void *ctx,
+		 const void *data, size_t len, void *out ) {
 	return pubkey->encrypt ( ctx, data, len, out );
 }
 
-static inline int pubkey_decrypt ( struct pubkey_algorithm *pubkey, void *ctx,
-				   const void *data, size_t len, void *out ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_decrypt ( struct pubkey_algorithm *pubkey, void *ctx,
+		 const void *data, size_t len, void *out ) {
 	return pubkey->decrypt ( ctx, data, len, out );
 }
 
-static inline int pubkey_sign ( struct pubkey_algorithm *pubkey, void *ctx,
-				struct digest_algorithm *digest,
-				const void *value, void *signature ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_sign ( struct pubkey_algorithm *pubkey, void *ctx,
+	      struct digest_algorithm *digest, const void *value,
+	      void *signature ) {
 	return pubkey->sign ( ctx, digest, value, signature );
 }
 
-static inline int pubkey_verify ( struct pubkey_algorithm *pubkey, void *ctx,
-				  struct digest_algorithm *digest,
-				  const void *value, const void *signature,
-				  size_t signature_len ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_verify ( struct pubkey_algorithm *pubkey, void *ctx,
+		struct digest_algorithm *digest, const void *value,
+		const void *signature, size_t signature_len ) {
 	return pubkey->verify ( ctx, digest, value, signature, signature_len );
 }
 
-static inline void pubkey_final ( struct pubkey_algorithm *pubkey, void *ctx ) {
+static inline __attribute__ (( always_inline )) void
+pubkey_final ( struct pubkey_algorithm *pubkey, void *ctx ) {
 	pubkey->final ( ctx );
 }
 
-static inline int pubkey_match ( struct pubkey_algorithm *pubkey,
-				 const void *private_key,
-				 size_t private_key_len, const void *public_key,
-				 size_t public_key_len ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_match ( struct pubkey_algorithm *pubkey,
+	       const void *private_key, size_t private_key_len,
+	       const void *public_key, size_t public_key_len ) {
 	return pubkey->match ( private_key, private_key_len, public_key,
 			       public_key_len );
 }
 
-static inline int elliptic_multiply ( struct elliptic_curve *curve,
-				      const void *base, const void *scalar,
-				      void *result ) {
+static inline __attribute__ (( always_inline )) int
+elliptic_multiply ( struct elliptic_curve *curve,
+		    const void *base, const void *scalar, void *result ) {
 	return curve->multiply ( base, scalar, result );
 }
 

From e5f3ba0ca773e8ea4dcfec6e10f18a06d14e79e0 Mon Sep 17 00:00:00 2001
From: Geert Stappers <stappers@stappers.it>
Date: Sun, 18 Feb 2024 12:29:59 +0100
Subject: [PATCH 081/237] [drivers] Sort PCI_ROM() entries numerically

Done with the help of this Perl script:

$MARKER = 'PCI_ROM';  # a regex
$AB = 1;  # At Begin
@HEAD = ();
@ITEMS = ();
@TAIL = ();

foreach $fn (@ARGV) {
    open(IN, $fn) or die "Can't open file '$fn': $!\n";
    while (<IN>) {
        if (/$MARKER/) {
            push @ITEMS, $_;
            $AB = 0;  # not anymore at begin
        }
        else {
            if ($AB) {
                push @HEAD, $_;
            }
            else {
                push @TAIL, $_;
            }
        }
    }
} continue {
    close IN;
    open(OUT, ">$fn") or die "Can't open file '$fn' for output: $!\n";
    print OUT @HEAD;
    print OUT sort @ITEMS;
    print OUT @TAIL;
    close OUT;
    # For a next file
    $AB = 1;
    @HEAD = ();
    @ITEMS = ();
    @TAIL = ();
}

Executed that script while src/drivers/ as current working directory,
provided '$(grep -rl PCI_ROM)' as argument.

Signed-off-by: Geert Stappers <stappers@stappers.it>
---
 src/drivers/infiniband/arbel.c    |   2 +-
 src/drivers/infiniband/hermon.c   |   9 +--
 src/drivers/net/3c595.c           |   4 +-
 src/drivers/net/3c90x.c           |  12 +--
 src/drivers/net/ath/ath5k/ath5k.c |   8 +-
 src/drivers/net/b44.c             |   2 +-
 src/drivers/net/bnxt/bnxt.c       |  22 +++---
 src/drivers/net/davicom.c         |   2 +-
 src/drivers/net/dmfe.c            |   2 +-
 src/drivers/net/eepro100.c        |  10 +--
 src/drivers/net/forcedeth.c       |  26 +++----
 src/drivers/net/ns8390.c          |  12 +--
 src/drivers/net/pcnet32.c         |   2 +-
 src/drivers/net/prism2_plx.c      |   4 +-
 src/drivers/net/rhine.c           |   4 +-
 src/drivers/net/rtl818x/rtl8180.c |   2 +-
 src/drivers/net/sky2.c            |   2 +-
 src/drivers/net/sundance.c        |   2 +-
 src/drivers/net/tg3/tg3.c         | 118 +++++++++++++++---------------
 src/drivers/net/tlan.c            |  14 ++--
 src/drivers/net/tulip.c           |  28 +++----
 21 files changed, 143 insertions(+), 144 deletions(-)

diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c
index 24c0b53b0..8be06d93d 100644
--- a/src/drivers/infiniband/arbel.c
+++ b/src/drivers/infiniband/arbel.c
@@ -3139,8 +3139,8 @@ static void arbel_remove ( struct pci_device *pci ) {
 }
 
 static struct pci_device_id arbel_nics[] = {
-	PCI_ROM ( 0x15b3, 0x6282, "mt25218", "MT25218 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x6274, "mt25204", "MT25204 HCA driver", 0 ),
+	PCI_ROM ( 0x15b3, 0x6282, "mt25218", "MT25218 HCA driver", 0 ),
 };
 
 struct pci_driver arbel_driver __pci_driver = {
diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c
index 6fc7d8bd4..e5c3544fa 100644
--- a/src/drivers/infiniband/hermon.c
+++ b/src/drivers/infiniband/hermon.c
@@ -4214,6 +4214,9 @@ static void hermon_bofm_remove ( struct pci_device *pci ) {
 }
 
 static struct pci_device_id hermon_nics[] = {
+	/* Mellanox ConnectX-3 VPI (ethernet + infiniband) */
+	PCI_ROM ( 0x15b3, 0x1003, "mt4099", "ConnectX-3 HCA driver", 0 ),
+	PCI_ROM ( 0x15b3, 0x1007, "mt4103", "ConnectX-3 Pro HCA driver", 0 ),
 	/* Mellanox ConnectX VPI (ethernet + infiniband) */
 	PCI_ROM ( 0x15b3, 0x6340, "mt25408", "MT25408 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x634a, "mt25418", "MT25418 HCA driver", 0 ),
@@ -4226,17 +4229,13 @@ static struct pci_device_id hermon_nics[] = {
 	PCI_ROM ( 0x15b3, 0x6732, "mt26418", "MT26418 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x673c, "mt26428", "MT26428 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x6746, "mt26438", "MT26438 HCA driver", 0 ),
-	PCI_ROM ( 0x15b3, 0x6778, "mt26488", "MT26488 HCA driver", 0 ),
 
 	/* Mellanox ConnectX-2 EN (ethernet only) */
 	PCI_ROM ( 0x15b3, 0x6750, "mt26448", "MT26448 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x675a, "mt26458", "MT26458 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x6764, "mt26468", "MT26468 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x676e, "mt26478", "MT26478 HCA driver", 0 ),
-
-	/* Mellanox ConnectX-3 VPI (ethernet + infiniband) */
-	PCI_ROM ( 0x15b3, 0x1003, "mt4099", "ConnectX-3 HCA driver", 0 ),
-	PCI_ROM ( 0x15b3, 0x1007, "mt4103", "ConnectX-3 Pro HCA driver", 0 ),
+	PCI_ROM ( 0x15b3, 0x6778, "mt26488", "MT26488 HCA driver", 0 ),
 };
 
 struct pci_driver hermon_driver __pci_driver = {
diff --git a/src/drivers/net/3c595.c b/src/drivers/net/3c595.c
index 92d38cfc5..c69831005 100644
--- a/src/drivers/net/3c595.c
+++ b/src/drivers/net/3c595.c
@@ -523,10 +523,12 @@ static struct nic_operations t595_operations = {
 };
 
 static struct pci_device_id t595_nics[] = {
+PCI_ROM(0x10b7, 0x4500, "3c450-1",         "3Com450 HomePNA Tornado", 0),
 PCI_ROM(0x10b7, 0x5900, "3c590",           "3Com590", 0),		/* Vortex 10Mbps */
 PCI_ROM(0x10b7, 0x5950, "3c595",           "3Com595", 0),		/* Vortex 100baseTx */
 PCI_ROM(0x10b7, 0x5951, "3c595-1",         "3Com595", 0),		/* Vortex 100baseT4 */
 PCI_ROM(0x10b7, 0x5952, "3c595-2",         "3Com595", 0),		/* Vortex 100base-MII */
+PCI_ROM(0x10b7, 0x7646, "3csoho100-tx-1",  "3CSOHO100-TX", 0),	/* Hurricane */
 PCI_ROM(0x10b7, 0x9000, "3c900-tpo",       "3Com900-TPO", 0),	/* 10 Base TPO */
 PCI_ROM(0x10b7, 0x9001, "3c900-t4",        "3Com900-Combo", 0),	/* 10/100 T4 */
 PCI_ROM(0x10b7, 0x9004, "3c900b-tpo",      "3Com900B-TPO", 0),	/* 10 Base TPO */
@@ -535,8 +537,6 @@ PCI_ROM(0x10b7, 0x9006, "3c900b-tpb2",     "3Com900B-2/T", 0),	/* 10 Base TP and
 PCI_ROM(0x10b7, 0x900a, "3c900b-fl",       "3Com900B-FL", 0),	/* 10 Base F */
 PCI_ROM(0x10b7, 0x9800, "3c980-cyclone-1", "3Com980-Cyclone", 0),	/* Cyclone */
 PCI_ROM(0x10b7, 0x9805, "3c9805-1",        "3Com9805", 0),		/* Dual Port Server Cyclone */
-PCI_ROM(0x10b7, 0x7646, "3csoho100-tx-1",  "3CSOHO100-TX", 0),	/* Hurricane */
-PCI_ROM(0x10b7, 0x4500, "3c450-1",         "3Com450 HomePNA Tornado", 0),
 };
 
 PCI_DRIVER ( t595_driver, t595_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/3c90x.c b/src/drivers/net/3c90x.c
index 1b8190c46..a94473efe 100644
--- a/src/drivers/net/3c90x.c
+++ b/src/drivers/net/3c90x.c
@@ -955,16 +955,20 @@ static int a3c90x_probe(struct pci_device *pci)
 
 static struct pci_device_id a3c90x_nics[] = {
 /* Original 90x revisions: */
+	PCI_ROM(0x10b7, 0x1201, "3c982a", "3Com982A", 0),
+	PCI_ROM(0x10b7, 0x1202, "3c982b", "3Com982B", 0),
+	PCI_ROM(0x10b7, 0x4500, "3c450", "3Com450 HomePNA Tornado", 0),
 	PCI_ROM(0x10b7, 0x6055, "3c556", "3C556", 0),	/* Huricane */
+	PCI_ROM(0x10b7, 0x7646, "3csoho100-tx", "3CSOHO100-TX", 0),	/* Hurricane */
 	PCI_ROM(0x10b7, 0x9000, "3c905-tpo", "3Com900-TPO", 0),	/* 10 Base TPO */
 	PCI_ROM(0x10b7, 0x9001, "3c905-t4", "3Com900-Combo", 0),	/* 10/100 T4 */
-	PCI_ROM(0x10b7, 0x9050, "3c905-tpo100", "3Com905-TX", 0),	/* 100 Base TX / 10/100 TPO */
-	PCI_ROM(0x10b7, 0x9051, "3c905-combo", "3Com905-T4", 0),	/* 100 Base T4 / 10 Base Combo */
 /* Newer 90xB revisions: */
 	PCI_ROM(0x10b7, 0x9004, "3c905b-tpo", "3Com900B-TPO", 0),	/* 10 Base TPO */
 	PCI_ROM(0x10b7, 0x9005, "3c905b-combo", "3Com900B-Combo", 0),	/* 10 Base Combo */
 	PCI_ROM(0x10b7, 0x9006, "3c905b-tpb2", "3Com900B-2/T", 0),	/* 10 Base TP and Base2 */
 	PCI_ROM(0x10b7, 0x900a, "3c905b-fl", "3Com900B-FL", 0),	/* 10 Base FL */
+	PCI_ROM(0x10b7, 0x9050, "3c905-tpo100", "3Com905-TX", 0),	/* 100 Base TX / 10/100 TPO */
+	PCI_ROM(0x10b7, 0x9051, "3c905-combo", "3Com905-T4", 0),	/* 100 Base T4 / 10 Base Combo */
 	PCI_ROM(0x10b7, 0x9055, "3c905b-tpo100", "3Com905B-TX", 0),	/* 10/100 TPO */
 	PCI_ROM(0x10b7, 0x9056, "3c905b-t4", "3Com905B-T4", 0),	/* 10/100 T4 */
 	PCI_ROM(0x10b7, 0x9058, "3c905b-9058", "3Com905B-9058", 0),	/* Cyclone 10/100/BNC */
@@ -975,10 +979,6 @@ static struct pci_device_id a3c90x_nics[] = {
 	PCI_ROM(0x10b7, 0x9210, "3c920b-emb-wnm", "3Com20B-EMB WNM", 0),
 	PCI_ROM(0x10b7, 0x9800, "3c980", "3Com980-Cyclone", 0),	/* Cyclone */
 	PCI_ROM(0x10b7, 0x9805, "3c9805", "3Com9805", 0),	/* Dual Port Server Cyclone */
-	PCI_ROM(0x10b7, 0x7646, "3csoho100-tx", "3CSOHO100-TX", 0),	/* Hurricane */
-	PCI_ROM(0x10b7, 0x4500, "3c450", "3Com450 HomePNA Tornado", 0),
-	PCI_ROM(0x10b7, 0x1201, "3c982a", "3Com982A", 0),
-	PCI_ROM(0x10b7, 0x1202, "3c982b", "3Com982B", 0),
 };
 
 struct pci_driver a3c90x_driver __pci_driver = {
diff --git a/src/drivers/net/ath/ath5k/ath5k.c b/src/drivers/net/ath/ath5k/ath5k.c
index e43eb0aaf..643884d46 100644
--- a/src/drivers/net/ath/ath5k/ath5k.c
+++ b/src/drivers/net/ath/ath5k/ath5k.c
@@ -65,14 +65,11 @@ FILE_LICENCE ( BSD3 );
 
 /* Known PCI ids */
 static struct pci_device_id ath5k_nics[] = {
-	PCI_ROM(0x168c, 0x0207, "ath5210e", "Atheros 5210 early", AR5K_AR5210),
+	PCI_ROM(0x10b7, 0x0013, "rdag675", "3com 3CRDAG675", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x0007, "ath5210", "Atheros 5210", AR5K_AR5210),
 	PCI_ROM(0x168c, 0x0011, "ath5311", "Atheros 5311 (AHB)", AR5K_AR5211),
 	PCI_ROM(0x168c, 0x0012, "ath5211", "Atheros 5211", AR5K_AR5211),
 	PCI_ROM(0x168c, 0x0013, "ath5212", "Atheros 5212", AR5K_AR5212),
-	PCI_ROM(0xa727, 0x0013, "ath5212c","3com Ath 5212", AR5K_AR5212),
-	PCI_ROM(0x10b7, 0x0013, "rdag675", "3com 3CRDAG675", AR5K_AR5212),
-	PCI_ROM(0x168c, 0x1014, "ath5212m", "Ath 5212 miniPCI", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x0014, "ath5212x14", "Atheros 5212 x14", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x0015, "ath5212x15", "Atheros 5212 x15", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x0016, "ath5212x16", "Atheros 5212 x16", AR5K_AR5212),
@@ -83,6 +80,9 @@ static struct pci_device_id ath5k_nics[] = {
 	PCI_ROM(0x168c, 0x001b, "ath5413", "Atheros 5413 Eagle", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x001c, "ath5212e", "Atheros 5212 PCI-E", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x001d, "ath2417", "Atheros 2417 Nala", AR5K_AR5212),
+	PCI_ROM(0x168c, 0x0207, "ath5210e", "Atheros 5210 early", AR5K_AR5210),
+	PCI_ROM(0x168c, 0x1014, "ath5212m", "Ath 5212 miniPCI", AR5K_AR5212),
+	PCI_ROM(0xa727, 0x0013, "ath5212c","3com Ath 5212", AR5K_AR5212),
 };
 
 #define ATH5K_SPMBL_NO   1
diff --git a/src/drivers/net/b44.c b/src/drivers/net/b44.c
index 1ca7e2e53..30ece5574 100644
--- a/src/drivers/net/b44.c
+++ b/src/drivers/net/b44.c
@@ -945,8 +945,8 @@ static struct net_device_operations b44_operations = {
 
 
 static struct pci_device_id b44_nics[] = {
-	PCI_ROM(0x14e4, 0x4401, "BCM4401", "BCM4401", 0),
 	PCI_ROM(0x14e4, 0x170c, "BCM4401-B0", "BCM4401-B0", 0),
+	PCI_ROM(0x14e4, 0x4401, "BCM4401", "BCM4401", 0),
 	PCI_ROM(0x14e4, 0x4402, "BCM4401-B1", "BCM4401-B1", 0),
 };
 
diff --git a/src/drivers/net/bnxt/bnxt.c b/src/drivers/net/bnxt/bnxt.c
index a99f29ec0..a127f6cef 100644
--- a/src/drivers/net/bnxt/bnxt.c
+++ b/src/drivers/net/bnxt/bnxt.c
@@ -24,6 +24,11 @@ static int bnxt_rx_complete ( struct net_device *dev, struct rx_pkt_cmpl *rx );
 void bnxt_link_evt ( struct bnxt *bp, struct hwrm_async_event_cmpl *evt );
 
 static struct pci_device_id bnxt_nics[] = {
+	PCI_ROM( 0x14e4, 0x1604, "14e4-1604", "14e4-1604", 0 ),
+	PCI_ROM( 0x14e4, 0x1605, "14e4-1605", "14e4-1605", 0 ),
+	PCI_ROM( 0x14e4, 0x1606, "14e4-1606", "14e4-1606", 0 ),
+	PCI_ROM( 0x14e4, 0x1609, "14e4-1609", "14e4-1609", 0 ),
+	PCI_ROM( 0x14e4, 0x1614, "14e4-1614", "14e4-1614", 0 ),
 	PCI_ROM( 0x14e4, 0x16c0, "14e4-16C0", "14e4-16C0", 0 ),
 	PCI_ROM( 0x14e4, 0x16c1, "14e4-16C1", "14e4-16C1", BNXT_FLAG_PCI_VF ),
 	PCI_ROM( 0x14e4, 0x16c8, "14e4-16C8", "14e4-16C8", 0 ),
@@ -62,27 +67,22 @@ static struct pci_device_id bnxt_nics[] = {
 	PCI_ROM( 0x14e4, 0x16ef, "14e4-16EF", "14e4-16EF", 0 ),
 	PCI_ROM( 0x14e4, 0x16f0, "14e4-16F0", "14e4-16F0", 0 ),
 	PCI_ROM( 0x14e4, 0x16f1, "14e4-16F1", "14e4-16F1", 0 ),
-	PCI_ROM( 0x14e4, 0x1604, "14e4-1604", "14e4-1604", 0 ),
-	PCI_ROM( 0x14e4, 0x1605, "14e4-1605", "14e4-1605", 0 ),
-	PCI_ROM( 0x14e4, 0x1606, "14e4-1606", "14e4-1606", 0 ),
-	PCI_ROM( 0x14e4, 0x1609, "14e4-1609", "14e4-1609", 0 ),
-	PCI_ROM( 0x14e4, 0x1614, "14e4-1614", "14e4-1614", 0 ),
-	PCI_ROM( 0x14e4, 0xd802, "14e4-D802", "14e4-D802", 0 ),
-	PCI_ROM( 0x14e4, 0xd804, "14e4-D804", "14e4-D804", 0 ),
 	PCI_ROM( 0x14e4, 0x1750, "14e4-1750", "14e4-1750", 0 ),
-	PCI_ROM( 0x14e4, 0x1802, "14e4-1802", "14e4-1802", 0 ),
-	PCI_ROM( 0x14e4, 0x1805, "14e4-1805", "14e4-1805", 0 ),
 	PCI_ROM( 0x14e4, 0x1751, "14e4-1751", "14e4-1751", 0 ),
-	PCI_ROM( 0x14e4, 0x1801, "14e4-1801", "14e4-1801", 0 ),
-	PCI_ROM( 0x14e4, 0x1804, "14e4-1804", "14e4-1804", 0 ),
 	PCI_ROM( 0x14e4, 0x1752, "14e4-1752", "14e4-1752", 0 ),
 	PCI_ROM( 0x14e4, 0x1760, "14e4-1760", "14e4-1760", 0 ),
 	PCI_ROM( 0x14e4, 0x1800, "14e4-1800", "14e4-1800", 0 ),
+	PCI_ROM( 0x14e4, 0x1801, "14e4-1801", "14e4-1801", 0 ),
+	PCI_ROM( 0x14e4, 0x1802, "14e4-1802", "14e4-1802", 0 ),
 	PCI_ROM( 0x14e4, 0x1803, "14e4-1803", "14e4-1803", 0 ),
+	PCI_ROM( 0x14e4, 0x1804, "14e4-1804", "14e4-1804", 0 ),
+	PCI_ROM( 0x14e4, 0x1805, "14e4-1805", "14e4-1805", 0 ),
 	PCI_ROM( 0x14e4, 0x1806, "14e4-1806", "14e4-1806", BNXT_FLAG_PCI_VF ),
 	PCI_ROM( 0x14e4, 0x1807, "14e4-1807", "14e4-1807", BNXT_FLAG_PCI_VF ),
 	PCI_ROM( 0x14e4, 0x1808, "14e4-1808", "14e4-1808", BNXT_FLAG_PCI_VF ),
 	PCI_ROM( 0x14e4, 0x1809, "14e4-1809", "14e4-1809", BNXT_FLAG_PCI_VF ),
+	PCI_ROM( 0x14e4, 0xd802, "14e4-D802", "14e4-D802", 0 ),
+	PCI_ROM( 0x14e4, 0xd804, "14e4-D804", "14e4-D804", 0 ),
 };
 
 /**
diff --git a/src/drivers/net/davicom.c b/src/drivers/net/davicom.c
index 9d3d8b915..0c96796df 100644
--- a/src/drivers/net/davicom.c
+++ b/src/drivers/net/davicom.c
@@ -689,9 +689,9 @@ static struct nic_operations davicom_operations = {
 };
 
 static struct pci_device_id davicom_nics[] = {
+PCI_ROM(0x1282, 0x9009, "davicom9009", "Davicom 9009", 0),
 PCI_ROM(0x1282, 0x9100, "davicom9100", "Davicom 9100", 0),
 PCI_ROM(0x1282, 0x9102, "davicom9102", "Davicom 9102", 0),
-PCI_ROM(0x1282, 0x9009, "davicom9009", "Davicom 9009", 0),
 PCI_ROM(0x1282, 0x9132, "davicom9132", "Davicom 9132", 0),	/* Needs probably some fixing */
 };
 
diff --git a/src/drivers/net/dmfe.c b/src/drivers/net/dmfe.c
index 2ea0d2b2b..53b05815b 100644
--- a/src/drivers/net/dmfe.c
+++ b/src/drivers/net/dmfe.c
@@ -1208,9 +1208,9 @@ static struct nic_operations dmfe_operations = {
 };
 
 static struct pci_device_id dmfe_nics[] = {
+	PCI_ROM(0x1282, 0x9009, "dmfe9009", "Davicom 9009", 0),
 	PCI_ROM(0x1282, 0x9100, "dmfe9100", "Davicom 9100", 0),
 	PCI_ROM(0x1282, 0x9102, "dmfe9102", "Davicom 9102", 0),
-	PCI_ROM(0x1282, 0x9009, "dmfe9009", "Davicom 9009", 0),
 	PCI_ROM(0x1282, 0x9132, "dmfe9132", "Davicom 9132", 0),	/* Needs probably some fixing */
 };
 
diff --git a/src/drivers/net/eepro100.c b/src/drivers/net/eepro100.c
index a0551a89b..49b00d443 100644
--- a/src/drivers/net/eepro100.c
+++ b/src/drivers/net/eepro100.c
@@ -1126,8 +1126,12 @@ PCI_ROM(0x8086, 0x103b, "82562etb",      "Intel PRO100 VE 82562ETB", 0),
 PCI_ROM(0x8086, 0x103c, "eepro100-103c", "Intel PRO/100 VM Network Connection", 0),
 PCI_ROM(0x8086, 0x103d, "eepro100-103d", "Intel PRO/100 VE Network Connection", 0),
 PCI_ROM(0x8086, 0x103e, "eepro100-103e", "Intel PRO/100 VM Network Connection", 0),
+PCI_ROM(0x8086, 0x1050, "82562ez",       "Intel 82562EZ Network Connection", 0),
 PCI_ROM(0x8086, 0x1051, "prove",         "Intel PRO/100 VE Network Connection", 0),
 PCI_ROM(0x8086, 0x1059, "82551qm",       "Intel PRO/100 M Mobile Connection", 0),
+PCI_ROM(0x8086, 0x1065, "82562-3",       "Intel 82562 based Fast Ethernet Connection", 0),
+PCI_ROM(0x8086, 0x1092, "82562-3",       "Intel Pro/100 VE Network", 0),
+PCI_ROM(0x8086, 0x10fe, "82552",         "Intel 82552 10/100 Network Connection", 0),
 PCI_ROM(0x8086, 0x1209, "82559er",       "Intel EtherExpressPro100 82559ER", 0),
 PCI_ROM(0x8086, 0x1227, "82865",         "Intel 82865 EtherExpress PRO/100A", 0),
 PCI_ROM(0x8086, 0x1228, "82556",         "Intel 82556 EtherExpress PRO/100 Smart", 0),
@@ -1135,13 +1139,9 @@ PCI_ROM(0x8086, 0x1229, "eepro100",      "Intel EtherExpressPro100", 0),
 PCI_ROM(0x8086, 0x2449, "82562em",       "Intel EtherExpressPro100 82562EM", 0),
 PCI_ROM(0x8086, 0x2459, "82562-1",       "Intel 82562 based Fast Ethernet Connection", 0),
 PCI_ROM(0x8086, 0x245d, "82562-2",       "Intel 82562 based Fast Ethernet Connection", 0),
-PCI_ROM(0x8086, 0x1050, "82562ez",       "Intel 82562EZ Network Connection", 0),
-PCI_ROM(0x8086, 0x1065, "82562-3",       "Intel 82562 based Fast Ethernet Connection", 0),
+PCI_ROM(0x8086, 0x27dc, "eepro100-27dc", "Intel 82801G (ICH7) Chipset Ethernet Controller", 0),
 PCI_ROM(0x8086, 0x5200, "eepro100-5200", "Intel EtherExpress PRO/100 Intelligent Server", 0),
 PCI_ROM(0x8086, 0x5201, "eepro100-5201", "Intel EtherExpress PRO/100 Intelligent Server", 0),
-PCI_ROM(0x8086, 0x1092, "82562-3",       "Intel Pro/100 VE Network", 0),
-PCI_ROM(0x8086, 0x27dc, "eepro100-27dc", "Intel 82801G (ICH7) Chipset Ethernet Controller", 0),
-PCI_ROM(0x8086, 0x10fe, "82552",         "Intel 82552 10/100 Network Connection", 0),
 };
 
 /* Cards with device ids 0x1030 to 0x103F, 0x2449, 0x2459 or 0x245D might need
diff --git a/src/drivers/net/forcedeth.c b/src/drivers/net/forcedeth.c
index ec3a5bdb2..b4019d04e 100644
--- a/src/drivers/net/forcedeth.c
+++ b/src/drivers/net/forcedeth.c
@@ -1928,17 +1928,17 @@ forcedeth_remove ( struct pci_device *pdev )
 }
 
 static struct pci_device_id forcedeth_nics[] = {
-	PCI_ROM(0x10DE, 0x01C3, "nForce", "nForce Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
-	PCI_ROM(0x10DE, 0x0066, "nForce2", "nForce2 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
-	PCI_ROM(0x10DE, 0x00D6, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
-	PCI_ROM(0x10DE, 0x0086, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
-	PCI_ROM(0x10DE, 0x008C, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
-	PCI_ROM(0x10DE, 0x00E6, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
-	PCI_ROM(0x10DE, 0x00DF, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
-	PCI_ROM(0x10DE, 0x0056, "CK804", "CK804 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
-	PCI_ROM(0x10DE, 0x0057, "CK804", "CK804 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
 	PCI_ROM(0x10DE, 0x0037, "MCP04", "MCP04 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
 	PCI_ROM(0x10DE, 0x0038, "MCP04", "MCP04 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
+	PCI_ROM(0x10DE, 0x0056, "CK804", "CK804 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
+	PCI_ROM(0x10DE, 0x0057, "CK804", "CK804 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
+	PCI_ROM(0x10DE, 0x0066, "nForce2", "nForce2 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
+	PCI_ROM(0x10DE, 0x0086, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
+	PCI_ROM(0x10DE, 0x008C, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
+	PCI_ROM(0x10DE, 0x00D6, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
+	PCI_ROM(0x10DE, 0x00DF, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
+	PCI_ROM(0x10DE, 0x00E6, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
+	PCI_ROM(0x10DE, 0x01C3, "nForce", "nForce Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
 	PCI_ROM(0x10DE, 0x0268, "MCP51", "MCP51 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_STATISTICS_V1|DEV_NEED_LOW_POWER_FIX),
 	PCI_ROM(0x10DE, 0x0269, "MCP51", "MCP51 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_STATISTICS_V1|DEV_NEED_LOW_POWER_FIX),
 	PCI_ROM(0x10DE, 0x0372, "MCP55", "MCP55 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_VLAN|DEV_HAS_MSI|DEV_HAS_MSI_X| DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V1| DEV_HAS_STATISTICS_V2|DEV_HAS_TEST_EXTENDED| DEV_HAS_MGMT_UNIT|DEV_NEED_TX_LIMIT|DEV_NEED_MSI_FIX),
@@ -1955,14 +1955,14 @@ static struct pci_device_id forcedeth_nics[] = {
 	PCI_ROM(0x10DE, 0x054D, "MCP67", "MCP67 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x054E, "MCP67", "MCP67 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x054F, "MCP67", "MCP67 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
-	PCI_ROM(0x10DE, 0x07DC, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
-	PCI_ROM(0x10DE, 0x07DD, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
-	PCI_ROM(0x10DE, 0x07DE, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
-	PCI_ROM(0x10DE, 0x07DF, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0760, "MCP77", "MCP77 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_CHECKSUM|DEV_HAS_HIGH_DMA| DEV_HAS_MSI|DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V2| DEV_HAS_STATISTICS_V3|DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT| DEV_HAS_CORRECT_MACADDR|DEV_HAS_COLLISION_FIX| DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE|DEV_NEED_PHY_INIT_FIX| DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0761, "MCP77", "MCP77 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_CHECKSUM|DEV_HAS_HIGH_DMA| DEV_HAS_MSI|DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V2| DEV_HAS_STATISTICS_V3|DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT| DEV_HAS_CORRECT_MACADDR|DEV_HAS_COLLISION_FIX| DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE|DEV_NEED_PHY_INIT_FIX| DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0762, "MCP77", "MCP77 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_CHECKSUM|DEV_HAS_HIGH_DMA| DEV_HAS_MSI|DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V2| DEV_HAS_STATISTICS_V3|DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT| DEV_HAS_CORRECT_MACADDR|DEV_HAS_COLLISION_FIX| DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE|DEV_NEED_PHY_INIT_FIX| DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0763, "MCP77", "MCP77 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_CHECKSUM|DEV_HAS_HIGH_DMA| DEV_HAS_MSI|DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V2| DEV_HAS_STATISTICS_V3|DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT| DEV_HAS_CORRECT_MACADDR|DEV_HAS_COLLISION_FIX| DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE|DEV_NEED_PHY_INIT_FIX| DEV_NEED_MSI_FIX),
+	PCI_ROM(0x10DE, 0x07DC, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
+	PCI_ROM(0x10DE, 0x07DD, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
+	PCI_ROM(0x10DE, 0x07DE, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
+	PCI_ROM(0x10DE, 0x07DF, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0AB0, "MCP79", "MCP79 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_MSI|DEV_HAS_POWER_CNTRL| DEV_HAS_PAUSEFRAME_TX_V3|DEV_HAS_STATISTICS_V3| DEV_HAS_TEST_EXTENDED|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE| DEV_NEED_PHY_INIT_FIX|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0AB1, "MCP79", "MCP79 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_MSI|DEV_HAS_POWER_CNTRL| DEV_HAS_PAUSEFRAME_TX_V3|DEV_HAS_STATISTICS_V3| DEV_HAS_TEST_EXTENDED|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE| DEV_NEED_PHY_INIT_FIX|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0AB2, "MCP79", "MCP79 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_MSI|DEV_HAS_POWER_CNTRL| DEV_HAS_PAUSEFRAME_TX_V3|DEV_HAS_STATISTICS_V3| DEV_HAS_TEST_EXTENDED|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE| DEV_NEED_PHY_INIT_FIX|DEV_NEED_MSI_FIX),
diff --git a/src/drivers/net/ns8390.c b/src/drivers/net/ns8390.c
index 0ffc6216b..8e8d8500f 100644
--- a/src/drivers/net/ns8390.c
+++ b/src/drivers/net/ns8390.c
@@ -1006,17 +1006,17 @@ ISA_ROM("ne","NE1000/2000 and clones");
 #ifdef	INCLUDE_NS8390
 static struct pci_device_id nepci_nics[] = {
 /* A few NE2000 PCI clones, list not exhaustive */
-PCI_ROM(0x10ec, 0x8029, "rtl8029",      "Realtek 8029", 0),
-PCI_ROM(0x1186, 0x0300, "dlink-528",    "D-Link DE-528", 0),
 PCI_ROM(0x1050, 0x0940, "winbond940",   "Winbond NE2000-PCI", 0),		/* Winbond 86C940 / 89C940 */
 PCI_ROM(0x1050, 0x5a5a, "winbond940f",  "Winbond W89c940F", 0),		/* Winbond 89C940F */
+PCI_ROM(0x10bd, 0x0e34, "surecom-ne34", "Surecom NE34", 0),
+PCI_ROM(0x10ec, 0x8029, "rtl8029",      "Realtek 8029", 0),
+PCI_ROM(0x1106, 0x0926, "via86c926",    "Via 86c926", 0),
+PCI_ROM(0x1186, 0x0300, "dlink-528",    "D-Link DE-528", 0),
 PCI_ROM(0x11f6, 0x1401, "compexrl2000", "Compex ReadyLink 2000", 0),
-PCI_ROM(0x8e2e, 0x3000, "ktiet32p2",    "KTI ET32P2", 0),
-PCI_ROM(0x4a14, 0x5000, "nv5000sc",     "NetVin NV5000SC", 0),
 PCI_ROM(0x12c3, 0x0058, "holtek80232",  "Holtek HT80232", 0),
 PCI_ROM(0x12c3, 0x5598, "holtek80229",  "Holtek HT80229", 0),
-PCI_ROM(0x10bd, 0x0e34, "surecom-ne34", "Surecom NE34", 0),
-PCI_ROM(0x1106, 0x0926, "via86c926",    "Via 86c926", 0),
+PCI_ROM(0x4a14, 0x5000, "nv5000sc",     "NetVin NV5000SC", 0),
+PCI_ROM(0x8e2e, 0x3000, "ktiet32p2",    "KTI ET32P2", 0),
 };
 
 PCI_DRIVER ( nepci_driver, nepci_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/pcnet32.c b/src/drivers/net/pcnet32.c
index 7da884e5c..a9286d6a4 100644
--- a/src/drivers/net/pcnet32.c
+++ b/src/drivers/net/pcnet32.c
@@ -1149,8 +1149,8 @@ pcnet32_remove ( struct pci_device *pdev )
 
 static struct pci_device_id pcnet32_nics[] = {
 	PCI_ROM(0x1022, 0x2000, "pcnet32", "AMD PCnet/PCI", 0),
-	PCI_ROM(0x1022, 0x2625, "pcnetfastiii", "AMD PCNet FAST III", 0),
 	PCI_ROM(0x1022, 0x2001, "amdhomepna", "AMD PCnet/HomePNA", 0),
+	PCI_ROM(0x1022, 0x2625, "pcnetfastiii", "AMD PCNet FAST III", 0),
 };
 
 struct pci_driver pcnet32_driver __pci_driver = {
diff --git a/src/drivers/net/prism2_plx.c b/src/drivers/net/prism2_plx.c
index a73b0e087..770cf3288 100644
--- a/src/drivers/net/prism2_plx.c
+++ b/src/drivers/net/prism2_plx.c
@@ -104,9 +104,10 @@ static void prism2_plx_disable ( struct nic *nic ) {
 }
 
 static struct pci_device_id prism2_plx_nics[] = {
-PCI_ROM(0x1385, 0x4100, "ma301",         "Netgear MA301", 0),
 PCI_ROM(0x10b7, 0x7770, "3c-airconnect", "3Com AirConnect", 0),
 PCI_ROM(0x111a, 0x1023, "ss1023",        "Siemens SpeedStream SS1023", 0),
+PCI_ROM(0x126c, 0x8030, "emobility",     "Nortel emobility", 0),
+PCI_ROM(0x1385, 0x4100, "ma301",         "Netgear MA301", 0),
 PCI_ROM(0x15e8, 0x0130, "correga",       "Correga", 0),
 PCI_ROM(0x1638, 0x1100, "smc2602w",      "SMC EZConnect SMC2602W", 0),	/* or Eumitcom PCI WL11000, Addtron AWA-100 */
 PCI_ROM(0x16ab, 0x1100, "gl24110p",      "Global Sun Tech GL24110P", 0),
@@ -114,7 +115,6 @@ PCI_ROM(0x16ab, 0x1101, "16ab-1101",     "Unknown", 0),
 PCI_ROM(0x16ab, 0x1102, "wdt11",         "Linksys WDT11", 0),
 PCI_ROM(0x16ec, 0x3685, "usr2415",       "USR 2415", 0),
 PCI_ROM(0xec80, 0xec00, "f5d6000",       "Belkin F5D6000", 0),
-PCI_ROM(0x126c, 0x8030, "emobility",     "Nortel emobility", 0),
 };
 
 PCI_DRIVER ( prism2_plx_driver, prism2_plx_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/rhine.c b/src/drivers/net/rhine.c
index f4d3a2580..fa0876ad7 100644
--- a/src/drivers/net/rhine.c
+++ b/src/drivers/net/rhine.c
@@ -775,10 +775,10 @@ static void rhine_remove ( struct pci_device *pci ) {
 
 /** Rhine PCI device IDs */
 static struct pci_device_id rhine_nics[] = {
-	PCI_ROM ( 0x1106, 0x3065, "dlink-530tx", "VIA VT6102", 0 ),
-	PCI_ROM ( 0x1106, 0x3106, "vt6105", "VIA VT6105", 0 ),
 	PCI_ROM ( 0x1106, 0x3043, "dlink-530tx-old", "VIA VT3043", 0 ),
 	PCI_ROM ( 0x1106, 0x3053, "vt6105m", "VIA VT6105M", 0 ),
+	PCI_ROM ( 0x1106, 0x3065, "dlink-530tx", "VIA VT6102", 0 ),
+	PCI_ROM ( 0x1106, 0x3106, "vt6105", "VIA VT6105", 0 ),
 	PCI_ROM ( 0x1106, 0x6100, "via-rhine-old", "VIA 86C100A", 0 )
 };
 
diff --git a/src/drivers/net/rtl818x/rtl8180.c b/src/drivers/net/rtl818x/rtl8180.c
index 5f97480fa..b3f685419 100644
--- a/src/drivers/net/rtl818x/rtl8180.c
+++ b/src/drivers/net/rtl818x/rtl8180.c
@@ -7,9 +7,9 @@ FILE_LICENCE(GPL2_OR_LATER);
 
 static struct pci_device_id rtl8180_nics[] = {
 	PCI_ROM(0x10ec, 0x8180, "rtl8180", "Realtek 8180", 0),
+	PCI_ROM(0x1186, 0x3300, "dwl510",  "D-Link DWL-510", 0),
 	PCI_ROM(0x1799, 0x6001, "f5d6001", "Belkin F5D6001", 0),
 	PCI_ROM(0x1799, 0x6020, "f5d6020", "Belkin F5D6020", 0),
-	PCI_ROM(0x1186, 0x3300, "dwl510",  "D-Link DWL-510", 0),
 };
 
 struct pci_driver rtl8180_driver __pci_driver = {
diff --git a/src/drivers/net/sky2.c b/src/drivers/net/sky2.c
index 26396585f..4f8ec3e42 100644
--- a/src/drivers/net/sky2.c
+++ b/src/drivers/net/sky2.c
@@ -81,8 +81,8 @@ FILE_LICENCE ( GPL2_ONLY );
 static struct pci_device_id sky2_id_table[] = {
 	PCI_ROM(0x1148, 0x9000, "sk9sxx", "Syskonnect SK-9Sxx", 0),
 	PCI_ROM(0x1148, 0x9e00, "sk9exx", "Syskonnect SK-9Exx", 0),
-	PCI_ROM(0x1186, 0x4b00, "dge560t", "D-Link DGE-560T", 0),
 	PCI_ROM(0x1186, 0x4001, "dge550sx", "D-Link DGE-550SX", 0),
+	PCI_ROM(0x1186, 0x4b00, "dge560t", "D-Link DGE-560T", 0),
 	PCI_ROM(0x1186, 0x4b02, "dge560sx", "D-Link DGE-560SX", 0),
 	PCI_ROM(0x1186, 0x4b03, "dge550t", "D-Link DGE-550T", 0),
 	PCI_ROM(0x11ab, 0x4340, "m88e8021", "Marvell 88E8021", 0),
diff --git a/src/drivers/net/sundance.c b/src/drivers/net/sundance.c
index 9127fa2cd..8eb09b988 100644
--- a/src/drivers/net/sundance.c
+++ b/src/drivers/net/sundance.c
@@ -880,9 +880,9 @@ static void set_rx_mode(struct nic *nic __unused)
 }
 
 static struct pci_device_id sundance_nics[] = {
-	PCI_ROM(0x13f0, 0x0201, "sundance", "ST201 Sundance 'Alta' based Adaptor", 0),
 	PCI_ROM(0x1186, 0x1002, "dfe530txs", "D-Link DFE530TXS (Sundance ST201 Alta)", 0),
 	PCI_ROM(0x13f0, 0x0200, "ip100a", "IC+ IP100A", 0),
+	PCI_ROM(0x13f0, 0x0201, "sundance", "ST201 Sundance 'Alta' based Adaptor", 0),
 };
 
 PCI_DRIVER ( sundance_driver, sundance_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/tg3/tg3.c b/src/drivers/net/tg3/tg3.c
index 559c2d638..05af22d61 100644
--- a/src/drivers/net/tg3/tg3.c
+++ b/src/drivers/net/tg3/tg3.c
@@ -856,88 +856,88 @@ static void tg3_remove_one(struct pci_device *pci)
 }
 
 static struct pci_device_id tg3_nics[] = {
+	PCI_ROM(0x106b, 0x1645, "106b-1645", "106b-1645", 0),
+	PCI_ROM(0x1148, 0x4400, "1148-4400", "1148-4400", 0),
+	PCI_ROM(0x1148, 0x4500, "1148-4500", "1148-4500", 0),
+	PCI_ROM(0x14e4, 0x1600, "14e4-1600", "14e4-1600", 0),
+	PCI_ROM(0x14e4, 0x1601, "14e4-1601", "14e4-1601", 0),
 	PCI_ROM(0x14e4, 0x1644, "14e4-1644", "14e4-1644", 0),
 	PCI_ROM(0x14e4, 0x1645, "14e4-1645", "14e4-1645", 0),
 	PCI_ROM(0x14e4, 0x1646, "14e4-1646", "14e4-1646", 0),
 	PCI_ROM(0x14e4, 0x1647, "14e4-1647", "14e4-1647", 0),
 	PCI_ROM(0x14e4, 0x1648, "14e4-1648", "14e4-1648", 0),
+	PCI_ROM(0x14e4, 0x1649, "14e4-1649", "14e4-1649", 0),
 	PCI_ROM(0x14e4, 0x164d, "14e4-164d", "14e4-164d", 0),
 	PCI_ROM(0x14e4, 0x1653, "14e4-1653", "14e4-1653", 0),
 	PCI_ROM(0x14e4, 0x1654, "14e4-1654", "14e4-1654", 0),
+	PCI_ROM(0x14e4, 0x1655, "14e4-1655", "14e4-1655", 0),
+	PCI_ROM(0x14e4, 0x1656, "14e4-1656", "14e4-1656", 0),
+	PCI_ROM(0x14e4, 0x1657, "14e4-1657", "14e4-1657", 0),
+	PCI_ROM(0x14e4, 0x1659, "14e4-1659", "14e4-1659", 0),
+	PCI_ROM(0x14e4, 0x165a, "14e4-165a", "14e4-165a", 0),
+	PCI_ROM(0x14e4, 0x165b, "14e4-165b", "14e4-165b", 0),
 	PCI_ROM(0x14e4, 0x165d, "14e4-165d", "14e4-165d", 0),
 	PCI_ROM(0x14e4, 0x165e, "14e4-165e", "14e4-165e", 0),
+	PCI_ROM(0x14e4, 0x165f, "14e4-165f", "14e4-165f", 0),
+	PCI_ROM(0x14e4, 0x1668, "14e4-1668", "14e4-1668", 0),
+	PCI_ROM(0x14e4, 0x1669, "14e4-1669", "14e4-1669", 0),
+	PCI_ROM(0x14e4, 0x166a, "14e4-166a", "14e4-166a", 0),
+	PCI_ROM(0x14e4, 0x166b, "14e4-166b", "14e4-166b", 0),
+	PCI_ROM(0x14e4, 0x166e, "14e4-166e", "14e4-166e", 0),
+	PCI_ROM(0x14e4, 0x1672, "14e4-1672", "14e4-1672", 0),
+	PCI_ROM(0x14e4, 0x1673, "14e4-1673", "14e4-1673", 0),
+	PCI_ROM(0x14e4, 0x1674, "14e4-1674", "14e4-1674", 0),
+	PCI_ROM(0x14e4, 0x1677, "14e4-1677", "14e4-1677", 0),
+	PCI_ROM(0x14e4, 0x1678, "14e4-1678", "14e4-1678", 0),
+	PCI_ROM(0x14e4, 0x1679, "14e4-1679", "14e4-1679", 0),
+	PCI_ROM(0x14e4, 0x167a, "14e4-167a", "14e4-167a", 0),
+	PCI_ROM(0x14e4, 0x167b, "14e4-167b", "14e4-167b", 0),
+	PCI_ROM(0x14e4, 0x167d, "14e4-167d", "14e4-167d", 0),
+	PCI_ROM(0x14e4, 0x167e, "14e4-167e", "14e4-167e", 0),
+	PCI_ROM(0x14e4, 0x167f, "14e4-167f", "14e4-167f", 0),
+	PCI_ROM(0x14e4, 0x1680, "14e4-1680", "14e4-1680", 0),
+	PCI_ROM(0x14e4, 0x1681, "14e4-1681", "14e4-1681", 0),
+	PCI_ROM(0x14e4, 0x1682, "14e4-1682", "14e4-1682", 0),
+	PCI_ROM(0x14e4, 0x1684, "14e4-1684", "14e4-1684", 0),
+	PCI_ROM(0x14e4, 0x1686, "14e4-1686", "14e4-1686", 0),
+	PCI_ROM(0x14e4, 0x1688, "14e4-1688", "14e4-1688", 0),
+	PCI_ROM(0x14e4, 0x1689, "14e4-1689", "14e4-1689", 0),
+	PCI_ROM(0x14e4, 0x1690, "14e4-1690", "14e4-1690", 0),
+	PCI_ROM(0x14e4, 0x1691, "14e4-1691", "14e4-1691", 0),
+	PCI_ROM(0x14e4, 0x1692, "14e4-1692", "14e4-1692", 0),
+	PCI_ROM(0x14e4, 0x1693, "14e4-1693", "14e4-1693", 0),
+	PCI_ROM(0x14e4, 0x1694, "14e4-1694", "14e4-1694", 0),
+	PCI_ROM(0x14e4, 0x1696, "14e4-1696", "14e4-1696", 0),
+	PCI_ROM(0x14e4, 0x1698, "14e4-1698", "14e4-1698", 0),
+	PCI_ROM(0x14e4, 0x1699, "14e4-1699", "14e4-1699", 0),
+	PCI_ROM(0x14e4, 0x169a, "14e4-169a", "14e4-169a", 0),
+	PCI_ROM(0x14e4, 0x169b, "14e4-169b", "14e4-169b", 0),
+	PCI_ROM(0x14e4, 0x169c, "14e4-169c", "14e4-169c", 0),
+	PCI_ROM(0x14e4, 0x169d, "14e4-169d", "14e4-169d", 0),
+	PCI_ROM(0x14e4, 0x16a0, "14e4-16a0", "14e4-16a0", 0),
 	PCI_ROM(0x14e4, 0x16a6, "14e4-16a6", "14e4-16a6", 0),
 	PCI_ROM(0x14e4, 0x16a7, "14e4-16a7", "14e4-16a7", 0),
 	PCI_ROM(0x14e4, 0x16a8, "14e4-16a8", "14e4-16a8", 0),
+	PCI_ROM(0x14e4, 0x16b0, "14e4-16b0", "14e4-16b0", 0),
+	PCI_ROM(0x14e4, 0x16b1, "14e4-16b1", "14e4-16b1", 0),
+	PCI_ROM(0x14e4, 0x16b2, "14e4-16b2", "14e4-16b2", 0),
+	PCI_ROM(0x14e4, 0x16b4, "14e4-16b4", "14e4-16b4", 0),
+	PCI_ROM(0x14e4, 0x16b5, "14e4-16b5", "14e4-16b5", 0),
+	PCI_ROM(0x14e4, 0x16b6, "14e4-16b6", "14e4-16b6", 0),
 	PCI_ROM(0x14e4, 0x16c6, "14e4-16c6", "14e4-16c6", 0),
 	PCI_ROM(0x14e4, 0x16c7, "14e4-16c7", "14e4-16c7", 0),
-	PCI_ROM(0x14e4, 0x1696, "14e4-1696", "14e4-1696", 0),
-	PCI_ROM(0x14e4, 0x169c, "14e4-169c", "14e4-169c", 0),
-	PCI_ROM(0x14e4, 0x169d, "14e4-169d", "14e4-169d", 0),
-	PCI_ROM(0x14e4, 0x170d, "14e4-170d", "14e4-170d", 0),
-	PCI_ROM(0x14e4, 0x170e, "14e4-170e", "14e4-170e", 0),
-	PCI_ROM(0x14e4, 0x1649, "14e4-1649", "14e4-1649", 0),
-	PCI_ROM(0x14e4, 0x166e, "14e4-166e", "14e4-166e", 0),
-	PCI_ROM(0x14e4, 0x1659, "14e4-1659", "14e4-1659", 0),
-	PCI_ROM(0x14e4, 0x165a, "14e4-165a", "14e4-165a", 0),
-	PCI_ROM(0x14e4, 0x1677, "14e4-1677", "14e4-1677", 0),
-	PCI_ROM(0x14e4, 0x167d, "14e4-167d", "14e4-167d", 0),
-	PCI_ROM(0x14e4, 0x167e, "14e4-167e", "14e4-167e", 0),
-	PCI_ROM(0x14e4, 0x1600, "14e4-1600", "14e4-1600", 0),
-	PCI_ROM(0x14e4, 0x1601, "14e4-1601", "14e4-1601", 0),
+	PCI_ROM(0x14e4, 0x16dd, "14e4-16dd", "14e4-16dd", 0),
 	PCI_ROM(0x14e4, 0x16f7, "14e4-16f7", "14e4-16f7", 0),
 	PCI_ROM(0x14e4, 0x16fd, "14e4-16fd", "14e4-16fd", 0),
 	PCI_ROM(0x14e4, 0x16fe, "14e4-16fe", "14e4-16fe", 0),
-	PCI_ROM(0x14e4, 0x167a, "14e4-167a", "14e4-167a", 0),
-	PCI_ROM(0x14e4, 0x1672, "14e4-1672", "14e4-1672", 0),
-	PCI_ROM(0x14e4, 0x167b, "14e4-167b", "14e4-167b", 0),
-	PCI_ROM(0x14e4, 0x1673, "14e4-1673", "14e4-1673", 0),
-	PCI_ROM(0x14e4, 0x1674, "14e4-1674", "14e4-1674", 0),
-	PCI_ROM(0x14e4, 0x169a, "14e4-169a", "14e4-169a", 0),
-	PCI_ROM(0x14e4, 0x169b, "14e4-169b", "14e4-169b", 0),
-	PCI_ROM(0x14e4, 0x1693, "14e4-1693", "14e4-1693", 0),
-	PCI_ROM(0x14e4, 0x167f, "14e4-167f", "14e4-167f", 0),
-	PCI_ROM(0x14e4, 0x1668, "14e4-1668", "14e4-1668", 0),
-	PCI_ROM(0x14e4, 0x1669, "14e4-1669", "14e4-1669", 0),
-	PCI_ROM(0x14e4, 0x1678, "14e4-1678", "14e4-1678", 0),
-	PCI_ROM(0x14e4, 0x1679, "14e4-1679", "14e4-1679", 0),
-	PCI_ROM(0x14e4, 0x166a, "14e4-166a", "14e4-166a", 0),
-	PCI_ROM(0x14e4, 0x166b, "14e4-166b", "14e4-166b", 0),
-	PCI_ROM(0x14e4, 0x16dd, "14e4-16dd", "14e4-16dd", 0),
+	PCI_ROM(0x14e4, 0x170d, "14e4-170d", "14e4-170d", 0),
+	PCI_ROM(0x14e4, 0x170e, "14e4-170e", "14e4-170e", 0),
 	PCI_ROM(0x14e4, 0x1712, "14e4-1712", "14e4-1712", 0),
 	PCI_ROM(0x14e4, 0x1713, "14e4-1713", "14e4-1713", 0),
-	PCI_ROM(0x14e4, 0x1698, "14e4-1698", "14e4-1698", 0),
-	PCI_ROM(0x14e4, 0x1684, "14e4-1684", "14e4-1684", 0),
-	PCI_ROM(0x14e4, 0x165b, "14e4-165b", "14e4-165b", 0),
-	PCI_ROM(0x14e4, 0x1681, "14e4-1681", "14e4-1681", 0),
-	PCI_ROM(0x14e4, 0x1682, "14e4-1682", "14e4-1682", 0),
-	PCI_ROM(0x14e4, 0x1680, "14e4-1680", "14e4-1680", 0),
-	PCI_ROM(0x14e4, 0x1688, "14e4-1688", "14e4-1688", 0),
-	PCI_ROM(0x14e4, 0x1689, "14e4-1689", "14e4-1689", 0),
-	PCI_ROM(0x14e4, 0x1699, "14e4-1699", "14e4-1699", 0),
-	PCI_ROM(0x14e4, 0x16a0, "14e4-16a0", "14e4-16a0", 0),
-	PCI_ROM(0x14e4, 0x1692, "14e4-1692", "14e4-1692", 0),
-	PCI_ROM(0x14e4, 0x1690, "14e4-1690", "14e4-1690", 0),
-	PCI_ROM(0x14e4, 0x1694, "14e4-1694", "14e4-1694", 0),
-	PCI_ROM(0x14e4, 0x1691, "14e4-1691", "14e4-1691", 0),
-	PCI_ROM(0x14e4, 0x1655, "14e4-1655", "14e4-1655", 0),
-	PCI_ROM(0x14e4, 0x1656, "14e4-1656", "14e4-1656", 0),
-	PCI_ROM(0x14e4, 0x16b1, "14e4-16b1", "14e4-16b1", 0),
-	PCI_ROM(0x14e4, 0x16b5, "14e4-16b5", "14e4-16b5", 0),
-	PCI_ROM(0x14e4, 0x16b0, "14e4-16b0", "14e4-16b0", 0),
-	PCI_ROM(0x14e4, 0x16b4, "14e4-16b4", "14e4-16b4", 0),
-	PCI_ROM(0x14e4, 0x16b2, "14e4-16b2", "14e4-16b2", 0),
-	PCI_ROM(0x14e4, 0x16b6, "14e4-16b6", "14e4-16b6", 0),
-	PCI_ROM(0x14e4, 0x1657, "14e4-1657", "14e4-1657", 0),
-	PCI_ROM(0x14e4, 0x165f, "14e4-165f", "14e4-165f", 0),
-	PCI_ROM(0x14e4, 0x1686, "14e4-1686", "14e4-1686", 0),
-	PCI_ROM(0x1148, 0x4400, "1148-4400", "1148-4400", 0),
-	PCI_ROM(0x1148, 0x4500, "1148-4500", "1148-4500", 0),
 	PCI_ROM(0x173b, 0x03e8, "173b-03e8", "173b-03e8", 0),
 	PCI_ROM(0x173b, 0x03e9, "173b-03e9", "173b-03e9", 0),
-	PCI_ROM(0x173b, 0x03eb, "173b-03eb", "173b-03eb", 0),
 	PCI_ROM(0x173b, 0x03ea, "173b-03ea", "173b-03ea", 0),
-	PCI_ROM(0x106b, 0x1645, "106b-1645", "106b-1645", 0),
+	PCI_ROM(0x173b, 0x03eb, "173b-03eb", "173b-03eb", 0),
 };
 
 struct pci_driver tg3_pci_driver __pci_driver = {
diff --git a/src/drivers/net/tlan.c b/src/drivers/net/tlan.c
index 0e85b35b6..93533b438 100644
--- a/src/drivers/net/tlan.c
+++ b/src/drivers/net/tlan.c
@@ -1697,19 +1697,19 @@ void TLan_PhyMonitor(struct net_device *dev)
 #endif				/* MONITOR */
 
 static struct pci_device_id tlan_nics[] = {
-	PCI_ROM(0x0e11, 0xae34, "netel10", "Compaq Netelligent 10 T PCI UTP", 0),
 	PCI_ROM(0x0e11, 0xae32, "netel100","Compaq Netelligent 10/100 TX PCI UTP", 0),
+	PCI_ROM(0x0e11, 0xae34, "netel10", "Compaq Netelligent 10 T PCI UTP", 0),
 	PCI_ROM(0x0e11, 0xae35, "netflex3i", "Compaq Integrated NetFlex-3/P", 0),
+	PCI_ROM(0x0e11, 0xae40, "netel100d", "Compaq Netelligent Dual 10/100 TX PCI UTP", 0),
+	PCI_ROM(0x0e11, 0xae43, "netel100pi", "Compaq Netelligent Integrated 10/100 TX UTP", 0),
+	PCI_ROM(0x0e11, 0xb011, "netel100i", "Compaq Netelligent 10/100 TX Embedded UTP", 0),
+	PCI_ROM(0x0e11, 0xb012, "netelligent_10_t2", "Compaq Netelligent 10 T/2 PCI UTP/Coax", 0),
+	PCI_ROM(0x0e11, 0xb030, "netelligent_10_100_ws_5100", "Compaq Netelligent 10/100 TX UTP", 0),
 	PCI_ROM(0x0e11, 0xf130, "thunder", "Compaq NetFlex-3/P", 0),
 	PCI_ROM(0x0e11, 0xf150, "netflex3b", "Compaq NetFlex-3/P", 0),
-	PCI_ROM(0x0e11, 0xae43, "netel100pi", "Compaq Netelligent Integrated 10/100 TX UTP", 0),
-	PCI_ROM(0x0e11, 0xae40, "netel100d", "Compaq Netelligent Dual 10/100 TX PCI UTP", 0),
-	PCI_ROM(0x0e11, 0xb011, "netel100i", "Compaq Netelligent 10/100 TX Embedded UTP", 0),
-	PCI_ROM(0x108d, 0x0013, "oc2183", "Olicom OC-2183/2185", 0),
 	PCI_ROM(0x108d, 0x0012, "oc2325", "Olicom OC-2325", 0),
+	PCI_ROM(0x108d, 0x0013, "oc2183", "Olicom OC-2183/2185", 0),
 	PCI_ROM(0x108d, 0x0014, "oc2326", "Olicom OC-2326", 0),
-	PCI_ROM(0x0e11, 0xb030, "netelligent_10_100_ws_5100", "Compaq Netelligent 10/100 TX UTP", 0),
-	PCI_ROM(0x0e11, 0xb012, "netelligent_10_t2", "Compaq Netelligent 10 T/2 PCI UTP/Coax", 0),
 };
 
 PCI_DRIVER ( tlan_driver, tlan_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/tulip.c b/src/drivers/net/tulip.c
index e4e6ffa87..fddebfe5b 100644
--- a/src/drivers/net/tulip.c
+++ b/src/drivers/net/tulip.c
@@ -1921,31 +1921,30 @@ PCI_ROM(0x1011, 0x0002, "dc21040",     "Digital Tulip", 0),
 PCI_ROM(0x1011, 0x0009, "ds21140",     "Digital Tulip Fast", 0),
 PCI_ROM(0x1011, 0x0014, "dc21041",     "Digital Tulip+", 0),
 PCI_ROM(0x1011, 0x0019, "ds21142",     "Digital Tulip 21142", 0),
+PCI_ROM(0x104a, 0x0981, "tulip-0981",  "Tulip 0x104a 0x0981", 0),
+PCI_ROM(0x104a, 0x2774, "SGThomson-STE10100A", "Tulip 0x104a 0x2774", 0),	/*Modified by Ramesh Chander*/
 PCI_ROM(0x10b7, 0x9300, "3csoho100b-tx","3ComSOHO100B-TX", 0),
 PCI_ROM(0x10b9, 0x5261, "ali1563",     "ALi 1563 integrated ethernet", 0),
 PCI_ROM(0x10d9, 0x0512, "mx98713",     "Macronix MX987x3", 0),
 PCI_ROM(0x10d9, 0x0531, "mx98715",     "Macronix MX987x5", 0),
+PCI_ROM(0x1113, 0x1216, "an983",       "ADMTek AN983 Comet", 0),
 PCI_ROM(0x1113, 0x1217, "mxic-98715",  "Macronix MX987x5", 0),
-PCI_ROM(0x11ad, 0xc115, "lc82c115",    "LinkSys LNE100TX", 0),
+PCI_ROM(0x1113, 0x9511, "tulip-9511",  "Tulip 0x1113 0x9511", 0),
+PCI_ROM(0x115d, 0x0003, "xircomtulip", "Xircom Tulip", 0),
+PCI_ROM(0x1186, 0x1561, "tulip-1561",  "Tulip 0x1186 0x1561", 0),
 PCI_ROM(0x11ad, 0x0002, "82c168",      "Netgear FA310TX", 0),
+PCI_ROM(0x11ad, 0xc115, "lc82c115",    "LinkSys LNE100TX", 0),
+PCI_ROM(0x11f6, 0x9881, "rl100tx",     "Compex RL100-TX", 0),
+PCI_ROM(0x1259, 0xa120, "tulip-a120",  "Tulip 0x1259 0xa120", 0),
+PCI_ROM(0x125b, 0x1400, "ax88140",     "ASIX AX88140", 0),
+PCI_ROM(0x1282, 0x9009, "dm9009",      "Davicom 9009", 0),
 PCI_ROM(0x1282, 0x9100, "dm9100",      "Davicom 9100", 0),
 PCI_ROM(0x1282, 0x9102, "dm9102",      "Davicom 9102", 0),
-PCI_ROM(0x1282, 0x9009, "dm9009",      "Davicom 9009", 0),
 PCI_ROM(0x1282, 0x9132, "dm9132",      "Davicom 9132", 0),
-PCI_ROM(0x1317, 0x0985, "centaur-p",   "ADMtek Centaur-P", 0),
 PCI_ROM(0x1317, 0x0981, "an981",       "ADMtek AN981 Comet", 0),		/* ADMTek Centaur-P (stmicro) */
-PCI_ROM(0x1113, 0x1216, "an983",       "ADMTek AN983 Comet", 0),
-PCI_ROM(0x1317, 0x9511, "an983b",      "ADMTek Comet 983b", 0),
+PCI_ROM(0x1317, 0x0985, "centaur-p",   "ADMtek Centaur-P", 0),
 PCI_ROM(0x1317, 0x1985, "centaur-c",   "ADMTek Centaur-C", 0),
-PCI_ROM(0x8086, 0x0039, "intel21145",  "Intel Tulip", 0),
-PCI_ROM(0x125b, 0x1400, "ax88140",     "ASIX AX88140", 0),
-PCI_ROM(0x11f6, 0x9881, "rl100tx",     "Compex RL100-TX", 0),
-PCI_ROM(0x115d, 0x0003, "xircomtulip", "Xircom Tulip", 0),
-PCI_ROM(0x104a, 0x0981, "tulip-0981",  "Tulip 0x104a 0x0981", 0),
-PCI_ROM(0x104a, 0x2774, "SGThomson-STE10100A", "Tulip 0x104a 0x2774", 0),	/*Modified by Ramesh Chander*/
-PCI_ROM(0x1113, 0x9511, "tulip-9511",  "Tulip 0x1113 0x9511", 0),
-PCI_ROM(0x1186, 0x1561, "tulip-1561",  "Tulip 0x1186 0x1561", 0),
-PCI_ROM(0x1259, 0xa120, "tulip-a120",  "Tulip 0x1259 0xa120", 0),
+PCI_ROM(0x1317, 0x9511, "an983b",      "ADMTek Comet 983b", 0),
 PCI_ROM(0x13d1, 0xab02, "tulip-ab02",  "Tulip 0x13d1 0xab02", 0),
 PCI_ROM(0x13d1, 0xab03, "tulip-ab03",  "Tulip 0x13d1 0xab03", 0),
 PCI_ROM(0x13d1, 0xab08, "tulip-ab08",  "Tulip 0x13d1 0xab08", 0),
@@ -1953,6 +1952,7 @@ PCI_ROM(0x14f1, 0x1803, "lanfinity",   "Conexant LANfinity", 0),
 PCI_ROM(0x1626, 0x8410, "tulip-8410",  "Tulip 0x1626 0x8410", 0),
 PCI_ROM(0x1737, 0xab08, "tulip-1737-ab08","Tulip 0x1737 0xab08", 0),
 PCI_ROM(0x1737, 0xab09, "tulip-ab09",  "Tulip 0x1737 0xab09", 0),
+PCI_ROM(0x8086, 0x0039, "intel21145",  "Intel Tulip", 0),
 };
 
 PCI_DRIVER ( tulip_driver, tulip_nics, PCI_NO_CLASS );

From ee6185dcf5ad7b0d5e486e66c424341764fc221d Mon Sep 17 00:00:00 2001
From: Alexey Sheplyakov <asheplyakov@basealt.ru>
Date: Wed, 21 Feb 2024 13:07:21 +0400
Subject: [PATCH 082/237] [efi] Ignore new LoongArch PC-relative relocations
 and relaxations

Several new relocations types have been added in LoongArch ABI version
2.10.  In particular:

- R_LARCH_B16 (18-bit PC-relative jump)
- R_LARCH_B21 (23-bit PC-relative jump)
- R_LARCH_PCREL20_S2 (22-bit PC-relative offset)

Also relocation relaxations have been introduced.  Recent GCC (13.2)
and binutils 2.41+ use these types of relocations, which confuses
elf2efi tool.  As a result, iPXE EFI images for LoongArch fail to
build with the following error:

  Unrecognised relocation type 103

Fix by ignoring R_LARCH_B{16,21} and R_LARCH_PCREL20_S2 (as with other
PC-relative relocations), and by ignoring relaxations (R_LARCH_RELAX).
Relocation relaxations are basically optimizations: ignoring them
results in a correct binary (although it might be suboptimal).

Modified-by: Michael Brown <mcb30@ipxe.org>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/util/elf2efi.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 88713b663..4af587d87 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -140,6 +140,12 @@
 #ifndef R_LARCH_64
 #define R_LARCH_64 2
 #endif
+#ifndef R_LARCH_B16
+#define R_LARCH_B16 64
+#endif
+#ifndef R_LARCH_B21
+#define R_LARCH_B21 65
+#endif
 #ifndef R_LARCH_B26
 #define R_LARCH_B26 66
 #endif
@@ -155,6 +161,12 @@
 #ifndef R_LARCH_GOT_PC_LO12
 #define R_LARCH_GOT_PC_LO12 76
 #endif
+#ifndef R_LARCH_RELAX
+#define R_LARCH_RELAX 100
+#endif
+#ifndef R_LARCH_PCREL20_S2
+#define R_LARCH_PCREL20_S2 103
+#endif
 #ifndef R_X86_64_GOTPCRELX
 #define R_X86_64_GOTPCRELX 41
 #endif
@@ -849,16 +861,24 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST32_ABS_LO12_NC ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST64_ABS_LO12_NC ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST128_ABS_LO12_NC ) :
+		case ELF_MREL ( EM_LOONGARCH, R_LARCH_B16):
+		case ELF_MREL ( EM_LOONGARCH, R_LARCH_B21):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_B26):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCALA_HI20 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCALA_LO12 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_GOT_PC_HI20 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_GOT_PC_LO12 ):
+		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCREL20_S2 ):
 			/* Skip PC-relative relocations; all relative
 			 * offsets remain unaltered when the object is
 			 * loaded.
 			 */
 			break;
+		case ELF_MREL ( EM_LOONGARCH, R_LARCH_RELAX ):
+			/* Relocation can be relaxed (optimized out).
+			 * Ignore it for now.
+			 */
+			break;
 		case ELF_MREL ( EM_X86_64, R_X86_64_32 ) :
 			/* Ignore 32-bit relocations in a hybrid
 			 * 32-bit BIOS and 64-bit UEFI binary,

From 834f319f87bd2a1135c51375774f6bfe17a244cc Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 23 Feb 2024 15:58:51 +0000
Subject: [PATCH 083/237] [eap] Add progress debug messages

Add debug messages for each EAP Request and Response, and to show the
list of methods offered when sending a Nak.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/eap.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/net/eap.c b/src/net/eap.c
index 79b060aa6..696b7fe99 100644
--- a/src/net/eap.c
+++ b/src/net/eap.c
@@ -65,6 +65,8 @@ static int eap_tx_response ( struct eap_supplicant *supplicant,
 	msg->hdr.len = htons ( len );
 	msg->type = supplicant->type;
 	memcpy ( msg->data, rsp, rsp_len );
+	DBGC ( netdev, "EAP %s Response id %#02x type %d\n",
+	       netdev->name, msg->hdr.id, msg->type );
 
 	/* Transmit response */
 	if ( ( rc = supplicant->tx ( supplicant, msg, len ) ) != 0 ) {
@@ -86,18 +88,24 @@ static int eap_tx_response ( struct eap_supplicant *supplicant,
  * @ret rc		Return status code
  */
 static int eap_tx_nak ( struct eap_supplicant *supplicant ) {
+	struct net_device *netdev = supplicant->netdev;
 	unsigned int max = table_num_entries ( EAP_METHODS );
 	uint8_t methods[ max + 1 /* potential EAP_TYPE_NONE */ ];
 	unsigned int count = 0;
 	struct eap_method *method;
 
 	/* Populate methods list */
+	DBGC ( netdev, "EAP %s Nak offering types {", netdev->name );
 	for_each_table_entry ( method, EAP_METHODS ) {
-		if ( method->type > EAP_TYPE_NAK )
+		if ( method->type > EAP_TYPE_NAK ) {
+			DBGC ( netdev, "%s%d",
+			       ( count ? ", " : "" ), method->type );
 			methods[count++] = method->type;
+		}
 	}
 	if ( ! count )
 		methods[count++] = EAP_TYPE_NONE;
+	DBGC ( netdev, "}\n" );
 	assert ( count <= max );
 
 	/* Transmit response */
@@ -269,6 +277,8 @@ static int eap_rx_request ( struct eap_supplicant *supplicant,
 	/* Record request details */
 	supplicant->id = msg->hdr.id;
 	supplicant->type = msg->type;
+	DBGC ( netdev, "EAP %s Request id %#02x type %d\n",
+	       netdev->name, msg->hdr.id, msg->type );
 
 	/* Handle according to type */
 	for_each_table_entry ( method, EAP_METHODS ) {

From 25ffcd79bfd38da96f9905b78e3d5c3cab33dad3 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 23 Feb 2024 12:33:57 +0000
Subject: [PATCH 084/237] [eap] Allow MD5-Challenge authentication method to be
 disabled

RFC 3748 states that implementations must support the MD5-Challenge
method.  However, some network environments may wish to disable it as
a matter of policy.

Allow support for MD5-Challenge to be controllable via the build
configuration option EAP_METHOD_MD5 in config/general.h.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config_eap.c    |  39 +++++++++++++
 src/config/general.h       |   6 ++
 src/include/ipxe/eap.h     |   2 +
 src/include/ipxe/errfile.h |   1 +
 src/net/eap.c              |  90 +++-------------------------
 src/net/eap_md5.c          | 116 +++++++++++++++++++++++++++++++++++++
 6 files changed, 172 insertions(+), 82 deletions(-)
 create mode 100644 src/config/config_eap.c
 create mode 100644 src/net/eap_md5.c

diff --git a/src/config/config_eap.c b/src/config/config_eap.c
new file mode 100644
index 000000000..d3fd77aa2
--- /dev/null
+++ b/src/config/config_eap.c
@@ -0,0 +1,39 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/general.h>
+
+/** @file
+ *
+ * EAP configuration options
+ *
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+/*
+ * Drag in EAP authentication methods
+ */
+#ifdef EAP_METHOD_MD5
+REQUIRE_OBJECT ( eap_md5 );
+#endif
diff --git a/src/config/general.h b/src/config/general.h
index 6e8e86b2b..de009a878 100644
--- a/src/config/general.h
+++ b/src/config/general.h
@@ -91,6 +91,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define	CRYPTO_80211_WPA	/* WPA Personal, authenticating with passphrase */
 #define	CRYPTO_80211_WPA2	/* Add support for stronger WPA cryptography */
 
+/*
+ * 802.1x EAP authentication methods
+ *
+ */
+#define EAP_METHOD_MD5		/* MD5-Challenge port authentication */
+
 /*
  * Name resolution modules
  *
diff --git a/src/include/ipxe/eap.h b/src/include/ipxe/eap.h
index cf1c7c00d..fe1bb5282 100644
--- a/src/include/ipxe/eap.h
+++ b/src/include/ipxe/eap.h
@@ -166,6 +166,8 @@ struct eap_method {
 /** Declare an EAP method */
 #define __eap_method __table_entry ( EAP_METHODS, 01 )
 
+extern int eap_tx_response ( struct eap_supplicant *supplicant,
+			     const void *rsp, size_t rsp_len );
 extern int eap_rx ( struct eap_supplicant *supplicant,
 		    const void *data, size_t len );
 
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index f7a00dbe7..1768748d9 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -297,6 +297,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_httpntlm		( ERRFILE_NET | 0x004a0000 )
 #define ERRFILE_eap			( ERRFILE_NET | 0x004b0000 )
 #define ERRFILE_lldp			( ERRFILE_NET | 0x004c0000 )
+#define ERRFILE_eap_md5			( ERRFILE_NET | 0x004d0000 )
 
 #define ERRFILE_image		      ( ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_elf		      ( ERRFILE_IMAGE | 0x00010000 )
diff --git a/src/net/eap.c b/src/net/eap.c
index 696b7fe99..87327d723 100644
--- a/src/net/eap.c
+++ b/src/net/eap.c
@@ -28,8 +28,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <string.h>
 #include <byteswap.h>
 #include <ipxe/netdevice.h>
-#include <ipxe/md5.h>
-#include <ipxe/chap.h>
 #include <ipxe/eap.h>
 
 /** @file
@@ -46,8 +44,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v rsp_len		Length of response type data
  * @ret rc		Return status code
  */
-static int eap_tx_response ( struct eap_supplicant *supplicant,
-			     const void *rsp, size_t rsp_len ) {
+int eap_tx_response ( struct eap_supplicant *supplicant,
+		      const void *rsp, size_t rsp_len ) {
 	struct net_device *netdev = supplicant->netdev;
 	struct eap_message *msg;
 	size_t len;
@@ -167,84 +165,6 @@ struct eap_method eap_identity_method __eap_method = {
 	.rx = eap_rx_identity,
 };
 
-/**
- * Handle EAP MD5-Challenge
- *
- * @v req		Request type data
- * @v req_len		Length of request type data
- * @ret rc		Return status code
- */
-static int eap_rx_md5 ( struct eap_supplicant *supplicant,
-			const void *req, size_t req_len ) {
-	struct net_device *netdev = supplicant->netdev;
-	const struct eap_md5 *md5req = req;
-	struct {
-		uint8_t len;
-		uint8_t value[MD5_DIGEST_SIZE];
-	} __attribute__ (( packed )) md5rsp;
-	struct chap_response chap;
-	void *secret;
-	int secret_len;
-	int rc;
-
-	/* Sanity checks */
-	if ( req_len < sizeof ( *md5req ) ) {
-		DBGC ( netdev, "EAP %s underlength MD5-Challenge:\n",
-		       netdev->name );
-		DBGC_HDA ( netdev, 0, req, req_len );
-		rc = -EINVAL;
-		goto err_sanity;
-	}
-	if ( ( req_len - sizeof ( *md5req ) ) < md5req->len ) {
-		DBGC ( netdev, "EAP %s truncated MD5-Challenge:\n",
-		       netdev->name );
-		DBGC_HDA ( netdev, 0, req, req_len );
-		rc = -EINVAL;
-		goto err_sanity;
-	}
-
-	/* Construct response */
-	if ( ( rc = chap_init ( &chap, &md5_algorithm ) ) != 0 ) {
-		DBGC ( netdev, "EAP %s could not initialise CHAP: %s\n",
-		       netdev->name, strerror ( rc ) );
-		goto err_chap;
-	}
-	chap_set_identifier ( &chap, supplicant->id );
-	secret_len = fetch_raw_setting_copy ( netdev_settings ( netdev ),
-					      &password_setting, &secret );
-	if ( secret_len < 0 ) {
-		rc = secret_len;
-		DBGC ( netdev, "EAP %s has no secret: %s\n",
-		       netdev->name, strerror ( rc ) );
-		goto err_secret;
-	}
-	chap_update ( &chap, secret, secret_len );
-	chap_update ( &chap, md5req->value, md5req->len );
-	chap_respond ( &chap );
-	assert ( chap.response_len == sizeof ( md5rsp.value ) );
-	md5rsp.len = sizeof ( md5rsp.value );
-	memcpy ( md5rsp.value, chap.response, sizeof ( md5rsp.value ) );
-
-	/* Transmit response */
-	if ( ( rc = eap_tx_response ( supplicant, &md5rsp,
-				      sizeof ( md5rsp ) ) ) != 0 )
-		goto err_tx;
-
- err_tx:
-	free ( secret );
- err_secret:
-	chap_finish ( &chap );
- err_chap:
- err_sanity:
-	return rc;
-}
-
-/** EAP MD5-Challenge method */
-struct eap_method eap_md5_method __eap_method = {
-	.type = EAP_TYPE_MD5,
-	.rx = eap_rx_md5,
-};
-
 /**
  * Handle EAP Request
  *
@@ -370,3 +290,9 @@ int eap_rx ( struct eap_supplicant *supplicant, const void *data,
 		return -ENOTSUP;
 	}
 }
+
+/* Drag in objects via eap_rx() */
+REQUIRING_SYMBOL ( eap_rx );
+
+/* Drag in EAP configuration */
+REQUIRE_OBJECT ( config_eap );
diff --git a/src/net/eap_md5.c b/src/net/eap_md5.c
new file mode 100644
index 000000000..0664174f9
--- /dev/null
+++ b/src/net/eap_md5.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <ipxe/md5.h>
+#include <ipxe/chap.h>
+#include <ipxe/eap.h>
+
+/** @file
+ *
+ * EAP MD5-Challenge authentication method
+ *
+ */
+
+/**
+ * Handle EAP MD5-Challenge
+ *
+ * @v supplicant	EAP supplicant
+ * @v req		Request type data
+ * @v req_len		Length of request type data
+ * @ret rc		Return status code
+ */
+static int eap_rx_md5 ( struct eap_supplicant *supplicant,
+			const void *req, size_t req_len ) {
+	struct net_device *netdev = supplicant->netdev;
+	const struct eap_md5 *md5req = req;
+	struct {
+		uint8_t len;
+		uint8_t value[MD5_DIGEST_SIZE];
+	} __attribute__ (( packed )) md5rsp;
+	struct chap_response chap;
+	void *secret;
+	int secret_len;
+	int rc;
+
+	/* Sanity checks */
+	if ( req_len < sizeof ( *md5req ) ) {
+		DBGC ( netdev, "EAP %s underlength MD5-Challenge:\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+	if ( ( req_len - sizeof ( *md5req ) ) < md5req->len ) {
+		DBGC ( netdev, "EAP %s truncated MD5-Challenge:\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+
+	/* Construct response */
+	if ( ( rc = chap_init ( &chap, &md5_algorithm ) ) != 0 ) {
+		DBGC ( netdev, "EAP %s could not initialise CHAP: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_chap;
+	}
+	chap_set_identifier ( &chap, supplicant->id );
+	secret_len = fetch_raw_setting_copy ( netdev_settings ( netdev ),
+					      &password_setting, &secret );
+	if ( secret_len < 0 ) {
+		rc = secret_len;
+		DBGC ( netdev, "EAP %s has no secret: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_secret;
+	}
+	chap_update ( &chap, secret, secret_len );
+	chap_update ( &chap, md5req->value, md5req->len );
+	chap_respond ( &chap );
+	assert ( chap.response_len == sizeof ( md5rsp.value ) );
+	md5rsp.len = sizeof ( md5rsp.value );
+	memcpy ( md5rsp.value, chap.response, sizeof ( md5rsp.value ) );
+
+	/* Transmit response */
+	if ( ( rc = eap_tx_response ( supplicant, &md5rsp,
+				      sizeof ( md5rsp ) ) ) != 0 )
+		goto err_tx;
+
+ err_tx:
+	free ( secret );
+ err_secret:
+	chap_finish ( &chap );
+ err_chap:
+ err_sanity:
+	return rc;
+}
+
+/** EAP MD5-Challenge method */
+struct eap_method eap_md5_method __eap_method = {
+	.type = EAP_TYPE_MD5,
+	.rx = eap_rx_md5,
+};

From 43e385091a36af34e495ac8c6595bddab55665bb Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 23 Feb 2024 14:15:22 +0000
Subject: [PATCH 085/237] [eap] Add support for the MS-CHAPv2 authentication
 method

Add support for EAP-MSCHAPv2 (note that this is not the same as
PEAP-MSCHAPv2), controllable via the build configuration option
EAP_METHOD_MSCHAPV2 in config/general.h.

Our model for EAP does not encompass mutual authentication: we will
starting sending plaintext packets (e.g. DHCP requests) over the link
even before EAP completes, and our only use for an EAP success is to
mark the link as unblocked.

We therefore ignore the content of the EAP-MSCHAPv2 success request
(containing the MS-CHAPv2 authenticator response) and just send back
an EAP-MSCHAPv2 success response, so that the EAP authenticator will
complete the process and send through the real EAP success packet
(which will, in turn, cause us to unblock the link).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config_eap.c    |   3 +
 src/config/general.h       |   1 +
 src/include/ipxe/eap.h     |  29 +++++
 src/include/ipxe/errfile.h |   1 +
 src/net/eap_mschapv2.c     | 251 +++++++++++++++++++++++++++++++++++++
 5 files changed, 285 insertions(+)
 create mode 100644 src/net/eap_mschapv2.c

diff --git a/src/config/config_eap.c b/src/config/config_eap.c
index d3fd77aa2..e18c48cae 100644
--- a/src/config/config_eap.c
+++ b/src/config/config_eap.c
@@ -37,3 +37,6 @@ PROVIDE_REQUIRING_SYMBOL();
 #ifdef EAP_METHOD_MD5
 REQUIRE_OBJECT ( eap_md5 );
 #endif
+#ifdef EAP_METHOD_MSCHAPV2
+REQUIRE_OBJECT ( eap_mschapv2 );
+#endif
diff --git a/src/config/general.h b/src/config/general.h
index de009a878..c9cdb3dd4 100644
--- a/src/config/general.h
+++ b/src/config/general.h
@@ -96,6 +96,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 #define EAP_METHOD_MD5		/* MD5-Challenge port authentication */
+//#define EAP_METHOD_MSCHAPV2	/* MS-CHAPv2 port authentication */
 
 /*
  * Name resolution modules
diff --git a/src/include/ipxe/eap.h b/src/include/ipxe/eap.h
index fe1bb5282..a44f01e0a 100644
--- a/src/include/ipxe/eap.h
+++ b/src/include/ipxe/eap.h
@@ -60,6 +60,35 @@ struct eap_md5 {
 	uint8_t value[0];
 } __attribute__ (( packed ));
 
+/** EAP MS-CHAPv2 request/response */
+#define EAP_TYPE_MSCHAPV2 26
+
+/** EAP MS-CHAPv2 request/response type data */
+struct eap_mschapv2 {
+	/** Code
+	 *
+	 * This is in the same namespace as the EAP header's code
+	 * field, but is used to extend the handshake by allowing for
+	 * "success request" and "success response" packets.
+	 */
+	uint8_t code;
+	/** Identifier
+	 *
+	 * This field serves no purposes: it always has the same value
+	 * as the EAP header's identifier field (located 5 bytes
+	 * earlier in the same packet).
+	 */
+	uint8_t id;
+	/** Length
+	 *
+	 * This field serves no purpose: it always has the same value
+	 * as the EAP header's length field (located 5 bytes earlier
+	 * in the same packet), minus the 5 byte length of the EAP
+	 * header.
+	 */
+	uint16_t len;
+} __attribute__ (( packed ));
+
 /** EAP success */
 #define EAP_CODE_SUCCESS 3
 
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 1768748d9..662f84964 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -298,6 +298,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_eap			( ERRFILE_NET | 0x004b0000 )
 #define ERRFILE_lldp			( ERRFILE_NET | 0x004c0000 )
 #define ERRFILE_eap_md5			( ERRFILE_NET | 0x004d0000 )
+#define ERRFILE_eap_mschapv2		( ERRFILE_NET | 0x004e0000 )
 
 #define ERRFILE_image		      ( ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_elf		      ( ERRFILE_IMAGE | 0x00010000 )
diff --git a/src/net/eap_mschapv2.c b/src/net/eap_mschapv2.c
new file mode 100644
index 000000000..0be62ed59
--- /dev/null
+++ b/src/net/eap_mschapv2.c
@@ -0,0 +1,251 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/mschapv2.h>
+#include <ipxe/eap.h>
+
+/** @file
+ *
+ * EAP MS-CHAPv2 authentication method
+ *
+ * EAP-MSCHAPv2 was described in a draft RFC first published in 2002
+ * (draft-kamath-pppext-eap-mschapv2-02.txt).  The draft eventually
+ * expired in 2007 without becoming an official RFC, quite possibly
+ * because the protocol design was too ugly to be called an IETF
+ * standard.  It is, however, fairly widely used.
+ */
+
+/** An EAP MS-CHAPv2 request message */
+struct eap_mschapv2_request {
+	/** EAP-MSCHAPv2 header */
+	struct eap_mschapv2 hdr;
+	/** MS-CHAPv2 challenge length (fixed value) */
+	uint8_t len;
+	/** MS-CHAPv2 challenge */
+	struct mschapv2_challenge msg;
+} __attribute__ (( packed ));
+
+/** An EAP MS-CHAPv2 response message */
+struct eap_mschapv2_response {
+	/** EAP-MSCHAPv2 header */
+	struct eap_mschapv2 hdr;
+	/** MS-CHAPv2 response length (fixed value) */
+	uint8_t len;
+	/** MS-CHAPv2 response */
+	struct mschapv2_response msg;
+	/** User name */
+	char name[0];
+} __attribute__ (( packed ));
+
+/** An EAP MS-CHAPv2 success request message */
+struct eap_mschapv2_success_request {
+	/** EAP-MSCHAPv2 header */
+	struct eap_mschapv2 hdr;
+	/** Message */
+	char message[0];
+} __attribute__ (( packed ));
+
+/** An EAP MS-CHAPv2 success response message */
+struct eap_mschapv2_success_response {
+	/** Opcode */
+	uint8_t code;
+} __attribute__ (( packed ));
+
+/**
+ * Handle EAP MS-CHAPv2 request
+ *
+ * @v supplicant	EAP supplicant
+ * @v hdr		EAP-MSCHAPv2 header
+ * @v len		Message length
+ * @ret rc		Return status code
+ */
+static int eap_rx_mschapv2_request ( struct eap_supplicant *supplicant,
+				     const struct eap_mschapv2 *hdr,
+				     size_t len ) {
+	struct net_device *netdev = supplicant->netdev;
+	struct settings *settings = netdev_settings ( netdev );
+	const struct eap_mschapv2_request *msreq =
+		container_of ( hdr, struct eap_mschapv2_request, hdr );
+	struct eap_mschapv2_response *msrsp;
+	struct mschapv2_challenge peer;
+	char *username;
+	char *password;
+	int username_len;
+	int password_len;
+	size_t msrsp_len;
+	unsigned int i;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *msreq ) ) {
+		DBGC ( netdev, "EAP %s underlength MS-CHAPv2 request\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, hdr, len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+
+	/* Fetch username and password */
+	username_len = fetch_string_setting_copy ( settings, &username_setting,
+						   &username );
+	if ( username_len < 0 ) {
+		rc = username_len;
+		DBGC ( netdev, "EAP %s has no username: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_username;
+	}
+	password_len = fetch_string_setting_copy ( settings, &password_setting,
+						   &password );
+	if ( password_len < 0 ) {
+		rc = password_len;
+		DBGC ( netdev, "EAP %s has no password: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_password;
+	}
+
+	/* Construct a peer challenge.  We do not perform mutual
+	 * authentication, so this does not need to be strong.
+	 */
+	for ( i = 0 ; i < ( sizeof ( peer.byte ) /
+			    sizeof ( peer.byte[0] ) ) ; i++ ) {
+		peer.byte[i] = random();
+	}
+
+	/* Allocate response */
+	msrsp_len = ( sizeof ( *msrsp ) + username_len );
+	msrsp = malloc ( msrsp_len );
+	if ( ! msrsp ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Construct response */
+	msrsp->hdr.code = EAP_CODE_RESPONSE;
+	msrsp->hdr.id = msreq->hdr.id;
+	msrsp->hdr.len = htons ( msrsp_len );
+	msrsp->len = sizeof ( msrsp->msg );
+	mschapv2_response ( username, password, &msreq->msg, &peer,
+			    &msrsp->msg );
+	memcpy ( msrsp->name, username, username_len );
+
+	/* Send response */
+	if ( ( rc = eap_tx_response ( supplicant, msrsp, msrsp_len ) ) != 0 )
+		goto err_tx;
+
+ err_tx:
+	free ( msrsp );
+ err_alloc:
+	free ( password );
+ err_password:
+	free ( username );
+ err_username:
+ err_sanity:
+	return rc;
+}
+
+/**
+ * Handle EAP MS-CHAPv2 success request
+ *
+ * @v supplicant	EAP supplicant
+ * @v hdr		EAP-MSCHAPv2 header
+ * @v len		Message length
+ * @ret rc		Return status code
+ */
+static int eap_rx_mschapv2_success ( struct eap_supplicant *supplicant,
+				     const struct eap_mschapv2 *hdr,
+				     size_t len ) {
+	const struct eap_mschapv2_success_request *msreq =
+		container_of ( hdr, struct eap_mschapv2_success_request, hdr );
+	static const struct eap_mschapv2_success_response msrsp = {
+		.code = EAP_CODE_SUCCESS,
+	};
+
+	/* Sanity check */
+	assert ( len >= sizeof ( *msreq ) );
+
+	/* The success request contains the MS-CHAPv2 authenticator
+	 * response, which could potentially be used to verify that
+	 * the EAP authenticator also knew the password (or, at least,
+	 * the MD4 hash of the password).
+	 *
+	 * Our model for EAP does not encompass mutual authentication:
+	 * we will starting sending plaintext packets (e.g. DHCP
+	 * requests) over the link even before EAP completes, and our
+	 * only use for an EAP success is to mark the link as
+	 * unblocked.
+	 *
+	 * We therefore ignore the content of the success request and
+	 * just send back a success response, so that the EAP
+	 * authenticator will complete the process and send through
+	 * the real EAP success packet (which will, in turn, cause us
+	 * to unblock the link).
+	 */
+	return eap_tx_response ( supplicant, &msrsp, sizeof ( msrsp ) );
+}
+
+/**
+ * Handle EAP MS-CHAPv2
+ *
+ * @v supplicant	EAP supplicant
+ * @v req		Request type data
+ * @v req_len		Length of request type data
+ * @ret rc		Return status code
+ */
+static int eap_rx_mschapv2 ( struct eap_supplicant *supplicant,
+			     const void *req, size_t req_len ) {
+	struct net_device *netdev = supplicant->netdev;
+	const struct eap_mschapv2 *hdr = req;
+
+	/* Sanity check */
+	if ( req_len < sizeof ( *hdr ) ) {
+		DBGC ( netdev, "EAP %s underlength MS-CHAPv2:\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		return -EINVAL;
+	}
+
+	/* Handle according to opcode */
+	switch ( hdr->code ) {
+	case EAP_CODE_REQUEST:
+		return eap_rx_mschapv2_request ( supplicant, hdr, req_len );
+	case EAP_CODE_SUCCESS:
+		return eap_rx_mschapv2_success ( supplicant, hdr, req_len );
+	default:
+		DBGC ( netdev, "EAP %s unsupported MS-CHAPv2 opcode %d\n",
+		       netdev->name, hdr->code );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		return -ENOTSUP;
+	}
+}
+
+/** EAP MS-CHAPv2 method */
+struct eap_method eap_mschapv2_method __eap_method = {
+	.type = EAP_TYPE_MSCHAPV2,
+	.rx = eap_rx_mschapv2,
+};

From 182ee909313fc60875d3f1741cd4a0bb7dfd15e1 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 27 Feb 2024 13:34:17 +0000
Subject: [PATCH 086/237] [efi] Work around broken boot services table
 manipulation by UEFI shim

The UEFI shim installs wrappers around several boot services functions
before invoking its next stage bootloader, in an attempt to enforce
its desired behaviour upon the aforementioned bootloader.  For
example, shim checks that the bootloader has either invoked
StartImage() or has called into the "shim lock protocol" before
allowing an ExitBootServices() call to proceed.

When invoking a shim, iPXE will also install boot services function
wrappers in order to work around assorted bugs in the UEFI shim code
that would otherwise prevent it from being used to boot a kernel.  For
details on these workarounds, see commits 28184b7 ("[efi] Add support
for executing images via a shim") and 5b43181 ("[efi] Support versions
of shim that perform SBAT verification").

Using boot services function wrappers in this way is not intrinsically
problematic, provided that wrappers are installed before starting the
wrapped program, and uninstalled only after the wrapped program exits.
This strict ordering requirement ensures that all layers of wrappers
are called in the expected order, and that no calls are issued through
a no-longer-valid function pointer.

Unfortunately, the UEFI shim does not respect this strict ordering
requirement, and will instead uninstall (and reinstall) its wrappers
midway through the execution of the wrapped program.  This leaves the
wrapped program with an inconsistent view of the boot services table,
leading to incorrect behaviour.

This results in a boot failure when a first shim is used to boot iPXE,
which then uses a second shim to boot a Linux kernel:

  - First shim installs StartImage() and ExitBootServices() wrappers

  - First shim invokes iPXE via its own PE loader

  - iPXE installs ExitBootServices() wrapper

  - iPXE invokes second shim via StartImage()

At this point, the first shim's StartImage() wrapper will illegally
uninstall its ExitBootServices() wrapper, without first checking that
nothing else has modified the ExitBootServices function pointer.  This
effectively bypasses iPXE's own ExitBootServices() wrapper, which
causes a boot failure since the code within that wrapper does not get
called.

A proper fix would be for shim to install its wrappers before starting
the image and uninstall its wrappers only after the started image has
exited.  Instead of repeatedly uninstalling and reinstalling its
wrappers while the wrapped program is running, shim should simply use
a flag to keep track of whether or not it needs to modify the
behaviour of the wrapped calls.

Experience shows that there is unfortunately no point in trying to get
a fix for this upstreamed into shim.  We therefore work around the
shim bug by removing our ExitBootServices() wrapper and moving the
relevant code into our GetMemoryMap() wrapper.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_shim.c | 92 +++++++++++++++++-------------------
 1 file changed, 43 insertions(+), 49 deletions(-)

diff --git a/src/interface/efi/efi_shim.c b/src/interface/efi/efi_shim.c
index a46d79d0c..d5419512d 100644
--- a/src/interface/efi/efi_shim.c
+++ b/src/interface/efi/efi_shim.c
@@ -112,9 +112,6 @@ struct image_tag efi_shim __image_tag = {
 /** Original GetMemoryMap() function */
 static EFI_GET_MEMORY_MAP efi_shim_orig_get_memory_map;
 
-/** Original ExitBootServices() function */
-static EFI_EXIT_BOOT_SERVICES efi_shim_orig_exit_boot_services;
-
 /** Original SetVariable() function */
 static EFI_SET_VARIABLE efi_shim_orig_set_variable;
 
@@ -160,49 +157,6 @@ static void efi_shim_unlock ( void ) {
 	}
 }
 
-/**
- * Wrap GetMemoryMap()
- *
- * @v len		Memory map size
- * @v map		Memory map
- * @v key		Memory map key
- * @v desclen		Descriptor size
- * @v descver		Descriptor version
- * @ret efirc		EFI status code
- */
-static EFIAPI EFI_STATUS efi_shim_get_memory_map ( UINTN *len,
-						   EFI_MEMORY_DESCRIPTOR *map,
-						   UINTN *key, UINTN *desclen,
-						   UINT32 *descver ) {
-
-	/* Unlock shim */
-	if ( ! efi_shim_require_loader )
-		efi_shim_unlock();
-
-	/* Hand off to original GetMemoryMap() */
-	return efi_shim_orig_get_memory_map ( len, map, key, desclen,
-					      descver );
-}
-
-/**
- * Wrap ExitBootServices()
- *
- * @v handle		Image handle
- * @v key		Memory map key
- * @ret efirc		EFI status code
- */
-static EFIAPI EFI_STATUS efi_shim_exit_boot_services ( EFI_HANDLE handle,
-						       UINTN key ) {
-	EFI_RUNTIME_SERVICES *rs = efi_systab->RuntimeServices;
-
-	/* Restore original runtime services functions */
-	rs->SetVariable = efi_shim_orig_set_variable;
-	rs->GetVariable = efi_shim_orig_get_variable;
-
-	/* Hand off to original ExitBootServices() */
-	return efi_shim_orig_exit_boot_services ( handle, key );
-}
-
 /**
  * Wrap SetVariable()
  *
@@ -270,6 +224,47 @@ efi_shim_get_variable ( CHAR16 *name, EFI_GUID *guid, UINT32 *attrs,
 	return efirc;
 }
 
+/**
+ * Wrap GetMemoryMap()
+ *
+ * @v len		Memory map size
+ * @v map		Memory map
+ * @v key		Memory map key
+ * @v desclen		Descriptor size
+ * @v descver		Descriptor version
+ * @ret efirc		EFI status code
+ */
+static EFIAPI EFI_STATUS efi_shim_get_memory_map ( UINTN *len,
+						   EFI_MEMORY_DESCRIPTOR *map,
+						   UINTN *key, UINTN *desclen,
+						   UINT32 *descver ) {
+	EFI_RUNTIME_SERVICES *rs = efi_systab->RuntimeServices;
+
+	/* Unlock shim */
+	if ( ! efi_shim_require_loader )
+		efi_shim_unlock();
+
+	/* Uninstall runtime services wrappers, if still installed */
+	if ( rs->SetVariable == efi_shim_set_variable ) {
+		rs->SetVariable = efi_shim_orig_set_variable;
+		DBGC ( &efi_shim, "SHIM uninstalled SetVariable() wrapper\n" );
+	} else if ( rs->SetVariable != efi_shim_orig_set_variable ) {
+		DBGC ( &efi_shim, "SHIM could not uninstall SetVariable() "
+		       "wrapper!\n" );
+	}
+	if ( rs->GetVariable == efi_shim_get_variable ) {
+		rs->GetVariable = efi_shim_orig_get_variable;
+		DBGC ( &efi_shim, "SHIM uninstalled GetVariable() wrapper\n" );
+	} else if ( rs->GetVariable != efi_shim_orig_get_variable ) {
+		DBGC ( &efi_shim, "SHIM could not uninstall GetVariable() "
+		       "wrapper!\n" );
+	}
+
+	/* Hand off to original GetMemoryMap() */
+	return efi_shim_orig_get_memory_map ( len, map, key, desclen,
+					      descver );
+}
+
 /**
  * Inhibit use of PXE base code
  *
@@ -373,15 +368,14 @@ int efi_shim_install ( struct image *shim, EFI_HANDLE handle,
 
 	/* Record original boot and runtime services functions */
 	efi_shim_orig_get_memory_map = bs->GetMemoryMap;
-	efi_shim_orig_exit_boot_services = bs->ExitBootServices;
 	efi_shim_orig_set_variable = rs->SetVariable;
 	efi_shim_orig_get_variable = rs->GetVariable;
 
 	/* Wrap relevant boot and runtime services functions */
 	bs->GetMemoryMap = efi_shim_get_memory_map;
-	bs->ExitBootServices = efi_shim_exit_boot_services;
 	rs->SetVariable = efi_shim_set_variable;
 	rs->GetVariable = efi_shim_get_variable;
+	DBGC ( &efi_shim, "SHIM installed wrappers\n" );
 
 	return 0;
 }
@@ -396,7 +390,7 @@ void efi_shim_uninstall ( void ) {
 
 	/* Restore original boot and runtime services functions */
 	bs->GetMemoryMap = efi_shim_orig_get_memory_map;
-	bs->ExitBootServices = efi_shim_orig_exit_boot_services;
 	rs->SetVariable = efi_shim_orig_set_variable;
 	rs->GetVariable = efi_shim_orig_get_variable;
+	DBGC ( &efi_shim, "SHIM uninstalled wrappers\n" );
 }

From da7b2662890dfb284786f37237a6c92809eee217 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 29 Feb 2024 13:58:50 +0000
Subject: [PATCH 087/237] [uuid] Add uuid_aton() to parse a UUID from a string

Add uuid_aton() to parse a UUID value from a string (analogous to
inet_aton(), inet6_aton(), sock_aton(), etc), treating it as a
32-digit hex string with optional hyphen separators.  The placement of
the separators is not checked: each byte within the hex string may be
separated by a hyphen, or not separated at all.

Add dedicated self-tests for UUID parsing and formatting (already
partially covered by the ":uuid" and ":guid" settings self-tests).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/base16.c          |  15 +++-
 src/core/uuid.c            |  28 +++++++
 src/include/ipxe/base16.h  |   3 +
 src/include/ipxe/errfile.h |   1 +
 src/include/ipxe/uuid.h    |   1 +
 src/tests/tests.c          |   1 +
 src/tests/uuid_test.c      | 156 +++++++++++++++++++++++++++++++++++++
 7 files changed, 203 insertions(+), 2 deletions(-)
 create mode 100644 src/tests/uuid_test.c

diff --git a/src/core/base16.c b/src/core/base16.c
index f9e0f3364..47e35f414 100644
--- a/src/core/base16.c
+++ b/src/core/base16.c
@@ -78,12 +78,23 @@ int hex_decode ( char separator, const char *encoded, void *data, size_t len ) {
 	unsigned int count = 0;
 	unsigned int sixteens;
 	unsigned int units;
+	int optional;
 
+	/* Strip out optionality flag from separator character */
+	optional = ( separator & HEX_DECODE_OPTIONAL );
+	separator &= ~HEX_DECODE_OPTIONAL;
+
+	/* Decode string */
 	while ( *encoded ) {
 
 		/* Check separator, if applicable */
-		if ( count && separator && ( ( *(encoded++) != separator ) ) )
-			return -EINVAL;
+		if ( count && separator ) {
+			if ( *encoded == separator ) {
+				encoded++;
+			} else if ( ! optional ) {
+				return -EINVAL;
+			}
+		}
 
 		/* Extract digits.  Note that either digit may be NUL,
 		 * which would be interpreted as an invalid value by
diff --git a/src/core/uuid.c b/src/core/uuid.c
index c43d4216f..b6600af71 100644
--- a/src/core/uuid.c
+++ b/src/core/uuid.c
@@ -25,7 +25,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
+#include <errno.h>
 #include <byteswap.h>
+#include <ipxe/base16.h>
 #include <ipxe/uuid.h>
 
 /** @file
@@ -53,3 +55,29 @@ const char * uuid_ntoa ( const union uuid *uuid ) {
 		  uuid->canonical.e[4], uuid->canonical.e[5] );
 	return buf;
 }
+
+/**
+ * Parse UUID
+ *
+ * @v string		UUID string
+ * @v uuid		UUID to fill in
+ * @ret rc		Return status code
+ */
+int uuid_aton ( const char *string, union uuid *uuid ) {
+	int len;
+	int rc;
+
+	/* Decode as hex string with optional '-' separator */
+	len = hex_decode ( ( '-' | HEX_DECODE_OPTIONAL ), string, uuid->raw,
+			   sizeof ( *uuid ) );
+	if ( len < 0 ) {
+		rc = len;
+		return rc;
+	}
+
+	/* Check length */
+	if ( len != sizeof ( *uuid ) )
+		return -EINVAL;
+
+	return 0;
+}
diff --git a/src/include/ipxe/base16.h b/src/include/ipxe/base16.h
index 8c44da17e..c9e430e7e 100644
--- a/src/include/ipxe/base16.h
+++ b/src/include/ipxe/base16.h
@@ -12,6 +12,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <stdint.h>
 #include <string.h>
 
+/** Treat separator as optional while decoding */
+#define HEX_DECODE_OPTIONAL 0x80
+
 /**
  * Calculate length of base16-encoded data
  *
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 662f84964..7cff594d5 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -79,6 +79,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_cachedhcp	       ( ERRFILE_CORE | 0x00270000 )
 #define ERRFILE_acpimac		       ( ERRFILE_CORE | 0x00280000 )
 #define ERRFILE_efi_strings	       ( ERRFILE_CORE | 0x00290000 )
+#define ERRFILE_uuid		       ( ERRFILE_CORE | 0x002a0000 )
 
 #define ERRFILE_eisa		     ( ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_isa		     ( ERRFILE_DRIVER | 0x00010000 )
diff --git a/src/include/ipxe/uuid.h b/src/include/ipxe/uuid.h
index 24c46acaf..4874b7382 100644
--- a/src/include/ipxe/uuid.h
+++ b/src/include/ipxe/uuid.h
@@ -48,5 +48,6 @@ static inline void uuid_mangle ( union uuid *uuid ) {
 }
 
 extern const char * uuid_ntoa ( const union uuid *uuid );
+extern int uuid_aton ( const char *string, union uuid *uuid );
 
 #endif /* _IPXE_UUID_H */
diff --git a/src/tests/tests.c b/src/tests/tests.c
index 90cec9489..cb296049e 100644
--- a/src/tests/tests.c
+++ b/src/tests/tests.c
@@ -84,3 +84,4 @@ REQUIRE_OBJECT ( nap_test );
 REQUIRE_OBJECT ( x25519_test );
 REQUIRE_OBJECT ( des_test );
 REQUIRE_OBJECT ( mschapv2_test );
+REQUIRE_OBJECT ( uuid_test );
diff --git a/src/tests/uuid_test.c b/src/tests/uuid_test.c
new file mode 100644
index 000000000..42dc52644
--- /dev/null
+++ b/src/tests/uuid_test.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * UUID tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <string.h>
+#include <byteswap.h>
+#include <ipxe/uuid.h>
+#include <ipxe/test.h>
+
+/** Define an inline UUID value */
+#define UUID( A, B, C, D, E0, E1, E2, E3, E4, E5 ) {		\
+		.a = htonl ( A ),				\
+		.b = htons ( B ),				\
+		.c = htons ( C ),				\
+		.d = htons ( D ),				\
+		.e = { E0, E1, E2, E3, E4, E5 },		\
+	}
+
+/**
+ * Report a uuid_ntoa() test result
+ *
+ * @v uuid		UUID
+ * @v text		Expected textual representation
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void uuid_ntoa_okx ( const union uuid *uuid, const char *text,
+			    const char *file, unsigned int line ) {
+	const char *actual;
+
+	/* Format address */
+	actual = uuid_ntoa ( uuid );
+	DBG ( "uuid_ntoa ( %08x-%04x-%04x-%04x-%02x%02x%02x%02x%02x%02x ) = "
+	      "\"%s\"\n", ntohl ( uuid->canonical.a ),
+	      ntohs ( uuid->canonical.b ), ntohs ( uuid->canonical.c ),
+	      ntohs ( uuid->canonical.d ), uuid->canonical.e[0],
+	      uuid->canonical.e[1], uuid->canonical.e[2], uuid->canonical.e[3],
+	      uuid->canonical.e[4], uuid->canonical.e[5], actual );
+	okx ( strcmp ( actual, text ) == 0, file, line );
+}
+#define uuid_ntoa_ok( value, text ) do {				\
+	static const union uuid uuid = {				\
+		.canonical = value,					\
+	};								\
+	uuid_ntoa_okx ( &uuid, text, __FILE__, __LINE__ );		\
+	} while ( 0 )
+
+/**
+ * Report a uuid_aton() test result
+ *
+ * @v text		Textual representation
+ * @v uuid		Expected UUID
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void uuid_aton_okx ( const char *text, const union uuid *uuid,
+			    const char *file, unsigned int line ) {
+	union uuid actual;
+
+	/* Parse address */
+	okx ( uuid_aton ( text, &actual ) == 0, file, line );
+	DBG ( "uuid_aton ( \"%s\" ) = %s\n", text, uuid_ntoa ( &actual ) );
+	okx ( memcmp ( &actual, uuid, sizeof ( actual ) ) == 0, file, line );
+};
+#define uuid_aton_ok( text, value ) do {				\
+	static const union uuid uuid = {				\
+		.canonical = value,					\
+	};								\
+	uuid_aton_okx ( text, &uuid, __FILE__, __LINE__ );		\
+	} while ( 0 )
+
+/**
+ * Report a uuid_aton() failure test result
+ *
+ * @v text		Textual representation
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void uuid_aton_fail_okx ( const char *text, const char *file,
+				 unsigned int line ) {
+	union uuid actual;
+
+	/* Attempt to parse address */
+	okx ( uuid_aton ( text, &actual ) != 0, file, line );
+}
+#define uuid_aton_fail_ok( text ) \
+	uuid_aton_fail_okx ( text, __FILE__, __LINE__ )
+
+/**
+ * Perform UUID self-tests
+ *
+ */
+static void uuid_test_exec ( void ) {
+
+	/* uuid_ntoa() tests */
+	uuid_ntoa_ok ( UUID ( 0x18725ca6, 0xd699, 0x4e4d, 0xb501,
+			      0xc3, 0x80, 0x91, 0xd2, 0xa4, 0x33 ),
+		       "18725ca6-d699-4e4d-b501-c38091d2a433" );
+	uuid_ntoa_ok ( UUID ( 0x1a969b23, 0xc7d5, 0x40fe, 0xb79a,
+			      0xc9, 0x2e, 0xa3, 0x4a ,0xb4, 0x5b ),
+		       "1a969b23-c7d5-40fe-b79a-c92ea34ab45b" );
+
+	/* uuid_aton() tests */
+	uuid_aton_ok ( "62b907a8-e1a7-460e-82f7-667d84270c84",
+		       UUID ( 0x62b907a8, 0xe1a7, 0x460e, 0x82f7,
+			      0x66, 0x7d, 0x84, 0x27, 0x0c, 0x84 ) );
+	uuid_aton_ok ( "F5D0349C-EF7C-4AD4-B40B-FC2E522A7327",
+		       UUID ( 0xf5d0349c, 0xef7c, 0x4ad4, 0xb40b,
+			      0xfc, 0x2e, 0x52, 0x2a, 0x73, 0x27 ) );
+	uuid_aton_ok ( "4edd80ff7b43465589a02b1e7cffa196",
+		       UUID ( 0x4edd80ff, 0x7b43, 0x4655, 0x89a0,
+			      0x2b, 0x1e, 0x7c, 0xff, 0xa1, 0x96 ) );
+
+	/* uuid_aton() failure tests */
+	uuid_aton_fail_ok ( "628d677b-cf38-471e-9ad9-c8a5d9220055b6" );
+	uuid_aton_fail_ok ( "5071ca26-fc5f-4580-887a-46d9a103e4" );
+	uuid_aton_fail_ok ( "453aee96:0fb5-4aeb-aecd-d060b2121218" );
+	uuid_aton_fail_ok ( "1ccb524a-b8b9-4b17-x5e2-7996867edc7d" );
+	uuid_aton_fail_ok ( "" );
+}
+
+/** UUID self-test */
+struct self_test uuid_test __self_test = {
+	.name = "uuid",
+	.exec = uuid_test_exec,
+};

From 0eb8fbd0bfc9331ecdcd74c9406251e5a6e450af Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 29 Feb 2024 14:04:47 +0000
Subject: [PATCH 088/237] [settings] Add parsing for UUID and GUID settings
 types

The ":uuid" and ":guid" settings types are currently format-only: it
is possible to format a setting as a UUID (via e.g. "show foo:uuid")
but it is not currently possible to parse a string into a UUID setting
(via e.g. "set foo:uuid 406343fe-998b-44be-8a28-44ca38cb202b").

Use uuid_aton() to implement parsing of these settings types, and add
appropriate test cases for both.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/settings.c       | 33 +++++++++++++++++++++++++++++++++
 src/tests/settings_test.c | 14 +++++++++++---
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/src/core/settings.c b/src/core/settings.c
index da075baa8..4593876f0 100644
--- a/src/core/settings.c
+++ b/src/core/settings.c
@@ -2194,6 +2194,37 @@ const struct setting_type setting_type_base64 __setting_type = {
 	.format = format_base64_setting,
 };
 
+/**
+ * Parse UUID/GUID setting value
+ *
+ * @v type		Setting type
+ * @v value		Formatted setting value
+ * @v buf		Buffer to contain raw value
+ * @v len		Length of buffer
+ * @v size		Integer size, in bytes
+ * @ret len		Length of raw value, or negative error
+ */
+static int parse_uuid_setting ( const struct setting_type *type,
+				const char *value, void *buf, size_t len ) {
+	union uuid uuid;
+	int rc;
+
+	/* Parse UUID */
+	if ( ( rc = uuid_aton ( value, &uuid ) ) != 0 )
+		return rc;
+
+	/* Mangle GUID byte ordering */
+	if ( type == &setting_type_guid )
+		uuid_mangle ( &uuid );
+
+	/* Copy value */
+	if ( len > sizeof ( uuid ) )
+		len = sizeof ( uuid );
+	memcpy ( buf, uuid.raw, len );
+
+	return ( sizeof ( uuid ) );
+}
+
 /**
  * Format UUID/GUID setting value
  *
@@ -2227,12 +2258,14 @@ static int format_uuid_setting ( const struct setting_type *type,
 /** UUID setting type */
 const struct setting_type setting_type_uuid __setting_type = {
 	.name = "uuid",
+	.parse = parse_uuid_setting,
 	.format = format_uuid_setting,
 };
 
 /** GUID setting type */
 const struct setting_type setting_type_guid __setting_type = {
 	.name = "guid",
+	.parse = parse_uuid_setting,
 	.format = format_uuid_setting,
 };
 
diff --git a/src/tests/settings_test.c b/src/tests/settings_test.c
index 5da7eb008..edd7b9d70 100644
--- a/src/tests/settings_test.c
+++ b/src/tests/settings_test.c
@@ -420,13 +420,21 @@ static void settings_test_exec ( void ) {
 		    RAW ( 0x80, 0x81, 0x82, 0x83, 0x84, 0x00, 0xff ),
 		    "gIGCg4QA/w==" );
 
-	/* "uuid" setting type (no store capability) */
+	/* "uuid" setting type */
+	storef_ok ( &test_settings, &test_uuid_setting,
+		    "36d22ed9-b64f-4fdb-941b-a54a0854f991",
+		    RAW ( 0x36, 0xd2, 0x2e, 0xd9, 0xb6, 0x4f, 0x4f, 0xdb, 0x94,
+			  0x1b, 0xa5, 0x4a, 0x08, 0x54, 0xf9, 0x91 ) );
+	storef_ok ( &test_settings, &test_guid_setting,
+		    "7ad4478f-c270-4601-a245-78598f25a984",
+		    RAW ( 0x8f, 0x47, 0xd4, 0x7a, 0x70, 0xc2, 0x01, 0x46, 0xa2,
+			  0x45, 0x78, 0x59, 0x8f, 0x25, 0xa9, 0x84 ) );
 	fetchf_ok ( &test_settings, &test_uuid_setting,
-		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a,0xa8,
+		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a, 0xa8,
 			  0x7a, 0x7c, 0xfe, 0x4f, 0xca, 0x4a, 0x57 ),
 		    "1a6a749d-0eda-461a-a87a-7cfe4fca4a57" );
 	fetchf_ok ( &test_settings, &test_guid_setting,
-		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a,0xa8,
+		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a, 0xa8,
 			  0x7a, 0x7c, 0xfe, 0x4f, 0xca, 0x4a, 0x57 ),
 		    "9d746a1a-da0e-1a46-a87a-7cfe4fca4a57" );
 

From 7cd73884e51f6d3649aa2ea0ed0cec0b39192f61 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 29 Feb 2024 14:05:39 +0000
Subject: [PATCH 089/237] [parseopt] Add parse_uuid() for parsing UUID
 command-line arguments

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/parseopt.c         | 23 +++++++++++++++++++++++
 src/include/ipxe/parseopt.h |  2 ++
 2 files changed, 25 insertions(+)

diff --git a/src/core/parseopt.c b/src/core/parseopt.c
index cd3b3101c..7aff14851 100644
--- a/src/core/parseopt.c
+++ b/src/core/parseopt.c
@@ -31,6 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ctype.h>
 #include <errno.h>
 #include <getopt.h>
+#include <ipxe/uuid.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/menu.h>
 #include <ipxe/settings.h>
@@ -124,6 +125,28 @@ int parse_timeout ( char *text, unsigned long *value ) {
 	return 0;
 }
 
+/**
+ * Parse UUID
+ *
+ * @v text		Text
+ * @ret value		UUID value
+ * @ret rc		Return status code
+ */
+int parse_uuid ( char *text, union uuid *value ) {
+	int rc;
+
+	/* Sanity check */
+	assert ( text != NULL );
+
+	/* Parse UUID */
+	if ( ( rc = uuid_aton ( text, value ) ) != 0 ) {
+		printf ( "\"%s\": invalid UUID\n", text );
+		return rc;
+	}
+
+	return 0;
+}
+
 /**
  * Parse network device name
  *
diff --git a/src/include/ipxe/parseopt.h b/src/include/ipxe/parseopt.h
index 829b3431c..ebd18804f 100644
--- a/src/include/ipxe/parseopt.h
+++ b/src/include/ipxe/parseopt.h
@@ -11,6 +11,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
+#include <ipxe/uuid.h>
 #include <ipxe/settings.h>
 
 struct net_device;
@@ -128,6 +129,7 @@ struct named_setting {
 extern int parse_string ( char *text, char **value );
 extern int parse_integer ( char *text, unsigned int *value );
 extern int parse_timeout ( char *text, unsigned long *value );
+extern int parse_uuid ( char *text, union uuid *value );
 extern int parse_netdev ( char *text, struct net_device **netdev );
 extern int
 parse_netdev_configurator ( char *text,

From 1b23d4de25d1dacf164eec442780bbfb026fbda7 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 4 Mar 2024 12:08:01 +0000
Subject: [PATCH 090/237] [efi] Use long forms of device paths in debug
 messages

We currently call ConvertDevicePathToText() with DisplayOnly=TRUE when
constructing a device path to appear within a debug message.  For
ATAPI device paths, this will unfortunately omit some key information:
the textual representation will not indicate which ATA bus or drive is
represented.  This can lead to misleading debug messages that appear
to refer to identical devices.

Fix by setting DisplayOnly=FALSE to select the long form of device
path textual representations.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/interface/efi/efi_debug.c b/src/interface/efi/efi_debug.c
index ad79f0d37..52efebe5f 100644
--- a/src/interface/efi/efi_debug.c
+++ b/src/interface/efi/efi_debug.c
@@ -479,7 +479,7 @@ efi_devpath_text ( EFI_DEVICE_PATH_PROTOCOL *path ) {
 	}
 
 	/* Convert path to a textual representation */
-	wtext = efidpt->ConvertDevicePathToText ( path, TRUE, FALSE );
+	wtext = efidpt->ConvertDevicePathToText ( path, FALSE, FALSE );
 	if ( ! wtext )
 		return NULL;
 

From 75c7904482b5397c41b4a91ce266f25a8a196e13 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 4 Mar 2024 12:50:25 +0000
Subject: [PATCH 091/237] [block] Use drive number as debug message stream ID

We currently use the SAN device pointer as the debug message stream
identifier.  This pointer is not always available: for example, when
booting from a local disk there is no underlying SAN device.

Switch to using the drive number as the debug message colour stream
identifier, so that all block device debug messages may be colourised
consistently.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/interface/pcbios/int13.c | 210 +++++++++++++-------------
 src/core/dummy_sanboot.c              |   4 +-
 src/core/sanboot.c                    |  43 +++---
 src/interface/efi/efi_block.c         |  82 +++++-----
 4 files changed, 173 insertions(+), 166 deletions(-)

diff --git a/src/arch/x86/interface/pcbios/int13.c b/src/arch/x86/interface/pcbios/int13.c
index d6c4d7ebf..2c5e86241 100644
--- a/src/arch/x86/interface/pcbios/int13.c
+++ b/src/arch/x86/interface/pcbios/int13.c
@@ -183,8 +183,8 @@ static int int13_parse_eltorito ( struct san_device *sandev, void *scratch ) {
 	/* Read boot record volume descriptor */
 	if ( ( rc = sandev_read ( sandev, ELTORITO_LBA, 1,
 				  virt_to_user ( boot ) ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not read El Torito boot "
-		       "record volume descriptor: %s\n",
+		DBGC ( sandev->drive, "INT13 drive %02x could not read El "
+		       "Torito boot record volume descriptor: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		return rc;
 	}
@@ -192,10 +192,11 @@ static int int13_parse_eltorito ( struct san_device *sandev, void *scratch ) {
 	/* Check for an El Torito boot catalog */
 	if ( memcmp ( boot, &boot_check, sizeof ( boot_check ) ) == 0 ) {
 		int13->boot_catalog = boot->sector;
-		DBGC ( sandev, "INT13 drive %02x has an El Torito boot catalog "
-		       "at LBA %08x\n", sandev->drive, int13->boot_catalog );
+		DBGC ( sandev->drive, "INT13 drive %02x has an El Torito boot "
+		       "catalog at LBA %08x\n", sandev->drive,
+		       int13->boot_catalog );
 	} else {
-		DBGC ( sandev, "INT13 drive %02x has no El Torito boot "
+		DBGC ( sandev->drive, "INT13 drive %02x has no El Torito boot "
 		       "catalog\n", sandev->drive );
 	}
 
@@ -228,14 +229,14 @@ static int int13_guess_geometry_hdd ( struct san_device *sandev, void *scratch,
 
 	/* Read partition table */
 	if ( ( rc = sandev_read ( sandev, 0, 1, virt_to_user ( mbr ) ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not read "
+		DBGC ( sandev->drive, "INT13 drive %02x could not read "
 		       "partition table to guess geometry: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		return rc;
 	}
-	DBGC2 ( sandev, "INT13 drive %02x has MBR:\n", sandev->drive );
-	DBGC2_HDA ( sandev, 0, mbr, sizeof ( *mbr ) );
-	DBGC ( sandev, "INT13 drive %02x has signature %08x\n",
+	DBGC2 ( sandev->drive, "INT13 drive %02x has MBR:\n", sandev->drive );
+	DBGC2_HDA ( sandev->drive, 0, mbr, sizeof ( *mbr ) );
+	DBGC ( sandev->drive, "INT13 drive %02x has signature %08x\n",
 	       sandev->drive, mbr->signature );
 
 	/* Scan through partition table and modify guesses for
@@ -260,8 +261,8 @@ static int int13_guess_geometry_hdd ( struct san_device *sandev, void *scratch,
 		if ( ( start_cylinder == 0 ) && ( start_head != 0 ) ) {
 			*sectors = ( ( partition->start + 1 - start_sector ) /
 				     start_head );
-			DBGC ( sandev, "INT13 drive %02x guessing C/H/S "
-			       "xx/xx/%d based on partition %d\n",
+			DBGC ( sandev->drive, "INT13 drive %02x guessing "
+			       "C/H/S xx/xx/%d based on partition %d\n",
 			       sandev->drive, *sectors, ( i + 1 ) );
 		}
 
@@ -272,14 +273,14 @@ static int int13_guess_geometry_hdd ( struct san_device *sandev, void *scratch,
 		end_sector = PART_SECTOR ( partition->chs_end );
 		if ( ( end_head + 1 ) > *heads ) {
 			*heads = ( end_head + 1 );
-			DBGC ( sandev, "INT13 drive %02x guessing C/H/S "
-			       "xx/%d/xx based on partition %d\n",
+			DBGC ( sandev->drive, "INT13 drive %02x guessing "
+			       "C/H/S xx/%d/xx based on partition %d\n",
 			       sandev->drive, *heads, ( i + 1 ) );
 		}
 		if ( end_sector > *sectors ) {
 			*sectors = end_sector;
-			DBGC ( sandev, "INT13 drive %02x guessing C/H/S "
-			       "xx/xx/%d based on partition %d\n",
+			DBGC ( sandev->drive, "INT13 drive %02x guessing "
+			       "C/H/S xx/xx/%d based on partition %d\n",
 			       sandev->drive, *sectors, ( i + 1 ) );
 		}
 	}
@@ -343,9 +344,10 @@ static int int13_guess_geometry_fdd ( struct san_device *sandev,
 		*heads = INT13_FDD_HEADS ( geometry );
 		*sectors = INT13_FDD_SECTORS ( geometry );
 		if ( ( cylinders * (*heads) * (*sectors) ) == blocks ) {
-			DBGC ( sandev, "INT13 drive %02x guessing C/H/S "
-			       "%d/%d/%d based on size %dK\n", sandev->drive,
-			       cylinders, *heads, *sectors, ( blocks / 2 ) );
+			DBGC ( sandev->drive, "INT13 drive %02x guessing "
+			       "C/H/S %d/%d/%d based on size %dK\n",
+			       sandev->drive, cylinders, *heads, *sectors,
+			       ( blocks / 2 ) );
 			return 0;
 		}
 	}
@@ -355,8 +357,9 @@ static int int13_guess_geometry_fdd ( struct san_device *sandev,
 	 */
 	*heads = 2;
 	*sectors = 18;
-	DBGC ( sandev, "INT13 drive %02x guessing C/H/S xx/%d/%d based on size "
-	       "%dK\n", sandev->drive, *heads, *sectors, ( blocks / 2 ) );
+	DBGC ( sandev->drive, "INT13 drive %02x guessing C/H/S xx/%d/%d "
+	       "based on size %dK\n", sandev->drive, *heads, *sectors,
+	       ( blocks / 2 ) );
 	return 0;
 }
 
@@ -431,8 +434,8 @@ static void int13_sync_num_drives ( void ) {
 		required = ( ( max_drive & 0x7f ) + 1 );
 		if ( *counter < required ) {
 			*counter = required;
-			DBGC ( sandev, "INT13 drive %02x added to drive count: "
-			       "%d HDDs, %d FDDs\n",
+			DBGC ( sandev->drive, "INT13 drive %02x added to "
+			       "drive count: %d HDDs, %d FDDs\n",
 			       sandev->drive, num_drives, num_fdds );
 		}
 	}
@@ -472,7 +475,7 @@ static int int13_reset ( struct san_device *sandev,
 			 struct i386_all_regs *ix86 __unused ) {
 	int rc;
 
-	DBGC2 ( sandev, "Reset drive\n" );
+	DBGC2 ( sandev->drive, "Reset drive\n" );
 
 	/* Reset SAN device */
 	if ( ( rc = sandev_reset ( sandev ) ) != 0 )
@@ -491,7 +494,7 @@ static int int13_get_last_status ( struct san_device *sandev,
 				   struct i386_all_regs *ix86 __unused ) {
 	struct int13_data *int13 = sandev->priv;
 
-	DBGC2 ( sandev, "Get status of last operation\n" );
+	DBGC2 ( sandev->drive, "Get status of last operation\n" );
 	return int13->last_status;
 }
 
@@ -524,8 +527,8 @@ static int int13_rw_sectors ( struct san_device *sandev,
 
 	/* Validate blocksize */
 	if ( sandev_blksize ( sandev ) != INT13_BLKSIZE ) {
-		DBGC ( sandev, "\nINT 13 drive %02x invalid blocksize (%zd) "
-		       "for non-extended read/write\n",
+		DBGC ( sandev->drive, "\nINT 13 drive %02x invalid blocksize "
+		       "(%zd) for non-extended read/write\n",
 		       sandev->drive, sandev_blksize ( sandev ) );
 		return -INT13_STATUS_INVALID;
 	}
@@ -537,9 +540,10 @@ static int int13_rw_sectors ( struct san_device *sandev,
 	if ( ( cylinder >= int13->cylinders ) ||
 	     ( head >= int13->heads ) ||
 	     ( sector < 1 ) || ( sector > int13->sectors_per_track ) ) {
-		DBGC ( sandev, "C/H/S %d/%d/%d out of range for geometry "
-		       "%d/%d/%d\n", cylinder, head, sector, int13->cylinders,
-		       int13->heads, int13->sectors_per_track );
+		DBGC ( sandev->drive, "C/H/S %d/%d/%d out of range for "
+		       "geometry %d/%d/%d\n", cylinder, head, sector,
+		       int13->cylinders, int13->heads,
+		       int13->sectors_per_track );
 		return -INT13_STATUS_INVALID;
 	}
 	lba = ( ( ( ( cylinder * int13->heads ) + head )
@@ -547,13 +551,13 @@ static int int13_rw_sectors ( struct san_device *sandev,
 	count = ix86->regs.al;
 	buffer = real_to_user ( ix86->segs.es, ix86->regs.bx );
 
-	DBGC2 ( sandev, "C/H/S %d/%d/%d = LBA %08lx <-> %04x:%04x (count %d)\n",
-		cylinder, head, sector, lba, ix86->segs.es, ix86->regs.bx,
-		count );
+	DBGC2 ( sandev->drive, "C/H/S %d/%d/%d = LBA %08lx <-> %04x:%04x "
+		"(count %d)\n", cylinder, head, sector, lba, ix86->segs.es,
+		ix86->regs.bx, count );
 
 	/* Read from / write to block device */
 	if ( ( rc = sandev_rw ( sandev, lba, count, buffer ) ) != 0 ){
-		DBGC ( sandev, "INT13 drive %02x I/O failed: %s\n",
+		DBGC ( sandev->drive, "INT13 drive %02x I/O failed: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		return -INT13_STATUS_READ_ERROR;
 	}
@@ -577,7 +581,7 @@ static int int13_rw_sectors ( struct san_device *sandev,
 static int int13_read_sectors ( struct san_device *sandev,
 				struct i386_all_regs *ix86 ) {
 
-	DBGC2 ( sandev, "Read: " );
+	DBGC2 ( sandev->drive, "Read: " );
 	return int13_rw_sectors ( sandev, ix86, sandev_read );
 }
 
@@ -597,7 +601,7 @@ static int int13_read_sectors ( struct san_device *sandev,
 static int int13_write_sectors ( struct san_device *sandev,
 				 struct i386_all_regs *ix86 ) {
 
-	DBGC2 ( sandev, "Write: " );
+	DBGC2 ( sandev->drive, "Write: " );
 	return int13_rw_sectors ( sandev, ix86, sandev_write );
 }
 
@@ -619,12 +623,12 @@ static int int13_get_parameters ( struct san_device *sandev,
 	unsigned int max_head = int13->heads - 1;
 	unsigned int max_sector = int13->sectors_per_track; /* sic */
 
-	DBGC2 ( sandev, "Get drive parameters\n" );
+	DBGC2 ( sandev->drive, "Get drive parameters\n" );
 
 	/* Validate blocksize */
 	if ( sandev_blksize ( sandev ) != INT13_BLKSIZE ) {
-		DBGC ( sandev, "\nINT 13 drive %02x invalid blocksize (%zd) "
-		       "for non-extended parameters\n",
+		DBGC ( sandev->drive, "\nINT 13 drive %02x invalid blocksize "
+		       "(%zd) for non-extended parameters\n",
 		       sandev->drive, sandev_blksize ( sandev ) );
 		return -INT13_STATUS_INVALID;
 	}
@@ -657,7 +661,7 @@ static int int13_get_disk_type ( struct san_device *sandev,
 				 struct i386_all_regs *ix86 ) {
 	uint32_t blocks;
 
-	DBGC2 ( sandev, "Get disk type\n" );
+	DBGC2 ( sandev->drive, "Get disk type\n" );
 
 	if ( int13_is_fdd ( sandev ) ) {
 		return INT13_DISK_TYPE_FDD;
@@ -682,7 +686,7 @@ static int int13_extension_check ( struct san_device *sandev,
 				   struct i386_all_regs *ix86 ) {
 
 	if ( ( ix86->regs.bx == 0x55aa ) && ! int13_is_fdd ( sandev ) ) {
-		DBGC2 ( sandev, "INT13 extensions installation check\n" );
+		DBGC2 ( sandev->drive, "INT13 extensions check\n" );
 		ix86->regs.bx = 0xaa55;
 		ix86->regs.cx = ( INT13_EXTENSION_LINEAR |
 				  INT13_EXTENSION_EDD |
@@ -725,7 +729,8 @@ static int int13_extended_rw ( struct san_device *sandev,
 	get_real ( bufsize, ix86->segs.ds,
 		   ( ix86->regs.si + offsetof ( typeof ( addr ), bufsize ) ) );
 	if ( bufsize < offsetof ( typeof ( addr ), buffer_phys ) ) {
-		DBGC2 ( sandev, "<invalid buffer size %#02x\n>\n", bufsize );
+		DBGC2 ( sandev->drive, "<invalid buffer size %#02x\n>\n",
+			bufsize );
 		return -INT13_STATUS_INVALID;
 	}
 
@@ -733,17 +738,18 @@ static int int13_extended_rw ( struct san_device *sandev,
 	memset ( &addr, 0, sizeof ( addr ) );
 	copy_from_real ( &addr, ix86->segs.ds, ix86->regs.si, bufsize );
 	lba = addr.lba;
-	DBGC2 ( sandev, "LBA %08llx <-> ", ( ( unsigned long long ) lba ) );
+	DBGC2 ( sandev->drive, "LBA %08llx <-> ",
+		( ( unsigned long long ) lba ) );
 	if ( ( addr.count == 0xff ) ||
 	     ( ( addr.buffer.segment == 0xffff ) &&
 	       ( addr.buffer.offset == 0xffff ) ) ) {
 		buffer = phys_to_user ( addr.buffer_phys );
-		DBGC2 ( sandev, "%08llx",
+		DBGC2 ( sandev->drive, "%08llx",
 			( ( unsigned long long ) addr.buffer_phys ) );
 	} else {
 		buffer = real_to_user ( addr.buffer.segment,
 					addr.buffer.offset );
-		DBGC2 ( sandev, "%04x:%04x", addr.buffer.segment,
+		DBGC2 ( sandev->drive, "%04x:%04x", addr.buffer.segment,
 			addr.buffer.offset );
 	}
 	if ( addr.count <= 0x7f ) {
@@ -751,15 +757,15 @@ static int int13_extended_rw ( struct san_device *sandev,
 	} else if ( addr.count == 0xff ) {
 		count = addr.long_count;
 	} else {
-		DBGC2 ( sandev, " <invalid count %#02x>\n", addr.count );
+		DBGC2 ( sandev->drive, " <invalid count %#02x>\n", addr.count );
 		return -INT13_STATUS_INVALID;
 	}
-	DBGC2 ( sandev, " (count %ld)\n", count );
+	DBGC2 ( sandev->drive, " (count %ld)\n", count );
 
 	/* Read from / write to block device */
 	if ( ( rc = sandev_rw ( sandev, lba, count, buffer ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x extended I/O failed: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( sandev->drive, "INT13 drive %02x extended I/O failed: "
+		       "%s\n", sandev->drive, strerror ( rc ) );
 		/* Record that no blocks were transferred successfully */
 		addr.count = 0;
 		put_real ( addr.count, ix86->segs.ds,
@@ -781,7 +787,7 @@ static int int13_extended_rw ( struct san_device *sandev,
 static int int13_extended_read ( struct san_device *sandev,
 				 struct i386_all_regs *ix86 ) {
 
-	DBGC2 ( sandev, "Extended read: " );
+	DBGC2 ( sandev->drive, "Extended read: " );
 	return int13_extended_rw ( sandev, ix86, sandev_read );
 }
 
@@ -795,7 +801,7 @@ static int int13_extended_read ( struct san_device *sandev,
 static int int13_extended_write ( struct san_device *sandev,
 				  struct i386_all_regs *ix86 ) {
 
-	DBGC2 ( sandev, "Extended write: " );
+	DBGC2 ( sandev->drive, "Extended write: " );
 	return int13_extended_rw ( sandev, ix86, sandev_write );
 }
 
@@ -818,7 +824,7 @@ static int int13_extended_verify ( struct san_device *sandev,
 				 sizeof ( addr ));
 		lba = addr.lba;
 		count = addr.count;
-		DBGC2 ( sandev, "Verify: LBA %08llx (count %ld)\n",
+		DBGC2 ( sandev->drive, "Verify: LBA %08llx (count %ld)\n",
 			( ( unsigned long long ) lba ), count );
 	}
 
@@ -845,7 +851,7 @@ static int int13_extended_seek ( struct san_device *sandev,
 				 sizeof ( addr ));
 		lba = addr.lba;
 		count = addr.count;
-		DBGC2 ( sandev, "Seek: LBA %08llx (count %ld)\n",
+		DBGC2 ( sandev->drive, "Seek: LBA %08llx (count %ld)\n",
 			( ( unsigned long long ) lba ), count );
 	}
 
@@ -879,8 +885,8 @@ static int int13_device_path_info ( struct san_device *sandev,
 	/* Get underlying hardware device */
 	device = identify_device ( &sanpath->block );
 	if ( ! device ) {
-		DBGC ( sandev, "INT13 drive %02x cannot identify hardware "
-		       "device\n", sandev->drive );
+		DBGC ( sandev->drive, "INT13 drive %02x cannot identify "
+		       "hardware device\n", sandev->drive );
 		return -ENODEV;
 	}
 
@@ -895,16 +901,16 @@ static int int13_device_path_info ( struct san_device *sandev,
 		dpi->interface_path.pci.channel = 0xff; /* unused */
 		break;
 	default:
-		DBGC ( sandev, "INT13 drive %02x unrecognised bus type %d\n",
-		       sandev->drive, desc->bus_type );
+		DBGC ( sandev->drive, "INT13 drive %02x unrecognised bus "
+		       "type %d\n", sandev->drive, desc->bus_type );
 		return -ENOTSUP;
 	}
 
 	/* Get EDD block device description */
 	if ( ( rc = edd_describe ( &sanpath->block, &dpi->interface_type,
 				   &dpi->device_path ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x cannot identify block device: "
-		       "%s\n", sandev->drive, strerror ( rc ) );
+		DBGC ( sandev->drive, "INT13 drive %02x cannot identify "
+		       "block device: %s\n", sandev->drive, strerror ( rc ) );
 		return rc;
 	}
 
@@ -938,8 +944,8 @@ static int int13_get_extended_parameters ( struct san_device *sandev,
 	get_real ( bufsize, ix86->segs.ds,
 		   ( ix86->regs.si + offsetof ( typeof ( params ), bufsize )));
 
-	DBGC2 ( sandev, "Get extended drive parameters to %04x:%04x+%02x\n",
-		ix86->segs.ds, ix86->regs.si, bufsize );
+	DBGC2 ( sandev->drive, "Get extended drive parameters to "
+		"%04x:%04x+%02x\n", ix86->segs.ds, ix86->regs.si, bufsize );
 
 	/* Build drive parameters */
 	memset ( &params, 0, sizeof ( params ) );
@@ -955,8 +961,8 @@ static int int13_get_extended_parameters ( struct san_device *sandev,
 	params.sector_size = sandev_blksize ( sandev );
 	memset ( &params.dpte, 0xff, sizeof ( params.dpte ) );
 	if ( ( rc = int13_device_path_info ( sandev, &params.dpi ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not provide device "
-		       "path information: %s\n",
+		DBGC ( sandev->drive, "INT13 drive %02x could not provide "
+		       "device path information: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		len = offsetof ( typeof ( params ), dpi );
 	}
@@ -973,11 +979,11 @@ static int int13_get_extended_parameters ( struct san_device *sandev,
 		params.bufsize = offsetof ( typeof ( params ), dpi );
 	}
 
-	DBGC ( sandev, "INT 13 drive %02x described using extended "
+	DBGC ( sandev->drive, "INT 13 drive %02x described using extended "
 	       "parameters:\n", sandev->drive );
 	address.segment = ix86->segs.ds;
 	address.offset = ix86->regs.si;
-	DBGC_HDA ( sandev, address, &params, len );
+	DBGC_HDA ( sandev->drive, address, &params, len );
 
 	/* Return drive parameters */
 	if ( len > bufsize )
@@ -998,13 +1004,13 @@ static int int13_cdrom_status_terminate ( struct san_device *sandev,
 					  struct i386_all_regs *ix86 ) {
 	struct int13_cdrom_specification specification;
 
-	DBGC2 ( sandev, "Get CD-ROM emulation status to %04x:%04x%s\n",
+	DBGC2 ( sandev->drive, "Get CD-ROM emulation status to %04x:%04x%s\n",
 		ix86->segs.ds, ix86->regs.si,
 		( ix86->regs.al ? "" : " and terminate" ) );
 
 	/* Fail if we are not a CD-ROM */
 	if ( ! sandev->is_cdrom ) {
-		DBGC ( sandev, "INT13 drive %02x is not a CD-ROM\n",
+		DBGC ( sandev->drive, "INT13 drive %02x is not a CD-ROM\n",
 		       sandev->drive );
 		return -INT13_STATUS_INVALID;
 	}
@@ -1039,11 +1045,12 @@ static int int13_cdrom_read_boot_catalog ( struct san_device *sandev,
 	/* Read parameters from command packet */
 	copy_from_real ( &command, ix86->segs.ds, ix86->regs.si,
 			 sizeof ( command ) );
-	DBGC2 ( sandev, "Read CD-ROM boot catalog to %08x\n", command.buffer );
+	DBGC2 ( sandev->drive, "Read CD-ROM boot catalog to %08x\n",
+		command.buffer );
 
 	/* Fail if we have no boot catalog */
 	if ( ! int13->boot_catalog ) {
-		DBGC ( sandev, "INT13 drive %02x has no boot catalog\n",
+		DBGC ( sandev->drive, "INT13 drive %02x has no boot catalog\n",
 		       sandev->drive );
 		return -INT13_STATUS_INVALID;
 	}
@@ -1052,8 +1059,8 @@ static int int13_cdrom_read_boot_catalog ( struct san_device *sandev,
 	/* Read from boot catalog */
 	if ( ( rc = sandev_read ( sandev, start, command.count,
 				  phys_to_user ( command.buffer ) ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not read boot catalog: "
-		       "%s\n", sandev->drive, strerror ( rc ) );
+		DBGC ( sandev->drive, "INT13 drive %02x could not read boot "
+		       "catalog: %s\n", sandev->drive, strerror ( rc ) );
 		return -INT13_STATUS_READ_ERROR;
 	}
 
@@ -1080,8 +1087,8 @@ static __asmcall __used void int13 ( struct i386_all_regs *ix86 ) {
 		if ( bios_drive != sandev->drive ) {
 			/* Remap any accesses to this drive's natural number */
 			if ( bios_drive == int13->natural_drive ) {
-				DBGC2 ( sandev, "INT13,%02x (%02x) remapped to "
-					"(%02x)\n", ix86->regs.ah,
+				DBGC2 ( sandev->drive, "INT13,%02x (%02x) "
+					"remapped to (%02x)\n", ix86->regs.ah,
 					bios_drive, sandev->drive );
 				ix86->regs.dl = sandev->drive;
 				return;
@@ -1094,7 +1101,7 @@ static __asmcall __used void int13 ( struct i386_all_regs *ix86 ) {
 			}
 		}
 		
-		DBGC2 ( sandev, "INT13,%02x (%02x): ",
+		DBGC2 ( sandev->drive, "INT13,%02x (%02x): ",
 			ix86->regs.ah, bios_drive );
 
 		switch ( command ) {
@@ -1141,7 +1148,7 @@ static __asmcall __used void int13 ( struct i386_all_regs *ix86 ) {
 			status = int13_cdrom_read_boot_catalog ( sandev, ix86 );
 			break;
 		default:
-			DBGC2 ( sandev, "*** Unrecognised INT13 ***\n" );
+			DBGC2 ( sandev->drive, "*** Unrecognised INT13 ***\n" );
 			status = -INT13_STATUS_INVALID;
 			break;
 		}
@@ -1152,8 +1159,9 @@ static __asmcall __used void int13 ( struct i386_all_regs *ix86 ) {
 		/* Negative status indicates an error */
 		if ( status < 0 ) {
 			status = -status;
-			DBGC ( sandev, "INT13,%02x (%02x) failed with status "
-			       "%02x\n", ix86->regs.ah, sandev->drive, status );
+			DBGC ( sandev->drive, "INT13,%02x (%02x) failed with "
+			       "status %02x\n", ix86->regs.ah, sandev->drive,
+			       status );
 		} else {
 			ix86->flags &= ~CF;
 		}
@@ -1269,7 +1277,7 @@ static int int13_hook ( unsigned int drive, struct uri **uris,
 
 	/* Register SAN device */
 	if ( ( rc = register_sandev ( sandev, drive, flags ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not register: %s\n",
+		DBGC ( drive, "INT13 drive %02x could not register: %s\n",
 		       drive, strerror ( rc ) );
 		goto err_register;
 	}
@@ -1289,10 +1297,9 @@ static int int13_hook ( unsigned int drive, struct uri **uris,
 	     ( ( rc = int13_guess_geometry ( sandev, scratch ) ) != 0 ) )
 		goto err_guess_geometry;
 
-	DBGC ( sandev, "INT13 drive %02x (naturally %02x) registered with "
-	       "C/H/S geometry %d/%d/%d\n",
-	       sandev->drive, int13->natural_drive, int13->cylinders,
-	       int13->heads, int13->sectors_per_track );
+	DBGC ( drive, "INT13 drive %02x (naturally %02x) registered with "
+	       "C/H/S geometry %d/%d/%d\n", drive, int13->natural_drive,
+	       int13->cylinders, int13->heads, int13->sectors_per_track );
 
 	/* Hook INT 13 vector if not already hooked */
 	if ( need_hook ) {
@@ -1332,7 +1339,7 @@ static void int13_unhook ( unsigned int drive ) {
 	/* Find drive */
 	sandev = sandev_find ( drive );
 	if ( ! sandev ) {
-		DBG ( "INT13 cannot find drive %02x\n", drive );
+		DBGC ( drive, "INT13 drive %02x is not a SAN drive\n", drive );
 		return;
 	}
 
@@ -1343,7 +1350,7 @@ static void int13_unhook ( unsigned int drive ) {
 	 * to do so reliably.
 	 */
 
-	DBGC ( sandev, "INT13 drive %02x unregistered\n", sandev->drive );
+	DBGC ( drive, "INT13 drive %02x unregistered\n", drive );
 
 	/* Unhook INT 13 vector if no more drives */
 	if ( ! have_sandevs() ) {
@@ -1387,8 +1394,8 @@ static int int13_load_mbr ( unsigned int drive, struct segoff *address ) {
 			       : "a" ( 0x0201 ), "b" ( *address ),
 				 "c" ( 1 ), "d" ( drive ) );
 	if ( status ) {
-		DBG ( "INT13 drive %02x could not read MBR (status %04x)\n",
-		      drive, status );
+		DBGC ( drive, "INT13 drive %02x could not read MBR (status "
+		       "%04x)\n", drive, status );
 		return -EIO;
 	}
 
@@ -1397,8 +1404,8 @@ static int int13_load_mbr ( unsigned int drive, struct segoff *address ) {
 		   ( address->offset +
 		     offsetof ( struct master_boot_record, magic ) ) );
 	if ( magic != INT13_MBR_MAGIC ) {
-		DBG ( "INT13 drive %02x does not contain a valid MBR\n",
-		      drive );
+		DBGC ( drive, "INT13 drive %02x does not contain a valid MBR\n",
+		       drive );
 		return -ENOEXEC;
 	}
 
@@ -1444,8 +1451,8 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
 			       : "a" ( 0x4d00 ), "d" ( drive ),
 				 "S" ( __from_data16 ( &eltorito_cmd ) ) );
 	if ( status ) {
-		DBG ( "INT13 drive %02x could not read El Torito boot catalog "
-		      "(status %04x)\n", drive, status );
+		DBGC ( drive, "INT13 drive %02x could not read El Torito boot "
+		       "catalog (status %04x)\n", drive, status );
 		return -EIO;
 	}
 	copy_from_user ( &catalog, phys_to_user ( eltorito_cmd.buffer ), 0,
@@ -1453,26 +1460,27 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
 
 	/* Sanity checks */
 	if ( catalog.valid.platform_id != ELTORITO_PLATFORM_X86 ) {
-		DBG ( "INT13 drive %02x El Torito specifies unknown platform "
-		      "%02x\n", drive, catalog.valid.platform_id );
+		DBGC ( drive, "INT13 drive %02x El Torito specifies unknown "
+		       "platform %02x\n", drive, catalog.valid.platform_id );
 		return -ENOEXEC;
 	}
 	if ( catalog.boot.indicator != ELTORITO_BOOTABLE ) {
-		DBG ( "INT13 drive %02x El Torito is not bootable\n", drive );
+		DBGC ( drive, "INT13 drive %02x El Torito is not bootable\n",
+		       drive );
 		return -ENOEXEC;
 	}
 	if ( catalog.boot.media_type != ELTORITO_NO_EMULATION ) {
-		DBG ( "INT13 drive %02x El Torito requires emulation "
+		DBGC ( drive, "INT13 drive %02x El Torito requires emulation "
 		       "type %02x\n", drive, catalog.boot.media_type );
 		return -ENOTSUP;
 	}
-	DBG ( "INT13 drive %02x El Torito boot image at LBA %08x (count %d)\n",
-	      drive, catalog.boot.start, catalog.boot.length );
+	DBGC ( drive, "INT13 drive %02x El Torito boot image at LBA %08x "
+	       "(count %d)\n", drive, catalog.boot.start, catalog.boot.length );
 	address->segment = ( catalog.boot.load_segment ?
 			     catalog.boot.load_segment : 0x7c0 );
 	address->offset = 0;
-	DBG ( "INT13 drive %02x El Torito boot image loads at %04x:%04x\n",
-	      drive, address->segment, address->offset );
+	DBGC ( drive, "INT13 drive %02x El Torito boot image loads at "
+	       "%04x:%04x\n", drive, address->segment, address->offset );
 
 	/* Use INT 13, 42 to read the boot image */
 	eltorito_address.bufsize =
@@ -1491,8 +1499,8 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
 			       : "a" ( 0x4200 ), "d" ( drive ),
 				 "S" ( __from_data16 ( &eltorito_address ) ) );
 	if ( status ) {
-		DBG ( "INT13 drive %02x could not read El Torito boot image "
-		      "(status %04x)\n", drive, status );
+		DBGC ( drive, "INT13 drive %02x could not read El Torito boot "
+		       "image (status %04x)\n", drive, status );
 		return -EIO;
 	}
 
@@ -1533,8 +1541,8 @@ static int int13_boot ( unsigned int drive, const char *filename __unused ) {
 	/* Jump to boot sector */
 	if ( ( rc = call_bootsector ( address.segment, address.offset,
 				      drive ) ) != 0 ) {
-		DBG ( "INT13 drive %02x boot returned: %s\n",
-		      drive, strerror ( rc ) );
+		DBGC ( drive, "INT13 drive %02x boot returned: %s\n",
+		       drive, strerror ( rc ) );
 		return rc;
 	}
 
diff --git a/src/core/dummy_sanboot.c b/src/core/dummy_sanboot.c
index e6293099a..506b20c12 100644
--- a/src/core/dummy_sanboot.c
+++ b/src/core/dummy_sanboot.c
@@ -55,7 +55,7 @@ static int dummy_san_hook ( unsigned int drive, struct uri **uris,
 
 	/* Register SAN device */
 	if ( ( rc = register_sandev ( sandev, drive, flags ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x could not register: %s\n",
+		DBGC ( sandev->drive, "SAN %#02x could not register: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		goto err_register;
 	}
@@ -80,7 +80,7 @@ static void dummy_san_unhook ( unsigned int drive ) {
 	/* Find drive */
 	sandev = sandev_find ( drive );
 	if ( ! sandev ) {
-		DBG ( "SAN %#02x does not exist\n", drive );
+		DBGC ( drive, "SAN %#02x does not exist\n", drive );
 		return;
 	}
 
diff --git a/src/core/sanboot.c b/src/core/sanboot.c
index cabc48430..7e3a09e45 100644
--- a/src/core/sanboot.c
+++ b/src/core/sanboot.c
@@ -197,7 +197,7 @@ static int sanpath_open ( struct san_path *sanpath ) {
 
 	/* Open interface */
 	if ( ( rc = xfer_open_uri ( &sanpath->block, sanpath->uri ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x.%d could not (re)open URI: "
+		DBGC ( sandev->drive, "SAN %#02x.%d could not (re)open URI: "
 		       "%s\n", sandev->drive, sanpath->index, strerror ( rc ) );
 		return rc;
 	}
@@ -265,7 +265,7 @@ static void sanpath_block_close ( struct san_path *sanpath, int rc ) {
 	/* Any closure is an error from our point of view */
 	if ( rc == 0 )
 		rc = -ENOTCONN;
-	DBGC ( sandev, "SAN %#02x.%d closed: %s\n",
+	DBGC ( sandev->drive, "SAN %#02x.%d closed: %s\n",
 	       sandev->drive, sanpath->index, strerror ( rc ) );
 
 	/* Close path */
@@ -307,11 +307,11 @@ static void sanpath_step ( struct san_path *sanpath ) {
 
 	/* Mark as active path or close as applicable */
 	if ( ! sandev->active ) {
-		DBGC ( sandev, "SAN %#02x.%d is active\n",
+		DBGC ( sandev->drive, "SAN %#02x.%d is active\n",
 		       sandev->drive, sanpath->index );
 		sandev->active = sanpath;
 	} else {
-		DBGC ( sandev, "SAN %#02x.%d is available\n",
+		DBGC ( sandev->drive, "SAN %#02x.%d is available\n",
 		       sandev->drive, sanpath->index );
 		sanpath_close ( sanpath, 0 );
 	}
@@ -398,8 +398,9 @@ int sandev_reopen ( struct san_device *sandev ) {
 				rc = sanpath->path_rc;
 				break;
 			}
-			DBGC ( sandev, "SAN %#02x never became available: %s\n",
-			       sandev->drive, strerror ( rc ) );
+			DBGC ( sandev->drive, "SAN %#02x never became "
+			       "available: %s\n", sandev->drive,
+			       strerror ( rc ) );
 			goto err_none;
 		}
 	}
@@ -453,8 +454,9 @@ static int sandev_command_rw ( struct san_device *sandev,
 	if ( ( rc = params->rw.block_rw ( &sanpath->block, &sandev->command,
 					  params->rw.lba, params->rw.count,
 					  params->rw.buffer, len ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x.%d could not initiate read/write: "
-		       "%s\n", sandev->drive, sanpath->index, strerror ( rc ) );
+		DBGC ( sandev->drive, "SAN %#02x.%d could not initiate "
+		       "read/write: %s\n", sandev->drive, sanpath->index,
+		       strerror ( rc ) );
 		return rc;
 	}
 
@@ -480,8 +482,9 @@ sandev_command_read_capacity ( struct san_device *sandev,
 	/* Initiate read capacity command */
 	if ( ( rc = block_read_capacity ( &sanpath->block,
 					  &sandev->command ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x.%d could not initiate read capacity: "
-		       "%s\n", sandev->drive, sanpath->index, strerror ( rc ) );
+		DBGC ( sandev->drive, "SAN %#02x.%d could not initiate read "
+		       "capacity: %s\n", sandev->drive, sanpath->index,
+		       strerror ( rc ) );
 		return rc;
 	}
 
@@ -565,7 +568,7 @@ sandev_command ( struct san_device *sandev,
 int sandev_reset ( struct san_device *sandev ) {
 	int rc;
 
-	DBGC ( sandev, "SAN %#02x reset\n", sandev->drive );
+	DBGC ( sandev->drive, "SAN %#02x reset\n", sandev->drive );
 
 	/* Close and reopen underlying block device */
 	if ( ( rc = sandev_reopen ( sandev ) ) != 0 )
@@ -698,8 +701,8 @@ static int sandev_describe ( struct san_device *sandev ) {
 			if ( ! desc )
 				continue;
 			if ( ( rc = desc->model->complete ( desc ) ) != 0 ) {
-				DBGC ( sandev, "SAN %#02x.%d could not be "
-				       "described: %s\n", sandev->drive,
+				DBGC ( sandev->drive, "SAN %#02x.%d could not "
+				       "be described: %s\n", sandev->drive,
 				       sanpath->index, strerror ( rc ) );
 				return rc;
 			}
@@ -792,8 +795,8 @@ static int sandev_parse_iso9660 ( struct san_device *sandev ) {
 	/* Read primary volume descriptor */
 	if ( ( rc = sandev_read ( sandev, lba, count,
 				  virt_to_user ( scratch ) ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x could not read ISO9660 primary"
-		       "volume descriptor: %s\n",
+		DBGC ( sandev->drive, "SAN %#02x could not read ISO9660 "
+		       "primary volume descriptor: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		goto err_rw;
 	}
@@ -801,8 +804,8 @@ static int sandev_parse_iso9660 ( struct san_device *sandev ) {
 	/* Configure as CD-ROM if applicable */
 	if ( memcmp ( &scratch->primary.fixed, &primary_check,
 		      sizeof ( primary_check ) ) == 0 ) {
-		DBGC ( sandev, "SAN %#02x contains an ISO9660 filesystem; "
-		       "treating as CD-ROM\n", sandev->drive );
+		DBGC ( sandev->drive, "SAN %#02x contains an ISO9660 "
+		       "filesystem; treating as CD-ROM\n", sandev->drive );
 		sandev->blksize_shift = blksize_shift;
 		sandev->is_cdrom = 1;
 	}
@@ -871,7 +874,7 @@ int register_sandev ( struct san_device *sandev, unsigned int drive,
 
 	/* Check that drive number is not in use */
 	if ( sandev_find ( drive ) != NULL ) {
-		DBGC ( sandev, "SAN %#02x is already in use\n", drive );
+		DBGC ( sandev->drive, "SAN %#02x is already in use\n", drive );
 		rc = -EADDRINUSE;
 		goto err_in_use;
 	}
@@ -902,7 +905,7 @@ int register_sandev ( struct san_device *sandev, unsigned int drive,
 
 	/* Add to list of SAN devices */
 	list_add_tail ( &sandev->list, &san_devices );
-	DBGC ( sandev, "SAN %#02x registered\n", sandev->drive );
+	DBGC ( sandev->drive, "SAN %#02x registered\n", sandev->drive );
 
 	return 0;
 
@@ -936,7 +939,7 @@ void unregister_sandev ( struct san_device *sandev ) {
 	/* Remove ACPI descriptors */
 	sandev_undescribe ( sandev );
 
-	DBGC ( sandev, "SAN %#02x unregistered\n", sandev->drive );
+	DBGC ( sandev->drive, "SAN %#02x unregistered\n", sandev->drive );
 }
 
 /** The "san-drive" setting */
diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index cb73260d5..86186c0ad 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -101,7 +101,7 @@ static int efi_block_rw ( struct san_device *sandev, uint64_t lba,
 	/* Sanity check */
 	count = ( len / block->media.BlockSize );
 	if ( ( count * block->media.BlockSize ) != len ) {
-		DBGC ( sandev, "EFIBLK %#02x impossible length %#zx\n",
+		DBGC ( sandev->drive, "EFIBLK %#02x impossible length %#zx\n",
 		       sandev->drive, len );
 		return -EINVAL;
 	}
@@ -109,7 +109,7 @@ static int efi_block_rw ( struct san_device *sandev, uint64_t lba,
 	/* Read from / write to block device */
 	if ( ( rc = sandev_rw ( sandev, lba, count,
 				virt_to_user ( data ) ) ) != 0 ) {
-		DBGC ( sandev, "EFIBLK %#02x I/O failed: %s\n",
+		DBGC ( sandev->drive, "EFIBLK %#02x I/O failed: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		return rc;
 	}
@@ -131,7 +131,7 @@ static EFI_STATUS EFIAPI efi_block_io_reset ( EFI_BLOCK_IO_PROTOCOL *block_io,
 	struct san_device *sandev = block->sandev;
 	int rc;
 
-	DBGC2 ( sandev, "EFIBLK %#02x reset\n", sandev->drive );
+	DBGC2 ( sandev->drive, "EFIBLK %#02x reset\n", sandev->drive );
 	efi_snp_claim();
 	rc = sandev_reset ( sandev );
 	efi_snp_release();
@@ -156,7 +156,7 @@ efi_block_io_read ( EFI_BLOCK_IO_PROTOCOL *block_io, UINT32 media __unused,
 	struct san_device *sandev = block->sandev;
 	int rc;
 
-	DBGC2 ( sandev, "EFIBLK %#02x read LBA %#08llx to %p+%#08zx\n",
+	DBGC2 ( sandev->drive, "EFIBLK %#02x read LBA %#08llx to %p+%#08zx\n",
 		sandev->drive, lba, data, ( ( size_t ) len ) );
 	efi_snp_claim();
 	rc = efi_block_rw ( sandev, lba, data, len, sandev_read );
@@ -182,8 +182,8 @@ efi_block_io_write ( EFI_BLOCK_IO_PROTOCOL *block_io, UINT32 media __unused,
 	struct san_device *sandev = block->sandev;
 	int rc;
 
-	DBGC2 ( sandev, "EFIBLK %#02x write LBA %#08llx from %p+%#08zx\n",
-		sandev->drive, lba, data, ( ( size_t ) len ) );
+	DBGC2 ( sandev->drive, "EFIBLK %#02x write LBA %#08llx from "
+		"%p+%#08zx\n", sandev->drive, lba, data, ( ( size_t ) len ) );
 	efi_snp_claim();
 	rc = efi_block_rw ( sandev, lba, data, len, sandev_write );
 	efi_snp_release();
@@ -202,7 +202,7 @@ efi_block_io_flush ( EFI_BLOCK_IO_PROTOCOL *block_io ) {
 		container_of ( block_io, struct efi_block_data, block_io );
 	struct san_device *sandev = block->sandev;
 
-	DBGC2 ( sandev, "EFIBLK %#02x flush\n", sandev->drive );
+	DBGC2 ( sandev->drive, "EFIBLK %#02x flush\n", sandev->drive );
 
 	/* Nothing to do */
 	return 0;
@@ -223,12 +223,13 @@ static void efi_block_connect ( struct san_device *sandev ) {
 	if ( ( efirc = bs->ConnectController ( block->handle, NULL,
 					       NULL, TRUE ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev, "EFIBLK %#02x could not connect drivers: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( sandev->drive, "EFIBLK %#02x could not connect "
+		       "drivers: %s\n", sandev->drive, strerror ( rc ) );
 		/* May not be an error; may already be connected */
 	}
-	DBGC2 ( sandev, "EFIBLK %#02x supports protocols:\n", sandev->drive );
-	DBGC2_EFI_PROTOCOLS ( sandev, block->handle );
+	DBGC2 ( sandev->drive, "EFIBLK %#02x supports protocols:\n",
+		sandev->drive );
+	DBGC2_EFI_PROTOCOLS ( sandev->drive, block->handle );
 }
 
 /**
@@ -251,7 +252,7 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 
 	/* Sanity check */
 	if ( ! count ) {
-		DBG ( "EFIBLK has no URIs\n" );
+		DBGC ( drive, "EFIBLK %#02x has no URIs\n", drive );
 		rc = -ENOTTY;
 		goto err_no_uris;
 	}
@@ -275,7 +276,7 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 
 	/* Register SAN device */
 	if ( ( rc = register_sandev ( sandev, drive, flags ) ) != 0 ) {
-		DBGC ( sandev, "EFIBLK %#02x could not register: %s\n",
+		DBGC ( drive, "EFIBLK %#02x could not register: %s\n",
 		       drive, strerror ( rc ) );
 		goto err_register;
 	}
@@ -289,17 +290,17 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 	/* Construct device path */
 	if ( ! sandev->active ) {
 		rc = -ENODEV;
-		DBGC ( sandev, "EFIBLK %#02x not active after registration\n",
+		DBGC ( drive, "EFIBLK %#02x not active after registration\n",
 		       drive );
 		goto err_active;
 	}
 	block->path = efi_describe ( &sandev->active->block );
 	if ( ! block->path ) {
 		rc = -ENODEV;
-		DBGC ( sandev, "EFIBLK %#02x has no device path\n", drive );
+		DBGC ( drive, "EFIBLK %#02x has no device path\n", drive );
 		goto err_describe;
 	}
-	DBGC ( sandev, "EFIBLK %#02x has device path %s\n",
+	DBGC ( drive, "EFIBLK %#02x has device path %s\n",
 	       drive, efi_devpath_text ( block->path ) );
 
 	/* Install protocols */
@@ -309,8 +310,8 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 			&efi_device_path_protocol_guid, block->path,
 			NULL ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev, "EFIBLK %#02x could not install protocols: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x could not install protocols: %s\n",
+		       drive, strerror ( rc ) );
 		goto err_install;
 	}
 
@@ -324,8 +325,8 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 			&efi_block_io_protocol_guid, &block->block_io,
 			&efi_device_path_protocol_guid, block->path,
 			NULL ) ) != 0 ) {
-		DBGC ( sandev, "EFIBLK %#02x could not uninstall protocols: "
-		       "%s\n", sandev->drive, strerror ( -EEFI ( efirc ) ) );
+		DBGC ( drive, "EFIBLK %#02x could not uninstall protocols: "
+		       "%s\n", drive, strerror ( -EEFI ( efirc ) ) );
 		leak = 1;
 	}
 	efi_nullify_block ( &block->block_io );
@@ -342,10 +343,8 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 		sandev_put ( sandev );
  err_alloc:
  err_no_uris:
-	if ( leak ) {
-		DBGC ( sandev, "EFIBLK %#02x nullified and leaked\n",
-		       sandev->drive );
-	}
+	if ( leak )
+		DBGC ( drive, "EFIBLK %#02x nullified and leaked\n", drive );
 	return rc;
 }
 
@@ -364,7 +363,7 @@ static void efi_block_unhook ( unsigned int drive ) {
 	/* Find SAN device */
 	sandev = sandev_find ( drive );
 	if ( ! sandev ) {
-		DBG ( "EFIBLK cannot find drive %#02x\n", drive );
+		DBGC ( drive, "EFIBLK %#02x is not a SAN drive\n", drive );
 		return;
 	}
 	block = sandev->priv;
@@ -376,8 +375,8 @@ static void efi_block_unhook ( unsigned int drive ) {
 			&efi_block_io_protocol_guid, &block->block_io,
 			&efi_device_path_protocol_guid, block->path,
 			NULL ) ) != 0 ) ) {
-		DBGC ( sandev, "EFIBLK %#02x could not uninstall protocols: "
-		       "%s\n", sandev->drive, strerror ( -EEFI ( efirc ) ) );
+		DBGC ( drive, "EFIBLK %#02x could not uninstall protocols: "
+		       "%s\n", drive, strerror ( -EEFI ( efirc ) ) );
 		leak = 1;
 	}
 	efi_nullify_block ( &block->block_io );
@@ -396,10 +395,8 @@ static void efi_block_unhook ( unsigned int drive ) {
 		sandev_put ( sandev );
 
 	/* Report leakage, if applicable */
-	if ( leak && ( ! efi_shutdown_in_progress ) ) {
-		DBGC ( sandev, "EFIBLK %#02x nullified and leaked\n",
-		       sandev->drive );
-	}
+	if ( leak && ( ! efi_shutdown_in_progress ) )
+		DBGC ( drive, "EFIBLK %#02x nullified and leaked\n", drive );
 }
 
 /** An installed ACPI table */
@@ -539,8 +536,8 @@ static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
 					  &path.interface, efi_image_handle,
 					  handle,
 					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
-		DBGC ( sandev, "EFIBLK %#02x found filesystem with no device "
-		       "path??", sandev->drive );
+		DBGC ( sandev->drive, "EFIBLK %#02x found filesystem with no "
+		       "device path??", sandev->drive );
 		rc = -EEFI ( efirc );
 		goto err_open_device_path;
 	}
@@ -552,7 +549,7 @@ static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
 		rc = -ENOTTY;
 		goto err_not_child;
 	}
-	DBGC ( sandev, "EFIBLK %#02x found child device %s\n",
+	DBGC ( sandev->drive, "EFIBLK %#02x found child device %s\n",
 	       sandev->drive, efi_devpath_text ( path.path ) );
 
 	/* Construct device path for boot image */
@@ -583,7 +580,7 @@ static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
 	}
 	end = ( ( ( void * ) filepath ) + filepath_len );
 	efi_path_terminate ( end );
-	DBGC ( sandev, "EFIBLK %#02x trying to load %s\n",
+	DBGC ( sandev->drive, "EFIBLK %#02x trying to load %s\n",
 	       sandev->drive, efi_devpath_text ( boot_path ) );
 
 	/* Try loading boot image from this device */
@@ -591,7 +588,7 @@ static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
 	if ( ( efirc = bs->LoadImage ( FALSE, efi_image_handle, boot_path,
 				       NULL, 0, image ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev, "EFIBLK %#02x could not load image: %s\n",
+		DBGC ( sandev->drive, "EFIBLK %#02x could not load image: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		if ( efirc == EFI_SECURITY_VIOLATION )
 			bs->UnloadImage ( *image );
@@ -629,7 +626,7 @@ static int efi_block_boot ( unsigned int drive, const char *filename ) {
 	/* Find SAN device */
 	sandev = sandev_find ( drive );
 	if ( ! sandev ) {
-		DBG ( "EFIBLK cannot find drive %#02x\n", drive );
+		DBGC ( drive, "EFIBLK %#02x is not a SAN drive\n", drive );
 		rc = -ENODEV;
 		goto err_sandev_find;
 	}
@@ -645,8 +642,8 @@ static int efi_block_boot ( unsigned int drive, const char *filename ) {
 			ByProtocol, &efi_simple_file_system_protocol_guid,
 			NULL, &count, &handles ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev, "EFIBLK %#02x cannot locate file systems: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x cannot locate file systems: %s\n",
+		       drive, strerror ( rc ) );
 		goto err_locate_file_systems;
 	}
 
@@ -660,13 +657,12 @@ static int efi_block_boot ( unsigned int drive, const char *filename ) {
 		if ( ( rc = efi_block_boot_image ( sandev, handles[i], filename,
 						   &image ) ) != 0 )
 			continue;
-		DBGC ( sandev, "EFIBLK %#02x found boot image\n",
-		       sandev->drive );
+		DBGC ( drive, "EFIBLK %#02x found boot image\n", drive );
 		efirc = bs->StartImage ( image, NULL, NULL );
 		rc = ( efirc ? -EEFI ( efirc ) : 0 );
 		bs->UnloadImage ( image );
-		DBGC ( sandev, "EFIBLK %#02x boot image returned: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x boot image returned: %s\n",
+		       drive, strerror ( rc ) );
 		break;
 	}
 

From eb720d2224c3d0b311f526c922c7388819d97778 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 3 Mar 2024 15:25:11 +0000
Subject: [PATCH 092/237] [efi] Use file system protocol to check for SAN boot
 filename existence

The "sanboot" command allows a custom boot filename to be specified
via the "--filename" option.  We currently rely on LoadImage() to
perform both the existence check and to load the image ready for
execution.  This may give a false negative result if Secure Boot is
enabled and the boot file is not correctly signed.

Carry out the existence check using EFI_SIMPLE_FILE_SYSTEM_PROTOCOL
separately from loading the image via LoadImage().

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_block.c | 304 ++++++++++++++++++++++++----------
 1 file changed, 218 insertions(+), 86 deletions(-)

diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index 86186c0ad..a4eab42bd 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -505,69 +505,224 @@ static int efi_block_describe ( void ) {
 }
 
 /**
- * Try booting from child device of EFI block device
+ * Check for existence of a file within a filesystem
  *
  * @v sandev		SAN device
- * @v handle		EFI handle
+ * @v handle		Filesystem handle
  * @v filename		Filename (or NULL to use default)
- * @v image		Image handle to fill in
  * @ret rc		Return status code
  */
-static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
-				  const char *filename, EFI_HANDLE *image ) {
+static int efi_block_filename ( struct san_device *sandev, EFI_HANDLE handle,
+				const char *filename ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	struct efi_block_data *block = sandev->priv;
+	EFI_GUID *protocol = &efi_simple_file_system_protocol_guid;
+	CHAR16 tmp[ filename ? ( strlen ( filename ) + 1 /* wNUL */ ) : 0 ];
+	union {
+		EFI_SIMPLE_FILE_SYSTEM_PROTOCOL *fs;
+		void *interface;
+	} u;
+	CHAR16 *wname;
+	EFI_FILE_PROTOCOL *root;
+	EFI_FILE_PROTOCOL *file;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Construct filename */
+	if ( filename ) {
+		efi_snprintf ( tmp, sizeof ( tmp ), "%s", filename );
+		wname = tmp;
+	} else {
+		wname = efi_block_boot_filename;
+	}
+
+	/* Open file system protocol */
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( sandev->drive, "EFIBLK %#02x could not open %s device "
+		       "path: %s\n", sandev->drive, efi_handle_name ( handle ),
+		       strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Open root volume */
+	if ( ( efirc = u.fs->OpenVolume ( u.fs, &root ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( sandev->drive, "EFIBLK %#02x could not open %s root: "
+		       "%s\n", sandev->drive, efi_handle_name ( handle ),
+		       strerror ( rc ) );
+		goto err_volume;
+	}
+
+	/* Try opening file */
+	if ( ( efirc = root->Open ( root, &file, wname,
+				    EFI_FILE_MODE_READ, 0 ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( sandev->drive, "EFIBLK %#02x could not open %s/%ls: "
+		       "%s\n", sandev->drive, efi_handle_name ( handle ),
+		       wname, strerror ( rc ) );
+		goto err_file;
+	}
+
+	/* Success */
+	rc = 0;
+
+	file->Close ( file );
+ err_file:
+	root->Close ( root );
+ err_volume:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
+	return rc;
+}
+
+/**
+ * Check EFI block device filesystem match
+ *
+ * @v sandev		SAN device
+ * @v handle		Filesystem handle
+ * @v path		Block device path
+ * @v filename		Filename (or NULL to use default)
+ * @v fspath		Filesystem device path to fill in
+ * @ret rc		Return status code
+ */
+static int efi_block_match ( struct san_device *sandev, EFI_HANDLE handle,
+			     EFI_DEVICE_PATH_PROTOCOL *path,
+			     const char *filename,
+			     EFI_DEVICE_PATH_PROTOCOL **fspath ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_GUID *protocol = &efi_device_path_protocol_guid;
 	union {
 		EFI_DEVICE_PATH_PROTOCOL *path;
 		void *interface;
-	} path;
-	EFI_DEVICE_PATH_PROTOCOL *boot_path;
-	FILEPATH_DEVICE_PATH *filepath;
-	EFI_DEVICE_PATH_PROTOCOL *end;
-	size_t prefix_len;
-	size_t filepath_len;
-	size_t boot_path_len;
+	} u;
 	EFI_STATUS efirc;
 	int rc;
 
 	/* Identify device path */
-	if ( ( efirc = bs->OpenProtocol ( handle,
-					  &efi_device_path_protocol_guid,
-					  &path.interface, efi_image_handle,
-					  handle,
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
 					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
-		DBGC ( sandev->drive, "EFIBLK %#02x found filesystem with no "
-		       "device path??", sandev->drive );
 		rc = -EEFI ( efirc );
-		goto err_open_device_path;
+		DBGC ( sandev->drive, "EFIBLK %#02x could not open %s device "
+		       "path: %s\n", sandev->drive, efi_handle_name ( handle ),
+		       strerror ( rc ) );
+		goto err_open;
 	}
+	*fspath = u.path;
 
-	/* Check if this device is a child of our block device */
-	prefix_len = efi_path_len ( block->path );
-	if ( memcmp ( path.path, block->path, prefix_len ) != 0 ) {
+	/* Check if filesystem is a child of this block device */
+	if ( memcmp ( u.path, path, efi_path_len ( path ) ) != 0 ) {
 		/* Not a child device */
 		rc = -ENOTTY;
+		DBGC2 ( sandev->drive, "EFIBLK %#02x is not parent of %s\n",
+			sandev->drive, efi_handle_name ( handle ) );
 		goto err_not_child;
 	}
-	DBGC ( sandev->drive, "EFIBLK %#02x found child device %s\n",
-	       sandev->drive, efi_devpath_text ( path.path ) );
+	DBGC ( sandev->drive, "EFIBLK %#02x contains filesystem %s\n",
+	       sandev->drive, efi_devpath_text ( u.path ) );
+
+	/* Check if filesystem contains boot filename */
+	if ( ( rc = efi_block_filename ( sandev, handle, filename ) ) != 0 )
+		goto err_filename;
+
+	/* Success */
+	rc = 0;
+
+ err_filename:
+ err_not_child:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
+	return rc;
+}
+
+/**
+ * Scan EFI block device for a matching filesystem
+ *
+ * @v sandev		SAN device
+ * @v filename		Filename (or NULL to use default)
+ * @v fspath		Filesystem device path to fill in
+ * @ret rc		Return status code
+ */
+static int efi_block_scan ( struct san_device *sandev, const char *filename,
+			    EFI_DEVICE_PATH_PROTOCOL **fspath ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	struct efi_block_data *block = sandev->priv;
+	EFI_HANDLE *handles;
+	UINTN count;
+	unsigned int i;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Connect up possible file system drivers */
+	efi_block_connect ( sandev );
+
+	/* Locate all Simple File System protocol handles */
+	if ( ( efirc = bs->LocateHandleBuffer (
+			ByProtocol, &efi_simple_file_system_protocol_guid,
+			NULL, &count, &handles ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( sandev->drive, "EFIBLK %#02x cannot locate file "
+		       "systems: %s\n", sandev->drive, strerror ( rc ) );
+		goto err_locate;
+	}
+
+	/* Scan for a matching filesystem */
+	rc = -ENOENT;
+	for ( i = 0 ; i < count ; i++ ) {
+
+		/* Check for a matching filesystem */
+		if ( ( rc = efi_block_match ( sandev, handles[i], block->path,
+					      filename, fspath ) ) != 0 )
+			continue;
+
+		break;
+	}
+
+	bs->FreePool ( handles );
+ err_locate:
+	return rc;
+}
+
+/**
+ * Boot from EFI block device filesystem boot image
+ *
+ * @v sandev		SAN device
+ * @v fspath		Filesystem device path
+ * @v filename		Filename (or NULL to use default)
+ * @ret rc		Return status code
+ */
+static int efi_block_exec ( struct san_device *sandev,
+			    EFI_DEVICE_PATH_PROTOCOL *fspath,
+			    const char *filename ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_DEVICE_PATH_PROTOCOL *path;
+	FILEPATH_DEVICE_PATH *filepath;
+	EFI_DEVICE_PATH_PROTOCOL *end;
+	EFI_HANDLE image;
+	size_t fspath_len;
+	size_t filepath_len;
+	size_t path_len;
+	EFI_STATUS efirc;
+	int rc;
 
 	/* Construct device path for boot image */
-	end = efi_path_end ( path.path );
-	prefix_len = ( ( ( void * ) end ) - ( ( void * ) path.path ) );
+	end = efi_path_end ( fspath );
+	fspath_len = ( ( ( void * ) end ) - ( ( void * ) fspath ) );
 	filepath_len = ( SIZE_OF_FILEPATH_DEVICE_PATH +
 			 ( filename ?
 			   ( ( strlen ( filename ) + 1 /* NUL */ ) *
 			     sizeof ( filepath->PathName[0] ) ) :
 			   sizeof ( efi_block_boot_filename ) ) );
-	boot_path_len = ( prefix_len + filepath_len + sizeof ( *end ) );
-	boot_path = zalloc ( boot_path_len );
-	if ( ! boot_path ) {
+	path_len = ( fspath_len + filepath_len + sizeof ( *end ) );
+	path = zalloc ( path_len );
+	if ( ! path ) {
 		rc = -ENOMEM;
-		goto err_alloc_path;
+		goto err_alloc;
 	}
-	memcpy ( boot_path, path.path, prefix_len );
-	filepath = ( ( ( void * ) boot_path ) + prefix_len );
+	memcpy ( path, fspath, fspath_len );
+	filepath = ( ( ( void * ) path ) + fspath_len );
 	filepath->Header.Type = MEDIA_DEVICE_PATH;
 	filepath->Header.SubType = MEDIA_FILEPATH_DP;
 	filepath->Header.Length[0] = ( filepath_len & 0xff );
@@ -581,28 +736,33 @@ static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
 	end = ( ( ( void * ) filepath ) + filepath_len );
 	efi_path_terminate ( end );
 	DBGC ( sandev->drive, "EFIBLK %#02x trying to load %s\n",
-	       sandev->drive, efi_devpath_text ( boot_path ) );
+	       sandev->drive, efi_devpath_text ( path ) );
 
-	/* Try loading boot image from this device */
-	*image = NULL;
-	if ( ( efirc = bs->LoadImage ( FALSE, efi_image_handle, boot_path,
-				       NULL, 0, image ) ) != 0 ) {
+	/* Load image */
+	image = NULL;
+	if ( ( efirc = bs->LoadImage ( FALSE, efi_image_handle, path, NULL, 0,
+				       &image ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev->drive, "EFIBLK %#02x could not load image: %s\n",
+		DBGC ( sandev->drive, "EFIBLK %#02x could not load: %s\n",
 		       sandev->drive, strerror ( rc ) );
-		if ( efirc == EFI_SECURITY_VIOLATION )
-			bs->UnloadImage ( *image );
-		goto err_load_image;
+		if ( efirc == EFI_SECURITY_VIOLATION ) {
+			goto err_load_security_violation;
+		} else {
+			goto err_load;
+		}
 	}
 
-	/* Success */
-	rc = 0;
+	/* Start image */
+	efirc = bs->StartImage ( image, NULL, NULL );
+	rc = ( efirc ? -EEFI ( efirc ) : 0 );
+	DBGC ( sandev->drive, "EFIBLK %#02x boot image returned: %s\n",
+	       sandev->drive, strerror ( rc ) );
 
- err_load_image:
-	free ( boot_path );
- err_alloc_path:
- err_not_child:
- err_open_device_path:
+ err_load_security_violation:
+	bs->UnloadImage ( image );
+ err_load:
+	free ( path );
+ err_alloc:
 	return rc;
 }
 
@@ -614,13 +774,8 @@ static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
  * @ret rc		Return status code
  */
 static int efi_block_boot ( unsigned int drive, const char *filename ) {
-	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_DEVICE_PATH_PROTOCOL *fspath = NULL;
 	struct san_device *sandev;
-	EFI_HANDLE *handles;
-	EFI_HANDLE image = NULL;
-	UINTN count;
-	unsigned int i;
-	EFI_STATUS efirc;
 	int rc;
 
 	/* Find SAN device */
@@ -634,40 +789,17 @@ static int efi_block_boot ( unsigned int drive, const char *filename ) {
 	/* Release SNP devices */
 	efi_snp_release();
 
-	/* Connect all possible protocols */
-	efi_block_connect ( sandev );
-
-	/* Locate all handles supporting the Simple File System protocol */
-	if ( ( efirc = bs->LocateHandleBuffer (
-			ByProtocol, &efi_simple_file_system_protocol_guid,
-			NULL, &count, &handles ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( drive, "EFIBLK %#02x cannot locate file systems: %s\n",
-		       drive, strerror ( rc ) );
-		goto err_locate_file_systems;
+	/* Scan for a matching filesystem within this block device */
+	if ( ( rc = efi_block_scan ( sandev, filename, &fspath ) ) != 0 ) {
+		goto err_scan;
 	}
 
-	/* Try booting from any available child device containing a
-	 * suitable boot image.  This is something of a wild stab in
-	 * the dark, but should end up conforming to user expectations
-	 * most of the time.
-	 */
-	rc = -ENOENT;
-	for ( i = 0 ; i < count ; i++ ) {
-		if ( ( rc = efi_block_boot_image ( sandev, handles[i], filename,
-						   &image ) ) != 0 )
-			continue;
-		DBGC ( drive, "EFIBLK %#02x found boot image\n", drive );
-		efirc = bs->StartImage ( image, NULL, NULL );
-		rc = ( efirc ? -EEFI ( efirc ) : 0 );
-		bs->UnloadImage ( image );
-		DBGC ( drive, "EFIBLK %#02x boot image returned: %s\n",
-		       drive, strerror ( rc ) );
-		break;
-	}
+	/* Attempt to boot from this filesystem */
+	if ( ( rc = efi_block_exec ( sandev, fspath, filename ) ) != 0 )
+		goto err_exec;
 
-	bs->FreePool ( handles );
- err_locate_file_systems:
+ err_exec:
+ err_scan:
 	efi_snp_claim();
  err_sandev_find:
 	return rc;

From 37edfea72be8d202d2590c89e9b8416f3c615216 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 4 Mar 2024 14:21:59 +0000
Subject: [PATCH 093/237] [efi] Generalise block device boot to support
 arbitrary EFI handles

SAN devices created by iPXE are visible to the firmware, and may be
accessed using the firmware's standard block I/O device interface
(e.g. INT 13 for BIOS, or EFI_BLOCK_IO_PROTOCOL for UEFI).  The iPXE
code to perform a SAN boot acts as a client of this standard block I/O
device interface, even when the underlying block I/O is being
performed by iPXE itself.

We rely on this separation to allow the "sanboot" command to be used
to boot from a local disk: since the code to perform a SAN boot does
not need direct access to an underlying iPXE SAN device, it may be
used to boot from any device providing the firmware's standard block
I/O device interface.

Clean up the EFI SAN boot code to require only a drive number and an
EFI_BLOCK_IO_PROTOCOL handle, in preparation for adding support for
booting from a local disk under UEFI.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_block.c | 113 ++++++++++++++++++++--------------
 1 file changed, 66 insertions(+), 47 deletions(-)

diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index a4eab42bd..79ef7455f 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -211,25 +211,24 @@ efi_block_io_flush ( EFI_BLOCK_IO_PROTOCOL *block_io ) {
 /**
  * Connect all possible drivers to EFI block device
  *
- * @v sandev		SAN device
+ * @v drive		Drive number
+ * @v handle		Block device handle
  */
-static void efi_block_connect ( struct san_device *sandev ) {
+static void efi_block_connect ( unsigned int drive, EFI_HANDLE handle ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	struct efi_block_data *block = sandev->priv;
 	EFI_STATUS efirc;
 	int rc;
 
 	/* Try to connect all possible drivers to this block device */
-	if ( ( efirc = bs->ConnectController ( block->handle, NULL,
-					       NULL, TRUE ) ) != 0 ) {
+	if ( ( efirc = bs->ConnectController ( handle, NULL, NULL,
+					       TRUE ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev->drive, "EFIBLK %#02x could not connect "
-		       "drivers: %s\n", sandev->drive, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x could not connect drivers: %s\n",
+		       drive, strerror ( rc ) );
 		/* May not be an error; may already be connected */
 	}
-	DBGC2 ( sandev->drive, "EFIBLK %#02x supports protocols:\n",
-		sandev->drive );
-	DBGC2_EFI_PROTOCOLS ( sandev->drive, block->handle );
+	DBGC2 ( drive, "EFIBLK %#02x supports protocols:\n", drive );
+	DBGC2_EFI_PROTOCOLS ( drive, handle );
 }
 
 /**
@@ -316,7 +315,7 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 	}
 
 	/* Connect all possible protocols */
-	efi_block_connect ( sandev );
+	efi_block_connect ( drive, block->handle );
 
 	return drive;
 
@@ -507,12 +506,12 @@ static int efi_block_describe ( void ) {
 /**
  * Check for existence of a file within a filesystem
  *
- * @v sandev		SAN device
+ * @v drive		Drive number
  * @v handle		Filesystem handle
  * @v filename		Filename (or NULL to use default)
  * @ret rc		Return status code
  */
-static int efi_block_filename ( struct san_device *sandev, EFI_HANDLE handle,
+static int efi_block_filename ( unsigned int drive, EFI_HANDLE handle,
 				const char *filename ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	EFI_GUID *protocol = &efi_simple_file_system_protocol_guid;
@@ -540,8 +539,8 @@ static int efi_block_filename ( struct san_device *sandev, EFI_HANDLE handle,
 					  efi_image_handle, handle,
 					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
 		rc = -EEFI ( efirc );
-		DBGC ( sandev->drive, "EFIBLK %#02x could not open %s device "
-		       "path: %s\n", sandev->drive, efi_handle_name ( handle ),
+		DBGC ( drive, "EFIBLK %#02x could not open %s device path: "
+		       "%s\n", drive, efi_handle_name ( handle ),
 		       strerror ( rc ) );
 		goto err_open;
 	}
@@ -549,9 +548,8 @@ static int efi_block_filename ( struct san_device *sandev, EFI_HANDLE handle,
 	/* Open root volume */
 	if ( ( efirc = u.fs->OpenVolume ( u.fs, &root ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev->drive, "EFIBLK %#02x could not open %s root: "
-		       "%s\n", sandev->drive, efi_handle_name ( handle ),
-		       strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x could not open %s root: %s\n",
+		       drive, efi_handle_name ( handle ), strerror ( rc ) );
 		goto err_volume;
 	}
 
@@ -559,9 +557,9 @@ static int efi_block_filename ( struct san_device *sandev, EFI_HANDLE handle,
 	if ( ( efirc = root->Open ( root, &file, wname,
 				    EFI_FILE_MODE_READ, 0 ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev->drive, "EFIBLK %#02x could not open %s/%ls: "
-		       "%s\n", sandev->drive, efi_handle_name ( handle ),
-		       wname, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x could not open %s/%ls: %s\n",
+		       drive, efi_handle_name ( handle ), wname,
+		       strerror ( rc ) );
 		goto err_file;
 	}
 
@@ -580,14 +578,14 @@ static int efi_block_filename ( struct san_device *sandev, EFI_HANDLE handle,
 /**
  * Check EFI block device filesystem match
  *
- * @v sandev		SAN device
+ * @v drive		Drive number
  * @v handle		Filesystem handle
  * @v path		Block device path
  * @v filename		Filename (or NULL to use default)
  * @v fspath		Filesystem device path to fill in
  * @ret rc		Return status code
  */
-static int efi_block_match ( struct san_device *sandev, EFI_HANDLE handle,
+static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 			     EFI_DEVICE_PATH_PROTOCOL *path,
 			     const char *filename,
 			     EFI_DEVICE_PATH_PROTOCOL **fspath ) {
@@ -605,8 +603,8 @@ static int efi_block_match ( struct san_device *sandev, EFI_HANDLE handle,
 					  efi_image_handle, handle,
 					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
 		rc = -EEFI ( efirc );
-		DBGC ( sandev->drive, "EFIBLK %#02x could not open %s device "
-		       "path: %s\n", sandev->drive, efi_handle_name ( handle ),
+		DBGC ( drive, "EFIBLK %#02x could not open %s device path: "
+		       "%s\n", drive, efi_handle_name ( handle ),
 		       strerror ( rc ) );
 		goto err_open;
 	}
@@ -616,15 +614,15 @@ static int efi_block_match ( struct san_device *sandev, EFI_HANDLE handle,
 	if ( memcmp ( u.path, path, efi_path_len ( path ) ) != 0 ) {
 		/* Not a child device */
 		rc = -ENOTTY;
-		DBGC2 ( sandev->drive, "EFIBLK %#02x is not parent of %s\n",
-			sandev->drive, efi_handle_name ( handle ) );
+		DBGC2 ( drive, "EFIBLK %#02x is not parent of %s\n",
+			drive, efi_handle_name ( handle ) );
 		goto err_not_child;
 	}
-	DBGC ( sandev->drive, "EFIBLK %#02x contains filesystem %s\n",
-	       sandev->drive, efi_devpath_text ( u.path ) );
+	DBGC ( drive, "EFIBLK %#02x contains filesystem %s\n",
+	       drive, efi_devpath_text ( u.path ) );
 
 	/* Check if filesystem contains boot filename */
-	if ( ( rc = efi_block_filename ( sandev, handle, filename ) ) != 0 )
+	if ( ( rc = efi_block_filename ( drive, handle, filename ) ) != 0 )
 		goto err_filename;
 
 	/* Success */
@@ -640,15 +638,21 @@ static int efi_block_match ( struct san_device *sandev, EFI_HANDLE handle,
 /**
  * Scan EFI block device for a matching filesystem
  *
- * @v sandev		SAN device
+ * @v drive		Drive number
+ * @v handle		Block device handle
  * @v filename		Filename (or NULL to use default)
  * @v fspath		Filesystem device path to fill in
  * @ret rc		Return status code
  */
-static int efi_block_scan ( struct san_device *sandev, const char *filename,
+static int efi_block_scan ( unsigned int drive, EFI_HANDLE handle,
+			    const char *filename,
 			    EFI_DEVICE_PATH_PROTOCOL **fspath ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	struct efi_block_data *block = sandev->priv;
+	EFI_GUID *protocol = &efi_device_path_protocol_guid;
+	union {
+		EFI_DEVICE_PATH_PROTOCOL *path;
+		void *interface;
+	} u;
 	EFI_HANDLE *handles;
 	UINTN count;
 	unsigned int i;
@@ -656,15 +660,25 @@ static int efi_block_scan ( struct san_device *sandev, const char *filename,
 	int rc;
 
 	/* Connect up possible file system drivers */
-	efi_block_connect ( sandev );
+	efi_block_connect ( drive, handle );
+
+	/* Identify device path */
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not open device path: %s\n",
+		       drive, strerror ( rc ) );
+		goto err_open;
+	}
 
 	/* Locate all Simple File System protocol handles */
 	if ( ( efirc = bs->LocateHandleBuffer (
 			ByProtocol, &efi_simple_file_system_protocol_guid,
 			NULL, &count, &handles ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev->drive, "EFIBLK %#02x cannot locate file "
-		       "systems: %s\n", sandev->drive, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x cannot locate file systems: %s\n",
+		       drive, strerror ( rc ) );
 		goto err_locate;
 	}
 
@@ -673,7 +687,7 @@ static int efi_block_scan ( struct san_device *sandev, const char *filename,
 	for ( i = 0 ; i < count ; i++ ) {
 
 		/* Check for a matching filesystem */
-		if ( ( rc = efi_block_match ( sandev, handles[i], block->path,
+		if ( ( rc = efi_block_match ( drive, handles[i], u.path,
 					      filename, fspath ) ) != 0 )
 			continue;
 
@@ -682,18 +696,20 @@ static int efi_block_scan ( struct san_device *sandev, const char *filename,
 
 	bs->FreePool ( handles );
  err_locate:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
 	return rc;
 }
 
 /**
  * Boot from EFI block device filesystem boot image
  *
- * @v sandev		SAN device
+ * @v drive		Drive number
  * @v fspath		Filesystem device path
  * @v filename		Filename (or NULL to use default)
  * @ret rc		Return status code
  */
-static int efi_block_exec ( struct san_device *sandev,
+static int efi_block_exec ( unsigned int drive,
 			    EFI_DEVICE_PATH_PROTOCOL *fspath,
 			    const char *filename ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
@@ -735,16 +751,16 @@ static int efi_block_exec ( struct san_device *sandev,
 	}
 	end = ( ( ( void * ) filepath ) + filepath_len );
 	efi_path_terminate ( end );
-	DBGC ( sandev->drive, "EFIBLK %#02x trying to load %s\n",
-	       sandev->drive, efi_devpath_text ( path ) );
+	DBGC ( drive, "EFIBLK %#02x trying to load %s\n",
+	       drive, efi_devpath_text ( path ) );
 
 	/* Load image */
 	image = NULL;
 	if ( ( efirc = bs->LoadImage ( FALSE, efi_image_handle, path, NULL, 0,
 				       &image ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev->drive, "EFIBLK %#02x could not load: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x could not load: %s\n",
+		       drive, strerror ( rc ) );
 		if ( efirc == EFI_SECURITY_VIOLATION ) {
 			goto err_load_security_violation;
 		} else {
@@ -755,8 +771,8 @@ static int efi_block_exec ( struct san_device *sandev,
 	/* Start image */
 	efirc = bs->StartImage ( image, NULL, NULL );
 	rc = ( efirc ? -EEFI ( efirc ) : 0 );
-	DBGC ( sandev->drive, "EFIBLK %#02x boot image returned: %s\n",
-	       sandev->drive, strerror ( rc ) );
+	DBGC ( drive, "EFIBLK %#02x boot image returned: %s\n",
+	       drive, strerror ( rc ) );
 
  err_load_security_violation:
 	bs->UnloadImage ( image );
@@ -776,6 +792,7 @@ static int efi_block_exec ( struct san_device *sandev,
 static int efi_block_boot ( unsigned int drive, const char *filename ) {
 	EFI_DEVICE_PATH_PROTOCOL *fspath = NULL;
 	struct san_device *sandev;
+	struct efi_block_data *block;
 	int rc;
 
 	/* Find SAN device */
@@ -785,17 +802,19 @@ static int efi_block_boot ( unsigned int drive, const char *filename ) {
 		rc = -ENODEV;
 		goto err_sandev_find;
 	}
+	block = sandev->priv;
 
 	/* Release SNP devices */
 	efi_snp_release();
 
 	/* Scan for a matching filesystem within this block device */
-	if ( ( rc = efi_block_scan ( sandev, filename, &fspath ) ) != 0 ) {
+	if ( ( rc = efi_block_scan ( drive, block->handle, filename,
+				     &fspath ) ) != 0 ) {
 		goto err_scan;
 	}
 
 	/* Attempt to boot from this filesystem */
-	if ( ( rc = efi_block_exec ( sandev, fspath, filename ) ) != 0 )
+	if ( ( rc = efi_block_exec ( drive, fspath, filename ) ) != 0 )
 		goto err_exec;
 
  err_exec:

From 8da22a59eecda30087504efa2153ceddacda08fd Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 5 Mar 2024 12:50:25 +0000
Subject: [PATCH 094/237] [block] Allow for iteration over SAN device list in
 drive number order

Maintain the SAN device list in order of drive number, and provide
sandev_next() to locate the first SAN device at or above a given drive
number.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/sanboot.c         | 25 +++++++++++++++++++++++--
 src/include/ipxe/sanboot.h |  1 +
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/core/sanboot.c b/src/core/sanboot.c
index 7e3a09e45..5b7a21657 100644
--- a/src/core/sanboot.c
+++ b/src/core/sanboot.c
@@ -107,6 +107,22 @@ struct san_device * sandev_find ( unsigned int drive ) {
 	return NULL;
 }
 
+/**
+ * Find next SAN device by drive number
+ *
+ * @v drive		Minimum drive number
+ * @ret sandev		SAN device, or NULL
+ */
+struct san_device * sandev_next ( unsigned int drive ) {
+	struct san_device *sandev;
+
+	list_for_each_entry ( sandev, &san_devices, list ) {
+		if ( sandev->drive >= drive )
+			return sandev;
+	}
+	return NULL;
+}
+
 /**
  * Free SAN device
  *
@@ -870,6 +886,7 @@ struct san_device * alloc_sandev ( struct uri **uris, unsigned int count,
  */
 int register_sandev ( struct san_device *sandev, unsigned int drive,
 		      unsigned int flags ) {
+	struct san_device *before;
 	int rc;
 
 	/* Check that drive number is not in use */
@@ -903,8 +920,12 @@ int register_sandev ( struct san_device *sandev, unsigned int drive,
 	if ( ( rc = sandev_parse_iso9660 ( sandev ) ) != 0 )
 		goto err_iso9660;
 
-	/* Add to list of SAN devices */
-	list_add_tail ( &sandev->list, &san_devices );
+	/* Add to list of SAN devices, in drive order */
+	for_each_sandev ( before ) {
+		if ( before->drive > sandev->drive )
+			break;
+	}
+	list_add_tail ( &sandev->list, &before->list );
 	DBGC ( sandev->drive, "SAN %#02x registered\n", sandev->drive );
 
 	return 0;
diff --git a/src/include/ipxe/sanboot.h b/src/include/ipxe/sanboot.h
index b163a94b8..a1b6d7f3d 100644
--- a/src/include/ipxe/sanboot.h
+++ b/src/include/ipxe/sanboot.h
@@ -234,6 +234,7 @@ static inline int sandev_needs_reopen ( struct san_device *sandev ) {
 }
 
 extern struct san_device * sandev_find ( unsigned int drive );
+extern struct san_device * sandev_next ( unsigned int drive );
 extern int sandev_reopen ( struct san_device *sandev );
 extern int sandev_reset ( struct san_device *sandev );
 extern int sandev_read ( struct san_device *sandev, uint64_t lba,

From b1c13cc43ece3008f7012cf736fc943d5bb89131 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 5 Mar 2024 12:54:44 +0000
Subject: [PATCH 095/237] [efi] Allow booting from local disks via the
 "sanboot" command

Extend the EFI SAN boot code to allow for booting from a local disk,
as is already possible with the BIOS SAN boot code.

There is unfortunately no direct UEFI equivalent of the BIOS drive
number.  The UEFI shell does provide numbered mappings fs0:, blk0:,
etc, but these numberings exist only while the UEFI shell is running
and are not necessarily stable between shell invocations or across
reboots.

A substantial amount of existing third-party documentation for iPXE
will suggest using "sanboot --drive 0x80" to boot from a local disk
(when no SAN drives are present), since this suggestion has been
present in the official documentation for the "sanboot" command for
almost thirteen years.  We therefore aim to ensure that this
instruction will also work for UEFI, i.e. that in a situation where
there are local disks but no SAN disks, then the first local disk will
be treated as being drive 0x80.

We therefore assign local disks the virtual drive numbers 0x80, 0x81,
etc, matching the numbering typically used in a BIOS environment.
Where a SAN disk is already occupying one of these drive numbers, the
local disks' virtual drive numbers will be incremented as necessary.
This provides a rough approximation of the equivalent functionality
under BIOS, where existing local disks' drive numbers are remapped to
make way for SAN disks.

We do not make any attempt to sort the list of local disks: the order
used for allocating virtual drive numbers will be whatever order is
returned by LocateHandle().  This will typically match the creation
order of the EFI handles, which will typically match the hardware
enumeration order of the devices, which will typically match user
expectations as to which local disk is first, second, etc.

We explicitly do not attempt to match the numbering used by the UEFI
shell (which initially sorts in increasing order of device path, but
does not renumber when new devices are added or removed).  We can
never guarantee matching this partly transient UEFI shell numbering,
so it is best not to set any expectation that it will be matched.
(Using local drive numbers starting at 0x80 helps to avoid setting up
this impossible expectation, since the UEFI shell uses local drive
numbers starting at zero.)

Since floppy disks are essentially non-existent in any plausible UEFI
system, overload "--drive 0" to mean "boot from any drive containing
the specified (or default) boot filename".

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/sanboot.c            |  10 --
 src/include/ipxe/sanboot.h    |  10 ++
 src/interface/efi/efi_block.c | 177 ++++++++++++++++++++++++++++++----
 3 files changed, 166 insertions(+), 31 deletions(-)

diff --git a/src/core/sanboot.c b/src/core/sanboot.c
index 5b7a21657..e49a3f92d 100644
--- a/src/core/sanboot.c
+++ b/src/core/sanboot.c
@@ -44,16 +44,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/quiesce.h>
 #include <ipxe/sanboot.h>
 
-/**
- * Default SAN drive number
- *
- * The drive number is a meaningful concept only in a BIOS
- * environment, where it represents the INT13 drive number (0x80 for
- * the first hard disk).  We retain it in other environments to allow
- * for a simple way for iPXE commands to refer to SAN drives.
- */
-#define SAN_DEFAULT_DRIVE 0x80
-
 /**
  * Timeout for block device commands (in ticks)
  *
diff --git a/src/include/ipxe/sanboot.h b/src/include/ipxe/sanboot.h
index a1b6d7f3d..b11197f93 100644
--- a/src/include/ipxe/sanboot.h
+++ b/src/include/ipxe/sanboot.h
@@ -21,6 +21,16 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/acpi.h>
 #include <config/sanboot.h>
 
+/**
+ * Default SAN drive number
+ *
+ * The drive number is an externally defined concept only in a BIOS
+ * environment, where it represents the INT13 drive number (0x80 for
+ * the first hard disk).  We retain it in other environments to allow
+ * for a simple way for iPXE commands to refer to SAN drives.
+ */
+#define SAN_DEFAULT_DRIVE 0x80
+
 /** A SAN path */
 struct san_path {
 	/** Containing SAN device */
diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index 79ef7455f..11d11156b 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -299,8 +299,8 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 		DBGC ( drive, "EFIBLK %#02x has no device path\n", drive );
 		goto err_describe;
 	}
-	DBGC ( drive, "EFIBLK %#02x has device path %s\n",
-	       drive, efi_devpath_text ( block->path ) );
+	DBGC2 ( drive, "EFIBLK %#02x has device path %s\n",
+		drive, efi_devpath_text ( block->path ) );
 
 	/* Install protocols */
 	if ( ( efirc = bs->InstallMultipleProtocolInterfaces (
@@ -313,6 +313,8 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 		       drive, strerror ( rc ) );
 		goto err_install;
 	}
+	DBGC ( drive, "EFIBLK %#02x installed as SAN drive %s\n",
+	       drive, efi_handle_name ( block->handle ) );
 
 	/* Connect all possible protocols */
 	efi_block_connect ( drive, block->handle );
@@ -782,6 +784,65 @@ static int efi_block_exec ( unsigned int drive,
 	return rc;
 }
 
+/**
+ * Check that EFI block device is eligible for a local virtual drive number
+ *
+ * @v handle		Block device handle
+ * @ret rc		Return status code
+ *
+ * We assign virtual drive numbers for local (non-SAN) EFI block
+ * devices that represent complete disks, to provide roughly
+ * equivalent functionality to BIOS drive numbers.
+ */
+static int efi_block_local ( EFI_HANDLE handle ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_GUID *protocol = &efi_block_io_protocol_guid;
+	struct san_device *sandev;
+	struct efi_block_data *block;
+	union {
+		EFI_BLOCK_IO_PROTOCOL *blockio;
+		void *interface;
+	} u;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Check if handle belongs to a SAN device */
+	for_each_sandev ( sandev ) {
+		block = sandev->priv;
+		if ( handle == block->handle ) {
+			rc = -ENOTTY;
+			goto err_sandev;
+		}
+	}
+
+	/* Open block I/O protocol */
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( handle, "EFIBLK %s could not open block I/O: %s\n",
+		       efi_handle_name ( handle ), strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Do not assign drive numbers for partitions */
+	if ( u.blockio->Media->LogicalPartition ) {
+		rc = -ENOTTY;
+		DBGC2 ( handle, "EFLBLK %s is a partition\n",
+			efi_handle_name ( handle ) );
+		goto err_partition;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_partition:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
+ err_sandev:
+	return rc;
+}
+
 /**
  * Boot from EFI block device
  *
@@ -790,37 +851,111 @@ static int efi_block_exec ( unsigned int drive,
  * @ret rc		Return status code
  */
 static int efi_block_boot ( unsigned int drive, const char *filename ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	EFI_DEVICE_PATH_PROTOCOL *fspath = NULL;
+	EFI_HANDLE *handles;
+	EFI_HANDLE handle;
+	UINTN count;
 	struct san_device *sandev;
 	struct efi_block_data *block;
+	unsigned int vdrive;
+	unsigned int index;
+	EFI_STATUS efirc;
 	int rc;
 
-	/* Find SAN device */
-	sandev = sandev_find ( drive );
-	if ( ! sandev ) {
-		DBGC ( drive, "EFIBLK %#02x is not a SAN drive\n", drive );
-		rc = -ENODEV;
-		goto err_sandev_find;
-	}
-	block = sandev->priv;
-
 	/* Release SNP devices */
 	efi_snp_release();
 
-	/* Scan for a matching filesystem within this block device */
-	if ( ( rc = efi_block_scan ( drive, block->handle, filename,
-				     &fspath ) ) != 0 ) {
-		goto err_scan;
+	/* Locate all block I/O protocol handles */
+	if ( ( efirc = bs->LocateHandleBuffer ( ByProtocol,
+						&efi_block_io_protocol_guid,
+						NULL, &count,
+						&handles ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x cannot locate block I/O: %s\n",
+		       drive, strerror ( rc ) );
+		goto err_locate_block_io;
 	}
 
-	/* Attempt to boot from this filesystem */
-	if ( ( rc = efi_block_exec ( drive, fspath, filename ) ) != 0 )
-		goto err_exec;
+	/* Try booting from the first matching block device, if any */
+	rc = -ENOENT;
+	for ( vdrive = 0, index = 0 ; ; vdrive++ ) {
 
- err_exec:
- err_scan:
+		/* Identify next drive number and block I/O handle */
+		if ( ( sandev = sandev_next ( vdrive ) ) &&
+		     ( ( sandev->drive == vdrive ) ||
+		       ( sandev->drive <= SAN_DEFAULT_DRIVE ) ||
+		       ( index >= count ) ) ) {
+
+			/* There is a SAN drive that either:
+			 *
+			 * a) has the current virtual drive number, or
+			 * b) has a drive number below SAN_DEFAULT_DRIVE, or
+			 * c) has a drive number higher than any local drive
+			 *
+			 * Use this SAN drive, since the explicit SAN
+			 * drive numbering takes precedence over the
+			 * implicit local drive numbering.
+			 */
+			block = sandev->priv;
+			handle = block->handle;
+
+			/* Use SAN drive's explicit drive number */
+			vdrive = sandev->drive;
+			DBGC ( vdrive, "EFIBLK %#02x is SAN drive %s\n",
+			       vdrive, efi_handle_name ( handle ) );
+
+		} else if ( index < count ) {
+
+			/* There is no SAN drive meeting any of the
+			 * above criteria.  Try the next block I/O
+			 * handle.
+			 */
+			handle = handles[index++];
+
+			/* Check if this handle is eligible to be
+			 * given a local virtual drive number.
+			 */
+			if ( ( rc = efi_block_local ( handle ) ) != 0 ) {
+				/* Do not consume virtual drive number */
+				vdrive--;
+				continue;
+			}
+
+			/* Use the current virtual drive number, with
+			 * a minimum of SAN_DEFAULT_DRIVE to match
+			 * typical BIOS drive numbering.
+			 */
+			if ( vdrive < SAN_DEFAULT_DRIVE )
+				vdrive = SAN_DEFAULT_DRIVE;
+			DBGC ( vdrive, "EFIBLK %#02x is local drive %s\n",
+			       vdrive, efi_handle_name ( handle ) );
+
+		} else {
+
+			/* No more SAN or local drives */
+			break;
+		}
+
+		/* Skip non-matching drives */
+		if ( drive && ( drive != vdrive ) )
+			continue;
+		DBGC ( vdrive, "EFIBLK %#02x attempting to boot\n", vdrive );
+
+		/* Scan for a matching filesystem within this drive */
+		if ( ( rc = efi_block_scan ( vdrive, handle, filename,
+					     &fspath ) ) != 0 ) {
+			continue;
+		}
+
+		/* Attempt to boot from the matched filesystem */
+		rc = efi_block_exec ( vdrive, fspath, filename );
+		break;
+	}
+
+	bs->FreePool ( handles );
+ err_locate_block_io:
 	efi_snp_claim();
- err_sandev_find:
 	return rc;
 }
 

From 636ccb4ca55c73841e634f9d5986087fb3565da4 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 5 Mar 2024 20:20:10 +0000
Subject: [PATCH 096/237] [block] Allow for additional SAN boot parameters
 alongside filename

The drive specification alone does not necessarily contain enough
information to perform a SAN boot (or local disk boot) under UEFI.  If
the next-stage bootloader is installed in the EFI system partition
under a non-standard name (e.g. "\EFI\debian\grubx64.efi") then this
explicit boot filename must also be specified.

Generalise this concept to use a "SAN boot configuration parameters"
structure (currently containing only the optional explicit boot
filename), to allow for easy expansion to provide other parameters
such as the partition UUID or volume label.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/interface/pcbios/int13.c |  5 +++--
 src/core/dummy_sanboot.c              |  4 ++--
 src/core/null_sanboot.c               |  2 +-
 src/hci/commands/sanboot_cmd.c        |  6 +++++-
 src/include/ipxe/sanboot.h            | 10 ++++++++--
 src/include/usr/autoboot.h            |  3 ++-
 src/interface/efi/efi_block.c         | 23 +++++++++++++----------
 src/usr/autoboot.c                    | 15 +++++++++++----
 8 files changed, 45 insertions(+), 23 deletions(-)

diff --git a/src/arch/x86/interface/pcbios/int13.c b/src/arch/x86/interface/pcbios/int13.c
index 2c5e86241..372d40ba3 100644
--- a/src/arch/x86/interface/pcbios/int13.c
+++ b/src/arch/x86/interface/pcbios/int13.c
@@ -1511,7 +1511,7 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
  * Attempt to boot from an INT 13 drive
  *
  * @v drive		Drive number
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @ret rc		Return status code
  *
  * This boots from the specified INT 13 drive by loading the Master
@@ -1521,7 +1521,8 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
  *
  * Note that this function can never return success, by definition.
  */
-static int int13_boot ( unsigned int drive, const char *filename __unused ) {
+static int int13_boot ( unsigned int drive,
+			struct san_boot_config *config __unused ) {
 	struct memory_map memmap;
 	struct segoff address;
 	int rc;
diff --git a/src/core/dummy_sanboot.c b/src/core/dummy_sanboot.c
index 506b20c12..e22998da5 100644
--- a/src/core/dummy_sanboot.c
+++ b/src/core/dummy_sanboot.c
@@ -95,11 +95,11 @@ static void dummy_san_unhook ( unsigned int drive ) {
  * Boot from dummy SAN device
  *
  * @v drive		Drive number
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @ret rc		Return status code
  */
 static int dummy_san_boot ( unsigned int drive __unused,
-			    const char *filename __unused ) {
+			    struct san_boot_config *config __unused ) {
 
 	return -EOPNOTSUPP;
 }
diff --git a/src/core/null_sanboot.c b/src/core/null_sanboot.c
index 7c0680f58..2340cd2a8 100644
--- a/src/core/null_sanboot.c
+++ b/src/core/null_sanboot.c
@@ -38,7 +38,7 @@ static void null_san_unhook ( unsigned int drive __unused ) {
 }
 
 static int null_san_boot ( unsigned int drive __unused,
-			   const char *filename __unused ) {
+			   struct san_boot_config *config __unused ) {
 	return -EOPNOTSUPP;
 }
 
diff --git a/src/hci/commands/sanboot_cmd.c b/src/hci/commands/sanboot_cmd.c
index 3907276a0..9c7e9ab72 100644
--- a/src/hci/commands/sanboot_cmd.c
+++ b/src/hci/commands/sanboot_cmd.c
@@ -100,6 +100,7 @@ static int sanboot_core_exec ( int argc, char **argv,
 			       struct command_descriptor *cmd,
 			       int default_flags, int no_root_path_flags ) {
 	struct sanboot_options opts;
+	struct san_boot_config config;
 	struct uri *uris[argc];
 	int count;
 	int flags;
@@ -124,6 +125,9 @@ static int sanboot_core_exec ( int argc, char **argv,
 		}
 	}
 
+	/* Construct configuration parameters */
+	config.filename = opts.filename;
+
 	/* Construct flags */
 	flags = default_flags;
 	if ( opts.no_describe )
@@ -134,7 +138,7 @@ static int sanboot_core_exec ( int argc, char **argv,
 		flags |= no_root_path_flags;
 
 	/* Boot from root path */
-	if ( ( rc = uriboot ( NULL, uris, count, opts.drive, opts.filename,
+	if ( ( rc = uriboot ( NULL, uris, count, opts.drive, &config,
 			      flags ) ) != 0 )
 		goto err_uriboot;
 
diff --git a/src/include/ipxe/sanboot.h b/src/include/ipxe/sanboot.h
index b11197f93..d981226e9 100644
--- a/src/include/ipxe/sanboot.h
+++ b/src/include/ipxe/sanboot.h
@@ -105,6 +105,12 @@ enum san_device_flags {
 	SAN_NO_DESCRIBE = 0x0001,
 };
 
+/** SAN boot configuration parameters */
+struct san_boot_config {
+	/** Boot filename (or NULL to use default) */
+	const char *filename;
+};
+
 /**
  * Calculate static inline sanboot API function name
  *
@@ -165,10 +171,10 @@ void san_unhook ( unsigned int drive );
  * Attempt to boot from a SAN device
  *
  * @v drive		Drive number
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @ret rc		Return status code
  */
-int san_boot ( unsigned int drive, const char *filename );
+int san_boot ( unsigned int drive, struct san_boot_config *config );
 
 /**
  * Describe SAN devices for SAN-booted operating system
diff --git a/src/include/usr/autoboot.h b/src/include/usr/autoboot.h
index 3719b8243..a081a70df 100644
--- a/src/include/usr/autoboot.h
+++ b/src/include/usr/autoboot.h
@@ -14,6 +14,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 struct net_device;
 struct uri;
 struct settings;
+struct san_boot_config;
 
 /** uriboot() flags */
 enum uriboot_flags {
@@ -33,7 +34,7 @@ extern void set_autoboot_ll_addr ( const void *ll_addr, size_t len,
 
 extern int uriboot ( struct uri *filename, struct uri **root_paths,
 		     unsigned int root_path_count, int drive,
-		     const char *san_filename, unsigned int flags );
+		     struct san_boot_config *san_config, unsigned int flags );
 extern struct uri *
 fetch_next_server_and_filename ( struct settings *settings );
 extern int netboot ( struct net_device *netdev );
diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index 11d11156b..e81d6bb7e 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -583,13 +583,13 @@ static int efi_block_filename ( unsigned int drive, EFI_HANDLE handle,
  * @v drive		Drive number
  * @v handle		Filesystem handle
  * @v path		Block device path
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @v fspath		Filesystem device path to fill in
  * @ret rc		Return status code
  */
 static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 			     EFI_DEVICE_PATH_PROTOCOL *path,
-			     const char *filename,
+			     struct san_boot_config *config,
 			     EFI_DEVICE_PATH_PROTOCOL **fspath ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	EFI_GUID *protocol = &efi_device_path_protocol_guid;
@@ -624,8 +624,10 @@ static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 	       drive, efi_devpath_text ( u.path ) );
 
 	/* Check if filesystem contains boot filename */
-	if ( ( rc = efi_block_filename ( drive, handle, filename ) ) != 0 )
+	if ( ( rc = efi_block_filename ( drive, handle,
+					 config->filename ) ) != 0 ) {
 		goto err_filename;
+	}
 
 	/* Success */
 	rc = 0;
@@ -642,12 +644,12 @@ static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
  *
  * @v drive		Drive number
  * @v handle		Block device handle
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @v fspath		Filesystem device path to fill in
  * @ret rc		Return status code
  */
 static int efi_block_scan ( unsigned int drive, EFI_HANDLE handle,
-			    const char *filename,
+			    struct san_boot_config *config,
 			    EFI_DEVICE_PATH_PROTOCOL **fspath ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	EFI_GUID *protocol = &efi_device_path_protocol_guid;
@@ -690,7 +692,7 @@ static int efi_block_scan ( unsigned int drive, EFI_HANDLE handle,
 
 		/* Check for a matching filesystem */
 		if ( ( rc = efi_block_match ( drive, handles[i], u.path,
-					      filename, fspath ) ) != 0 )
+					      config, fspath ) ) != 0 )
 			continue;
 
 		break;
@@ -847,10 +849,11 @@ static int efi_block_local ( EFI_HANDLE handle ) {
  * Boot from EFI block device
  *
  * @v drive		Drive number
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @ret rc		Return status code
  */
-static int efi_block_boot ( unsigned int drive, const char *filename ) {
+static int efi_block_boot ( unsigned int drive,
+			    struct san_boot_config *config ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	EFI_DEVICE_PATH_PROTOCOL *fspath = NULL;
 	EFI_HANDLE *handles;
@@ -943,13 +946,13 @@ static int efi_block_boot ( unsigned int drive, const char *filename ) {
 		DBGC ( vdrive, "EFIBLK %#02x attempting to boot\n", vdrive );
 
 		/* Scan for a matching filesystem within this drive */
-		if ( ( rc = efi_block_scan ( vdrive, handle, filename,
+		if ( ( rc = efi_block_scan ( vdrive, handle, config,
 					     &fspath ) ) != 0 ) {
 			continue;
 		}
 
 		/* Attempt to boot from the matched filesystem */
-		rc = efi_block_exec ( vdrive, fspath, filename );
+		rc = efi_block_exec ( vdrive, fspath, config->filename );
 		break;
 	}
 
diff --git a/src/usr/autoboot.c b/src/usr/autoboot.c
index d1f259621..4b64ca82b 100644
--- a/src/usr/autoboot.c
+++ b/src/usr/autoboot.c
@@ -116,7 +116,7 @@ const struct setting skip_san_boot_setting __setting ( SETTING_SANBOOT_EXTRA,
  * @v root_paths	Root path(s)
  * @v root_path_count	Number of root paths
  * @v drive		SAN drive (if applicable)
- * @v san_filename	SAN filename (or NULL to use default)
+ * @v san_config	SAN boot configuration parameters
  * @v flags		Boot action flags
  * @ret rc		Return status code
  *
@@ -128,8 +128,9 @@ const struct setting skip_san_boot_setting __setting ( SETTING_SANBOOT_EXTRA,
  */
 int uriboot ( struct uri *filename, struct uri **root_paths,
 	      unsigned int root_path_count, int drive,
-	      const char *san_filename, unsigned int flags ) {
+	      struct san_boot_config *san_config, unsigned int flags ) {
 	struct image *image;
+	const char *san_filename;
 	int rc;
 
 	/* Hook SAN device, if applicable */
@@ -182,11 +183,12 @@ int uriboot ( struct uri *filename, struct uri **root_paths,
 
 	/* Attempt SAN boot if applicable */
 	if ( ! ( flags & URIBOOT_NO_SAN_BOOT ) ) {
+		san_filename = san_config->filename;
 		if ( fetch_intz_setting ( NULL, &skip_san_boot_setting) == 0 ) {
 			printf ( "Booting%s%s from SAN device %#02x\n",
 				 ( san_filename ? " " : "" ),
 				 ( san_filename ? san_filename : "" ), drive );
-			rc = san_boot ( drive, san_filename );
+			rc = san_boot ( drive, san_config );
 			printf ( "Boot from SAN device %#02x failed: %s\n",
 				 drive, strerror ( rc ) );
 		} else {
@@ -387,6 +389,7 @@ static int have_pxe_menu ( void ) {
  * @ret rc		Return status code
  */
 int netboot ( struct net_device *netdev ) {
+	struct san_boot_config san_config;
 	struct uri *filename;
 	struct uri *root_path;
 	char *san_filename;
@@ -421,6 +424,10 @@ int netboot ( struct net_device *netdev ) {
 	/* Fetch SAN filename (if any) */
 	san_filename = fetch_san_filename ( NULL );
 
+	/* Construct SAN boot configuration parameters */
+	memset ( &san_config, 0, sizeof ( san_config ) );
+	san_config.filename = san_filename;
+
 	/* If we have both a filename and a root path, ignore an
 	 * unsupported or missing URI scheme in the root path, since
 	 * it may represent an NFS root.
@@ -442,7 +449,7 @@ int netboot ( struct net_device *netdev ) {
 
 	/* Boot using next server, filename and root path */
 	if ( ( rc = uriboot ( filename, &root_path, ( root_path ? 1 : 0 ),
-			      san_default_drive(), san_filename,
+			      san_default_drive(), &san_config,
 			      ( root_path ? 0 : URIBOOT_NO_SAN ) ) ) != 0 )
 		goto err_uriboot;
 

From c4471e3408d85c51a69991974985fe283d869eac Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 5 Mar 2024 20:00:15 +0000
Subject: [PATCH 097/237] [efi] Add efi_path_guid() utility function

EFI provides no API for determining the partition GUID (if any) for a
specified device handle.  The partition GUID appears to be exposed
only as part of the device path.

Add efi_path_guid() to extract the partition GUID (if any) from a
device path.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/efi_path.h |  2 ++
 src/include/ipxe/errfile.h      |  1 +
 src/interface/efi/efi_path.c    | 43 +++++++++++++++++++++++++++++++++
 3 files changed, 46 insertions(+)

diff --git a/src/include/ipxe/efi/efi_path.h b/src/include/ipxe/efi/efi_path.h
index e75ae42c4..20ff43f64 100644
--- a/src/include/ipxe/efi/efi_path.h
+++ b/src/include/ipxe/efi/efi_path.h
@@ -20,6 +20,7 @@ struct aoe_device;
 struct fcp_description;
 struct ib_srp_device;
 struct usb_function;
+union uuid;
 
 /**
  * Terminate device path
@@ -43,6 +44,7 @@ extern EFI_DEVICE_PATH_PROTOCOL *
 efi_path_end ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern size_t efi_path_len ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern unsigned int efi_path_vlan ( EFI_DEVICE_PATH_PROTOCOL *path );
+extern int efi_path_guid ( EFI_DEVICE_PATH_PROTOCOL *path, union uuid *uuid );
 extern EFI_DEVICE_PATH_PROTOCOL * efi_paths ( EFI_DEVICE_PATH_PROTOCOL *first,
 					      ... );
 extern EFI_DEVICE_PATH_PROTOCOL * efi_netdev_path ( struct net_device *netdev );
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 7cff594d5..1fe34113d 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -80,6 +80,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_acpimac		       ( ERRFILE_CORE | 0x00280000 )
 #define ERRFILE_efi_strings	       ( ERRFILE_CORE | 0x00290000 )
 #define ERRFILE_uuid		       ( ERRFILE_CORE | 0x002a0000 )
+#define ERRFILE_efi_path	       ( ERRFILE_CORE | 0x002b0000 )
 
 #define ERRFILE_eisa		     ( ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_isa		     ( ERRFILE_DRIVER | 0x00010000 )
diff --git a/src/interface/efi/efi_path.c b/src/interface/efi/efi_path.c
index a78f97fce..d1e22eeaa 100644
--- a/src/interface/efi/efi_path.c
+++ b/src/interface/efi/efi_path.c
@@ -22,9 +22,11 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
+#include <errno.h>
 #include <byteswap.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/vlan.h>
+#include <ipxe/uuid.h>
 #include <ipxe/tcpip.h>
 #include <ipxe/uri.h>
 #include <ipxe/iscsi.h>
@@ -132,6 +134,47 @@ unsigned int efi_path_vlan ( EFI_DEVICE_PATH_PROTOCOL *path ) {
 	return 0;
 }
 
+/**
+ * Get partition GUID from device path
+ *
+ * @v path		Device path
+ * @v guid		Partition GUID to fill in
+ * @ret rc		Return status code
+ */
+int efi_path_guid ( EFI_DEVICE_PATH_PROTOCOL *path, union uuid *guid ) {
+	EFI_DEVICE_PATH_PROTOCOL *next;
+	HARDDRIVE_DEVICE_PATH *hd;
+	int rc;
+
+	/* Search for most specific partition device path */
+	rc = -ENOENT;
+	for ( ; ( next = efi_path_next ( path ) ) ; path = next ) {
+
+		/* Skip non-harddrive device paths */
+		if ( path->Type != MEDIA_DEVICE_PATH )
+			continue;
+		if ( path->SubType != MEDIA_HARDDRIVE_DP )
+			continue;
+
+		/* Skip non-GUID signatures */
+		hd = container_of ( path, HARDDRIVE_DEVICE_PATH, Header );
+		if ( hd->SignatureType != SIGNATURE_TYPE_GUID )
+			continue;
+
+		/* Extract GUID */
+		memcpy ( guid, hd->Signature, sizeof ( *guid ) );
+		uuid_mangle ( guid );
+
+		/* Record success, but continue searching in case
+		 * there exists a more specific GUID (e.g. a partition
+		 * GUID rather than a disk GUID).
+		 */
+		rc = 0;
+	}
+
+	return rc;
+}
+
 /**
  * Concatenate EFI device paths
  *

From cea22d76e4cfdbd2d5a3a29bd541346710760457 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 6 Mar 2024 08:55:55 +0000
Subject: [PATCH 098/237] [block] Allow SAN boot device to be identified by
 UUID

Add a "--uuid" option which may be used to specify a boot device UUID,
to be matched against the GPT partition GUID.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/parseopt.c            |  7 ++++---
 src/hci/commands/sanboot_cmd.c | 10 +++++++---
 src/include/ipxe/parseopt.h    | 10 +++++++++-
 src/include/ipxe/sanboot.h     |  3 +++
 src/interface/efi/efi_block.c  | 18 ++++++++++++++++++
 5 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/src/core/parseopt.c b/src/core/parseopt.c
index 7aff14851..8410e6e90 100644
--- a/src/core/parseopt.c
+++ b/src/core/parseopt.c
@@ -129,20 +129,21 @@ int parse_timeout ( char *text, unsigned long *value ) {
  * Parse UUID
  *
  * @v text		Text
- * @ret value		UUID value
+ * @ret uuid		UUID value
  * @ret rc		Return status code
  */
-int parse_uuid ( char *text, union uuid *value ) {
+int parse_uuid ( char *text, struct uuid_option *uuid ) {
 	int rc;
 
 	/* Sanity check */
 	assert ( text != NULL );
 
 	/* Parse UUID */
-	if ( ( rc = uuid_aton ( text, value ) ) != 0 ) {
+	if ( ( rc = uuid_aton ( text, &uuid->buf ) ) != 0 ) {
 		printf ( "\"%s\": invalid UUID\n", text );
 		return rc;
 	}
+	uuid->value = &uuid->buf;
 
 	return 0;
 }
diff --git a/src/hci/commands/sanboot_cmd.c b/src/hci/commands/sanboot_cmd.c
index 9c7e9ab72..b3d9ccad3 100644
--- a/src/hci/commands/sanboot_cmd.c
+++ b/src/hci/commands/sanboot_cmd.c
@@ -49,12 +49,14 @@ struct sanboot_options {
 	int keep;
 	/** Filename */
 	char *filename;
+	/** UUID */
+	struct uuid_option uuid;
 };
 
 /** "sanboot" option list */
 static union {
-	/* "sanboot" takes all four options */
-	struct option_descriptor sanboot[4];
+	/* "sanboot" takes all options */
+	struct option_descriptor sanboot[5];
 	/* "sanhook" takes only --drive and --no-describe */
 	struct option_descriptor sanhook[2];
 	/* "sanunhook" takes only --drive */
@@ -69,10 +71,11 @@ static union {
 			      struct sanboot_options, keep, parse_flag ),
 		OPTION_DESC ( "filename", 'f', required_argument,
 			      struct sanboot_options, filename, parse_string ),
+		OPTION_DESC ( "uuid", 'u', required_argument,
+			      struct sanboot_options, uuid, parse_uuid ),
 	},
 };
 
-
 /** "sanhook" command descriptor */
 static struct command_descriptor sanhook_cmd =
 	COMMAND_DESC ( struct sanboot_options, opts.sanhook, 1, MAX_ARGUMENTS,
@@ -127,6 +130,7 @@ static int sanboot_core_exec ( int argc, char **argv,
 
 	/* Construct configuration parameters */
 	config.filename = opts.filename;
+	config.uuid = opts.uuid.value;
 
 	/* Construct flags */
 	flags = default_flags;
diff --git a/src/include/ipxe/parseopt.h b/src/include/ipxe/parseopt.h
index ebd18804f..61010f22a 100644
--- a/src/include/ipxe/parseopt.h
+++ b/src/include/ipxe/parseopt.h
@@ -126,10 +126,18 @@ struct named_setting {
 	struct setting setting;
 };
 
+/** A UUID command-line option */
+struct uuid_option {
+	/** UUID */
+	union uuid *value;
+	/** Storage buffer */
+	union uuid buf;
+};
+
 extern int parse_string ( char *text, char **value );
 extern int parse_integer ( char *text, unsigned int *value );
 extern int parse_timeout ( char *text, unsigned long *value );
-extern int parse_uuid ( char *text, union uuid *value );
+extern int parse_uuid ( char *text, struct uuid_option *uuid );
 extern int parse_netdev ( char *text, struct net_device **netdev );
 extern int
 parse_netdev_configurator ( char *text,
diff --git a/src/include/ipxe/sanboot.h b/src/include/ipxe/sanboot.h
index d981226e9..9841edda5 100644
--- a/src/include/ipxe/sanboot.h
+++ b/src/include/ipxe/sanboot.h
@@ -19,6 +19,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/process.h>
 #include <ipxe/blockdev.h>
 #include <ipxe/acpi.h>
+#include <ipxe/uuid.h>
 #include <config/sanboot.h>
 
 /**
@@ -109,6 +110,8 @@ enum san_device_flags {
 struct san_boot_config {
 	/** Boot filename (or NULL to use default) */
 	const char *filename;
+	/** UUID (or NULL to ignore UUID) */
+	union uuid *uuid;
 };
 
 /**
diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index e81d6bb7e..f5b913f15 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -597,6 +597,7 @@ static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 		EFI_DEVICE_PATH_PROTOCOL *path;
 		void *interface;
 	} u;
+	union uuid guid;
 	EFI_STATUS efirc;
 	int rc;
 
@@ -623,6 +624,21 @@ static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 	DBGC ( drive, "EFIBLK %#02x contains filesystem %s\n",
 	       drive, efi_devpath_text ( u.path ) );
 
+	/* Check if filesystem matches GUID, if applicable */
+	if ( config->uuid ) {
+		if ( ( rc = efi_path_guid ( u.path, &guid ) ) != 0 ) {
+			DBGC ( drive, "EFIBLK %#02x could not determine GUID: "
+			       "%s\n", drive, strerror ( rc ) );
+			goto err_no_guid;
+		}
+		if ( memcmp ( config->uuid, &guid, sizeof ( guid ) ) != 0 ) {
+			DBGC ( drive, "EFIBLK %#02x has wrong GUID %s\n",
+			       drive, uuid_ntoa ( &guid ) );
+			rc = -ENOENT;
+			goto err_wrong_guid;
+		}
+	}
+
 	/* Check if filesystem contains boot filename */
 	if ( ( rc = efi_block_filename ( drive, handle,
 					 config->filename ) ) != 0 ) {
@@ -633,6 +649,8 @@ static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 	rc = 0;
 
  err_filename:
+ err_wrong_guid:
+ err_no_guid:
  err_not_child:
 	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
  err_open:

From 62b6d363351fb5d8890cdcd36c949d6d8563b4e9 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 7 Mar 2024 13:31:29 +0000
Subject: [PATCH 099/237] [block] Allow SAN boot device to be identified by an
 extra filename

Add an "--extra" option that can be used to specify an extra
(non-boot) filename that must exist within the booted filesystem.

Note that only files within the FAT-formatted bootable partition will
be visible to this filter.  Files within the operating system's root
disk (e.g. "/etc/redhat-release") are not generally accessible to the
firmware and so cannot be used as the existence check filter filename.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/commands/sanboot_cmd.c |  9 +++-
 src/include/ipxe/sanboot.h     |  2 +
 src/interface/efi/efi_block.c  | 93 +++++++++++++++++++++++-----------
 3 files changed, 72 insertions(+), 32 deletions(-)

diff --git a/src/hci/commands/sanboot_cmd.c b/src/hci/commands/sanboot_cmd.c
index b3d9ccad3..add0756da 100644
--- a/src/hci/commands/sanboot_cmd.c
+++ b/src/hci/commands/sanboot_cmd.c
@@ -47,8 +47,10 @@ struct sanboot_options {
 	int no_describe;
 	/** Keep SAN device */
 	int keep;
-	/** Filename */
+	/** Boot filename */
 	char *filename;
+	/** Required extra filename */
+	char *extra;
 	/** UUID */
 	struct uuid_option uuid;
 };
@@ -56,7 +58,7 @@ struct sanboot_options {
 /** "sanboot" option list */
 static union {
 	/* "sanboot" takes all options */
-	struct option_descriptor sanboot[5];
+	struct option_descriptor sanboot[6];
 	/* "sanhook" takes only --drive and --no-describe */
 	struct option_descriptor sanhook[2];
 	/* "sanunhook" takes only --drive */
@@ -71,6 +73,8 @@ static union {
 			      struct sanboot_options, keep, parse_flag ),
 		OPTION_DESC ( "filename", 'f', required_argument,
 			      struct sanboot_options, filename, parse_string ),
+		OPTION_DESC ( "extra", 'e', required_argument,
+			      struct sanboot_options, extra, parse_string ),
 		OPTION_DESC ( "uuid", 'u', required_argument,
 			      struct sanboot_options, uuid, parse_uuid ),
 	},
@@ -130,6 +134,7 @@ static int sanboot_core_exec ( int argc, char **argv,
 
 	/* Construct configuration parameters */
 	config.filename = opts.filename;
+	config.extra = opts.extra;
 	config.uuid = opts.uuid.value;
 
 	/* Construct flags */
diff --git a/src/include/ipxe/sanboot.h b/src/include/ipxe/sanboot.h
index 9841edda5..91c848b05 100644
--- a/src/include/ipxe/sanboot.h
+++ b/src/include/ipxe/sanboot.h
@@ -110,6 +110,8 @@ enum san_device_flags {
 struct san_boot_config {
 	/** Boot filename (or NULL to use default) */
 	const char *filename;
+	/** Required extra filename (or NULL to ignore) */
+	const char *extra;
 	/** UUID (or NULL to ignore UUID) */
 	union uuid *uuid;
 };
diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index f5b913f15..269b9abd7 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -506,24 +506,65 @@ static int efi_block_describe ( void ) {
 }
 
 /**
- * Check for existence of a file within a filesystem
+ * Open root directory within a filesystem
  *
  * @v drive		Drive number
  * @v handle		Filesystem handle
- * @v filename		Filename (or NULL to use default)
+ * @v root		Root directory file to fill in
  * @ret rc		Return status code
  */
-static int efi_block_filename ( unsigned int drive, EFI_HANDLE handle,
-				const char *filename ) {
+static int efi_block_root ( unsigned int drive, EFI_HANDLE handle,
+			    EFI_FILE_PROTOCOL **root ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	EFI_GUID *protocol = &efi_simple_file_system_protocol_guid;
-	CHAR16 tmp[ filename ? ( strlen ( filename ) + 1 /* wNUL */ ) : 0 ];
 	union {
 		EFI_SIMPLE_FILE_SYSTEM_PROTOCOL *fs;
 		void *interface;
 	} u;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Open filesystem protocol */
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not open %s filesystem: %s\n",
+		       drive, efi_handle_name ( handle ), strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Open root volume */
+	if ( ( efirc = u.fs->OpenVolume ( u.fs, root ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not open %s root: %s\n",
+		       drive, efi_handle_name ( handle ), strerror ( rc ) );
+		goto err_volume;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_volume:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
+	return rc;
+}
+
+/**
+ * Check for existence of a file within a filesystem
+ *
+ * @v drive		Drive number
+ * @v handle		Filesystem handle
+ * @v root		Root directory
+ * @v filename		Filename (or NULL to use default)
+ * @ret rc		Return status code
+ */
+static int efi_block_filename ( unsigned int drive, EFI_HANDLE handle,
+				EFI_FILE_PROTOCOL *root,
+				const char *filename ) {
+	CHAR16 tmp[ filename ? ( strlen ( filename ) + 1 /* wNUL */ ) : 0 ];
 	CHAR16 *wname;
-	EFI_FILE_PROTOCOL *root;
 	EFI_FILE_PROTOCOL *file;
 	EFI_STATUS efirc;
 	int rc;
@@ -536,25 +577,6 @@ static int efi_block_filename ( unsigned int drive, EFI_HANDLE handle,
 		wname = efi_block_boot_filename;
 	}
 
-	/* Open file system protocol */
-	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
-					  efi_image_handle, handle,
-					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
-		rc = -EEFI ( efirc );
-		DBGC ( drive, "EFIBLK %#02x could not open %s device path: "
-		       "%s\n", drive, efi_handle_name ( handle ),
-		       strerror ( rc ) );
-		goto err_open;
-	}
-
-	/* Open root volume */
-	if ( ( efirc = u.fs->OpenVolume ( u.fs, &root ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( drive, "EFIBLK %#02x could not open %s root: %s\n",
-		       drive, efi_handle_name ( handle ), strerror ( rc ) );
-		goto err_volume;
-	}
-
 	/* Try opening file */
 	if ( ( efirc = root->Open ( root, &file, wname,
 				    EFI_FILE_MODE_READ, 0 ) ) != 0 ) {
@@ -570,10 +592,6 @@ static int efi_block_filename ( unsigned int drive, EFI_HANDLE handle,
 
 	file->Close ( file );
  err_file:
-	root->Close ( root );
- err_volume:
-	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
- err_open:
 	return rc;
 }
 
@@ -597,6 +615,7 @@ static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 		EFI_DEVICE_PATH_PROTOCOL *path;
 		void *interface;
 	} u;
+	EFI_FILE *root;
 	union uuid guid;
 	EFI_STATUS efirc;
 	int rc;
@@ -639,16 +658,30 @@ static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 		}
 	}
 
+	/* Open root directory */
+	if ( ( rc = efi_block_root ( drive, handle, &root ) ) != 0 )
+		goto err_root;
+
 	/* Check if filesystem contains boot filename */
-	if ( ( rc = efi_block_filename ( drive, handle,
+	if ( ( rc = efi_block_filename ( drive, handle, root,
 					 config->filename ) ) != 0 ) {
 		goto err_filename;
 	}
 
+	/* Check if filesystem contains additional filename, if applicable */
+	if ( config->extra &&
+	     ( ( rc = efi_block_filename ( drive, handle, root,
+					   config->extra ) ) != 0 ) ) {
+		goto err_extra;
+	}
+
 	/* Success */
 	rc = 0;
 
+ err_extra:
  err_filename:
+	root->Close ( root );
+ err_root:
  err_wrong_guid:
  err_no_guid:
  err_not_child:

From 24a855f1fc8d97ad3ca6be3eaabdc1c8285e86e8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 7 Mar 2024 14:09:54 +0000
Subject: [PATCH 100/237] [block] Allow SAN boot device to be identified by
 filesystem label

Add a "--label" option that can be used to specify a filesystem label,
to be matched against the FAT volume label.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/commands/sanboot_cmd.c |  7 +++-
 src/include/ipxe/sanboot.h     |  2 +
 src/interface/efi/efi_block.c  | 73 ++++++++++++++++++++++++++++++++++
 3 files changed, 81 insertions(+), 1 deletion(-)

diff --git a/src/hci/commands/sanboot_cmd.c b/src/hci/commands/sanboot_cmd.c
index add0756da..6ab9e8844 100644
--- a/src/hci/commands/sanboot_cmd.c
+++ b/src/hci/commands/sanboot_cmd.c
@@ -51,6 +51,8 @@ struct sanboot_options {
 	char *filename;
 	/** Required extra filename */
 	char *extra;
+	/** Volume label */
+	char *label;
 	/** UUID */
 	struct uuid_option uuid;
 };
@@ -58,7 +60,7 @@ struct sanboot_options {
 /** "sanboot" option list */
 static union {
 	/* "sanboot" takes all options */
-	struct option_descriptor sanboot[6];
+	struct option_descriptor sanboot[7];
 	/* "sanhook" takes only --drive and --no-describe */
 	struct option_descriptor sanhook[2];
 	/* "sanunhook" takes only --drive */
@@ -75,6 +77,8 @@ static union {
 			      struct sanboot_options, filename, parse_string ),
 		OPTION_DESC ( "extra", 'e', required_argument,
 			      struct sanboot_options, extra, parse_string ),
+		OPTION_DESC ( "label", 'l', required_argument,
+			      struct sanboot_options, label, parse_string ),
 		OPTION_DESC ( "uuid", 'u', required_argument,
 			      struct sanboot_options, uuid, parse_uuid ),
 	},
@@ -135,6 +139,7 @@ static int sanboot_core_exec ( int argc, char **argv,
 	/* Construct configuration parameters */
 	config.filename = opts.filename;
 	config.extra = opts.extra;
+	config.label = opts.label;
 	config.uuid = opts.uuid.value;
 
 	/* Construct flags */
diff --git a/src/include/ipxe/sanboot.h b/src/include/ipxe/sanboot.h
index 91c848b05..e44367cdb 100644
--- a/src/include/ipxe/sanboot.h
+++ b/src/include/ipxe/sanboot.h
@@ -112,6 +112,8 @@ struct san_boot_config {
 	const char *filename;
 	/** Required extra filename (or NULL to ignore) */
 	const char *extra;
+	/** Filesystem label (or NULL to ignore volume label) */
+	const char *label;
 	/** UUID (or NULL to ignore UUID) */
 	union uuid *uuid;
 };
diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index 269b9abd7..28a7f7234 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -32,7 +32,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
+#include <stdio.h>
 #include <string.h>
+#include <strings.h>
 #include <errno.h>
 #include <ipxe/refcnt.h>
 #include <ipxe/list.h>
@@ -51,6 +53,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/efi/Protocol/BlockIo.h>
 #include <ipxe/efi/Protocol/SimpleFileSystem.h>
 #include <ipxe/efi/Protocol/AcpiTable.h>
+#include <ipxe/efi/Guid/FileSystemInfo.h>
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_strings.h>
 #include <ipxe/efi/efi_snp.h>
@@ -595,6 +598,68 @@ static int efi_block_filename ( unsigned int drive, EFI_HANDLE handle,
 	return rc;
 }
 
+/**
+ * Check for EFI block device filesystem label
+ *
+ * @v drive		Drive number
+ * @v root		Root directory
+ * @v label		Volume label
+ * @ret rc		Return status code
+ */
+static int efi_block_label ( unsigned int drive, EFI_FILE_PROTOCOL *root,
+			     const char *label ) {
+	EFI_FILE_SYSTEM_INFO *info;
+	UINTN size;
+	char *actual;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Get length of file system information */
+	size = 0;
+	root->GetInfo ( root, &efi_file_system_info_id, &size, NULL );
+
+	/* Allocate file system information */
+	info = malloc ( size );
+	if ( ! info ) {
+		rc = -ENOMEM;
+		goto err_alloc_info;
+	}
+
+	/* Get file system information */
+	if ( ( efirc = root->GetInfo ( root, &efi_file_system_info_id, &size,
+				       info ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not get filesystem info: "
+		       "%s\n", drive, strerror ( rc ) );
+		goto err_get_info;
+	}
+
+	/* Construct volume label for comparison */
+	if ( asprintf ( &actual, "%ls", info->VolumeLabel ) < 0 ) {
+		rc = -ENOMEM;
+		goto err_alloc_label;
+	}
+
+	/* Compare volume label */
+	if ( strcasecmp ( label, actual ) != 0 ) {
+		DBGC ( drive, "EFIBLK %#02x has wrong label \"%s\"\n",
+		       drive, actual );
+		rc = -ENOENT;
+		goto err_compare;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_compare:
+	free ( actual );
+ err_alloc_label:
+ err_get_info:
+	free ( info );
+ err_alloc_info:
+	return rc;
+}
+
 /**
  * Check EFI block device filesystem match
  *
@@ -675,9 +740,17 @@ static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
 		goto err_extra;
 	}
 
+	/* Check volume label, if applicable */
+	if ( config->label &&
+	     ( ( rc = efi_block_label ( drive, root,
+					config->label ) ) != 0 ) ) {
+		goto err_label;
+	}
+
 	/* Success */
 	rc = 0;
 
+ err_label:
  err_extra:
  err_filename:
 	root->Close ( root );

From 06e229590c6e94c1dd8606a374714f7cfc50241a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 7 Mar 2024 13:34:33 +0000
Subject: [PATCH 101/237] [efi] Do not report return status from
 efi_block_local()

The return status from efi_block_local() indicates whether or not the
handle is eligible to be assigned a local virtual drive number.  There
will always be several enumerated EFI_BLOCK_IO_PROTOCOL handles that
are not eligible for a local virtual drive number (e.g. the handles
corresponding to partitions, rather than to complete disks), and this
is not an interesting error to report.

Do not report errors from efi_block_local() as the overall error
status for a SAN boot, since doing so would be likely to mask a much
more relevant error from having previously attempted to scan for a
matching filesystem within an eligible block device handle.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_block.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index 28a7f7234..2f0187a0a 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -1042,8 +1042,12 @@ static int efi_block_boot ( unsigned int drive,
 
 			/* Check if this handle is eligible to be
 			 * given a local virtual drive number.
+			 *
+			 * Do not record this as the overall error
+			 * status, since it is not an interesting
+			 * error.
 			 */
-			if ( ( rc = efi_block_local ( handle ) ) != 0 ) {
+			if ( efi_block_local ( handle ) != 0 ) {
 				/* Do not consume virtual drive number */
 				vdrive--;
 				continue;

From 226531ed36ec682728a13d2bd327a3d587a88926 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 7 Mar 2024 23:25:35 +0000
Subject: [PATCH 102/237] [ci] Update action versions to silence GitHub
 warnings

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 .github/workflows/build.yml    | 14 +++++++-------
 .github/workflows/coverity.yml |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 232375adf..c6988a11f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -12,7 +12,7 @@ jobs:
         run: |
           sudo chown $(id -un) /var/cache/apt/archives
       - name: Cache packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /var/cache/apt/archives/*.deb
           key: apt-cache-${{ github.run_id }}-${{ github.run_attempt }}
@@ -32,14 +32,14 @@ jobs:
     needs: cache
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Cache permissions
         run: |
           sudo chown $(id -un) /var/cache/apt/archives
       - name: Cache packages
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: /var/cache/apt/archives/*.deb
           key: apt-cache-${{ github.run_id }}-${{ github.run_attempt }}
@@ -68,14 +68,14 @@ jobs:
     needs: cache
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Cache permissions
         run: |
           sudo chown $(id -un) /var/cache/apt/archives
       - name: Cache packages
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: /var/cache/apt/archives/*.deb
           key: apt-cache-${{ github.run_id }}-${{ github.run_attempt }}
@@ -97,14 +97,14 @@ jobs:
     needs: cache
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Cache permissions
         run: |
           sudo chown $(id -un) /var/cache/apt/archives
       - name: Cache packages
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: /var/cache/apt/archives/*.deb
           key: apt-cache-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
index ba8681801..4c3e2e482 100644
--- a/.github/workflows/coverity.yml
+++ b/.github/workflows/coverity.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Download Coverity Scan
         run: |
           curl --form token=${{ secrets.COVERITY_SCAN_TOKEN }} \

From df2f23e333601e587f72c68cb7b7b116796f9d3c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 11 Mar 2024 12:12:31 +0000
Subject: [PATCH 103/237] [efi] Update to current EDK2 headers

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/Guid/FileInfo.h          |   1 +
 .../ipxe/efi/IndustryStandard/Acpi30.h        |  14 +
 .../ipxe/efi/IndustryStandard/Acpi40.h        |   5 +
 .../ipxe/efi/IndustryStandard/Acpi50.h        |  10 +
 .../ipxe/efi/IndustryStandard/Acpi51.h        |  10 +
 .../ipxe/efi/IndustryStandard/Acpi60.h        |  10 +
 src/include/ipxe/efi/Library/BaseLib.h        | 284 ++++++++++++++++++
 src/include/ipxe/efi/Protocol/DebugSupport.h  |  14 +
 src/include/ipxe/efi/Protocol/FormBrowser2.h  |   6 +-
 .../ipxe/efi/Protocol/HiiConfigAccess.h       |   9 +-
 src/include/ipxe/efi/Protocol/Tcp6.h          |   4 +-
 src/include/ipxe/efi/Uefi/UefiSpec.h          |  65 +++-
 12 files changed, 421 insertions(+), 11 deletions(-)

diff --git a/src/include/ipxe/efi/Guid/FileInfo.h b/src/include/ipxe/efi/Guid/FileInfo.h
index 4fc9e8602..62c5f4c0c 100644
--- a/src/include/ipxe/efi/Guid/FileInfo.h
+++ b/src/include/ipxe/efi/Guid/FileInfo.h
@@ -49,6 +49,7 @@ typedef struct {
   UINT64      Attribute;
   ///
   /// The Null-terminated name of the file.
+  /// For a root directory, the name is an empty string.
   ///
   CHAR16      FileName[1];
 } EFI_FILE_INFO;
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi30.h b/src/include/ipxe/efi/IndustryStandard/Acpi30.h
index c7dfd5c7c..ff82bf203 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi30.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi30.h
@@ -19,6 +19,20 @@ FILE_LICENCE ( BSD2_PATENT );
 
 #define ACPI_EXTENDED_ADDRESS_SPACE_DESCRIPTOR  0x8B
 
+///
+/// C-state Coordination Types
+/// See s8.4.2.2 _CSD (C-State Dependency)
+///
+#define ACPI_AML_COORD_TYPE_SW_ALL  0xFC
+#define ACPI_AML_COORD_TYPE_SW_ANY  0xFD
+#define ACPI_AML_COORD_TYPE_HW_ALL  0xFE
+
+///
+/// _PSD Revision for ACPI 3.0
+// See s8.4.4.5 _PSD (P-State Dependency)
+///
+#define EFI_ACPI_3_0_AML_PSD_REVISION  0
+
 //
 // Ensure proper structure formats
 //
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi40.h b/src/include/ipxe/efi/IndustryStandard/Acpi40.h
index f6c70d74a..97b817039 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi40.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi40.h
@@ -12,6 +12,11 @@ FILE_LICENCE ( BSD2_PATENT );
 
 #include <ipxe/efi/IndustryStandard/Acpi30.h>
 
+///
+/// _PSD Revision for ACPI 4.0
+///
+#define EFI_ACPI_4_0_AML_PSD_REVISION  0
+
 //
 // Ensure proper structure formats
 //
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi50.h b/src/include/ipxe/efi/IndustryStandard/Acpi50.h
index 7d57b9ff9..2addcb008 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi50.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi50.h
@@ -25,6 +25,16 @@ FILE_LICENCE ( BSD2_PATENT );
 #define ACPI_GPIO_CONNECTION_DESCRIPTOR                0x8C
 #define ACPI_GENERIC_SERIAL_BUS_CONNECTION_DESCRIPTOR  0x8E
 
+///
+/// _PSD Revision for ACPI 5.0
+///
+#define EFI_ACPI_5_0_AML_PSD_REVISION  0
+
+///
+/// _CPC Revision for ACPI 5.0
+///
+#define EFI_ACPI_5_0_AML_CPC_REVISION  1
+
 #pragma pack(1)
 
 ///
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi51.h b/src/include/ipxe/efi/IndustryStandard/Acpi51.h
index 49bb972ef..a2079ecc5 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi51.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi51.h
@@ -15,6 +15,16 @@ FILE_LICENCE ( BSD2_PATENT );
 
 #include <ipxe/efi/IndustryStandard/Acpi50.h>
 
+///
+/// _PSD Revision for ACPI 5.1
+///
+#define EFI_ACPI_5_1_AML_PSD_REVISION  0
+
+///
+/// _CPC Revision for ACPI 5.1
+///
+#define EFI_ACPI_5_1_AML_CPC_REVISION  2
+
 //
 // Ensure proper structure formats
 //
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi60.h b/src/include/ipxe/efi/IndustryStandard/Acpi60.h
index 9bd821c7e..c8d99214c 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi60.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi60.h
@@ -14,6 +14,16 @@ FILE_LICENCE ( BSD2_PATENT );
 
 #include <ipxe/efi/IndustryStandard/Acpi51.h>
 
+///
+/// _PSD Revision for ACPI 6.0
+///
+#define EFI_ACPI_6_0_AML_PSD_REVISION  0
+
+///
+/// _CPC Revision for ACPI 6.0
+///
+#define EFI_ACPI_6_0_AML_CPC_REVISION  2
+
 //
 // Ensure proper structure formats
 //
diff --git a/src/include/ipxe/efi/Library/BaseLib.h b/src/include/ipxe/efi/Library/BaseLib.h
index 6d2365955..16ea35cd2 100644
--- a/src/include/ipxe/efi/Library/BaseLib.h
+++ b/src/include/ipxe/efi/Library/BaseLib.h
@@ -193,6 +193,11 @@ RiscVReadTimer (
   VOID
   );
 
+VOID
+RiscVSetSupervisorTimeCompareRegister (
+  IN UINT64
+  );
+
 VOID
 RiscVEnableTimerInterrupt (
   VOID
@@ -208,6 +213,59 @@ RiscVClearPendingTimerInterrupt (
   VOID
   );
 
+/**
+  RISC-V invalidate instruction cache.
+
+**/
+VOID
+EFIAPI
+RiscVInvalidateInstCacheFenceAsm (
+  VOID
+  );
+
+/**
+  RISC-V invalidate data cache.
+
+**/
+VOID
+EFIAPI
+RiscVInvalidateDataCacheFenceAsm (
+  VOID
+  );
+
+/**
+  RISC-V flush cache block. Atomically perform a clean operation
+  followed by an invalidate operation
+
+**/
+VOID
+EFIAPI
+RiscVCpuCacheFlushCmoAsm (
+  IN UINTN
+  );
+
+/**
+Perform a write transfer to another cache or to memory if the
+data in the copy of the cache block have been modified by a store
+operation
+
+**/
+VOID
+EFIAPI
+RiscVCpuCacheCleanCmoAsm (
+  IN UINTN
+  );
+
+/**
+Deallocate the copy of the cache block
+
+**/
+VOID
+EFIAPI
+RiscVCpuCacheInvalCmoAsm (
+  IN UINTN
+  );
+
 #endif // defined (MDE_CPU_RISCV64)
 
 #if defined (MDE_CPU_LOONGARCH64)
@@ -231,6 +289,227 @@ typedef struct {
 
 #define BASE_LIBRARY_JUMP_BUFFER_ALIGNMENT  8
 
+/*
+ * Set the exception base address for LoongArch.
+ *
+ * @param  ExceptionBaseAddress   The exception base address, must be aligned greater than or qeual to 4K .
+ */
+VOID
+SetExceptionBaseAddress (
+  IN UINT64
+  );
+
+/*
+ * Set the TlbRebase address for LoongArch.
+ *
+ * @param  TlbRebaseAddress   The TlbRebase address, must be aligned greater than or qeual to 4K .
+ */
+VOID
+SetTlbRebaseAddress (
+  IN UINT64
+  );
+
+/**
+  Enables local CPU interrupts.
+
+  @param  Needs to enable local interrupt bit.
+**/
+VOID
+EnableLocalInterrupts (
+  IN UINT16
+  );
+
+/**
+  Disables local CPU interrupts.
+
+  @param  Needs to disable local interrupt bit.
+**/
+VOID
+DisableLocalInterrupts (
+  IN UINT16
+  );
+
+/**
+  Read CPUCFG register.
+
+  @param  Index  Specifies the register number of the CPUCFG to read the data.
+  @param  Data   A pointer to the variable used to store the CPUCFG register value.
+**/
+VOID
+AsmCpucfg (
+  IN  UINT32  Index,
+  OUT UINT32  *Data
+  );
+
+/**
+  Gets the timer count value.
+
+  @param[] VOID
+  @retval  timer count value.
+
+**/
+UINTN
+AsmReadStableCounter (
+  VOID
+  );
+
+/**
+  CSR read operation.
+
+  @param[in]  Select   CSR read instruction select values.
+
+  @return     The return value of csrrd instruction, return -1 means no CSR instruction
+              is found.
+**/
+UINTN
+CsrRead (
+  IN UINT16  Select
+  );
+
+/**
+  CSR write operation.
+
+  @param[in]  Select   CSR write instruction select values.
+  @param[in]  Value    The csrwr will write the value.
+
+  @return     The return value of csrwr instruction, that is, store the old value of
+              the register, return -1 means no CSR instruction is found.
+**/
+UINTN
+CsrWrite (
+  IN UINT16  Select,
+  IN UINTN   Value
+  );
+
+/**
+  CSR exchange operation.
+
+  @param[in]  Select   CSR exchange instruction select values.
+  @param[in]  Value    The csrxchg will write the value.
+  @param[in]  Mask     The csrxchg mask value.
+
+  @return     The return value of csrxchg instruction, that is, store the old value of
+              the register, return -1 means no CSR instruction is found.
+**/
+UINTN
+CsrXChg (
+  IN UINT16  Select,
+  IN UINTN   Value,
+  IN UINTN   Mask
+  );
+
+/**
+  IO CSR read byte operation.
+
+  @param[in]  Select   IO CSR read instruction select values.
+
+  @return     The return value of iocsrrd.b instruction.
+
+**/
+UINT8
+IoCsrRead8 (
+  IN UINTN  Select
+  );
+
+/**
+  IO CSR read half word operation.
+
+  @param[in]  Select   IO CSR read instruction select values.
+
+  @return     The return value of iocsrrd.h instruction.
+
+**/
+UINT16
+IoCsrRead16 (
+  IN UINTN  Select
+  );
+
+/**
+  IO CSR read word operation.
+
+  @param[in]  Select   IO CSR read instruction select values.
+
+  @return     The return value of iocsrrd.w instruction.
+
+**/
+UINT32
+IoCsrRead32 (
+  IN UINTN  Select
+  );
+
+/**
+  IO CSR read double word operation. Only for LoongArch64.
+
+  @param[in]  Select   IO CSR read instruction select values.
+
+  @return     The return value of iocsrrd.d instruction.
+
+**/
+UINT64
+IoCsrRead64 (
+  IN UINTN  Select
+  );
+
+/**
+  IO CSR write byte operation.
+
+  @param[in]  Select   IO CSR write instruction select values.
+  @param[in]  Value    The iocsrwr.b will write the value.
+
+  @return     VOID.
+
+**/
+VOID
+IoCsrWrite8 (
+  IN UINTN  Select,
+  IN UINT8  Value
+  );
+
+/**
+  IO CSR write half word operation.
+
+  @param[in]  Select   IO CSR write instruction select values.
+  @param[in]  Value    The iocsrwr.h will write the value.
+
+  @return     VOID.
+
+**/
+VOID
+IoCsrWrite16 (
+  IN UINTN   Select,
+  IN UINT16  Value
+  );
+
+/**
+  IO CSR write word operation.
+
+  @param[in]  Select   IO CSR write instruction select values.
+  @param[in]  Value    The iocsrwr.w will write the value.
+
+  @return     VOID.
+
+**/
+VOID
+IoCsrWrite32 (
+  IN UINTN   Select,
+  IN UINT32  Value
+  );
+
+/**
+  IO CSR write double word operation. Only for LoongArch64.
+
+  @param[in]  Select   IO CSR write instruction select values.
+  @param[in]  Value    The iocsrwr.d will write the value.
+
+  @return     VOID.
+
+**/
+VOID
+IoCsrWrite64 (
+  IN UINTN   Select,
+  IN UINT64  Value
+  );
+
 #endif // defined (MDE_CPU_LOONGARCH64)
 
 //
@@ -4601,6 +4880,11 @@ CalculateCrc16Ansi (
   IN  UINT16      InitialValue
   );
 
+//
+// Initial value for the CRC16-ANSI algorithm, when no prior checksum has been calculated.
+//
+#define CRC16ANSI_INIT  0xffff
+
 /**
    Calculates the CRC32c checksum of the given buffer.
 
diff --git a/src/include/ipxe/efi/Protocol/DebugSupport.h b/src/include/ipxe/efi/Protocol/DebugSupport.h
index 453ea9752..8f930e335 100644
--- a/src/include/ipxe/efi/Protocol/DebugSupport.h
+++ b/src/include/ipxe/efi/Protocol/DebugSupport.h
@@ -685,6 +685,20 @@ typedef struct {
 //
 // LoongArch processor exception types.
 //
+// The exception types is located in the CSR ESTAT
+// register offset 16 bits, width 6 bits.
+//
+// If you want to register an exception hook, you can
+// shfit the number left by 16 bits, and the exception
+// handler will know the types.
+//
+// For example:
+// mCpu->CpuRegisterInterruptHandler (
+//         mCpu,
+//         (EXCEPT_LOONGARCH_PPI << CSR_ESTAT_EXC_SHIFT),
+//         PpiExceptionHandler
+//         );
+//
 #define EXCEPT_LOONGARCH_INT   0
 #define EXCEPT_LOONGARCH_PIL   1
 #define EXCEPT_LOONGARCH_PIS   2
diff --git a/src/include/ipxe/efi/Protocol/FormBrowser2.h b/src/include/ipxe/efi/Protocol/FormBrowser2.h
index b1c0d200b..5e6f940b8 100644
--- a/src/include/ipxe/efi/Protocol/FormBrowser2.h
+++ b/src/include/ipxe/efi/Protocol/FormBrowser2.h
@@ -57,6 +57,7 @@ typedef UINTN EFI_BROWSER_ACTION_REQUEST;
 #define EFI_BROWSER_ACTION_REQUEST_FORM_APPLY         6
 #define EFI_BROWSER_ACTION_REQUEST_FORM_DISCARD       7
 #define EFI_BROWSER_ACTION_REQUEST_RECONNECT          8
+#define EFI_BROWSER_ACTION_REQUEST_QUESTION_APPLY     9
 
 /**
   Initialize the browser to display the specified configuration forms.
@@ -140,10 +141,13 @@ EFI_STATUS
   @retval EFI_SUCCESS           The results have been distributed or are
                                 awaiting distribution.
 
-  @retval EFI_OUT_OF_RESOURCES  The ResultsDataSize specified
+  @retval EFI_BUFFER_TOO_SMALL  The ResultsDataSize specified
                                 was too small to contain the
                                 results data.
 
+  @retval EFI_UNSUPPORTED       Uncommitted browser state is not available
+                                at the current stage of execution.
+
 **/
 typedef
 EFI_STATUS
diff --git a/src/include/ipxe/efi/Protocol/HiiConfigAccess.h b/src/include/ipxe/efi/Protocol/HiiConfigAccess.h
index beae08203..aaa51a31b 100644
--- a/src/include/ipxe/efi/Protocol/HiiConfigAccess.h
+++ b/src/include/ipxe/efi/Protocol/HiiConfigAccess.h
@@ -104,9 +104,16 @@ typedef UINTN EFI_BROWSER_ACTION;
                                   string.
 
   @retval EFI_INVALID_PARAMETER   Unknown name. Progress points
-                                  to the & before the name in
+                                  to the "&" before the name in
                                   question.
 
+  @retval EFI_INVALID_PARAMETER   If Results or Progress is NULL.
+
+  @retval EFI_ACCESS_DENIED       The action violated a system policy.
+
+  @retval EFI_DEVICE_ERROR        Failed to extract the current configuration
+                                  for one or more named elements.
+
 **/
 typedef
 EFI_STATUS
diff --git a/src/include/ipxe/efi/Protocol/Tcp6.h b/src/include/ipxe/efi/Protocol/Tcp6.h
index eed2f7ccc..ddceaaf9f 100644
--- a/src/include/ipxe/efi/Protocol/Tcp6.h
+++ b/src/include/ipxe/efi/Protocol/Tcp6.h
@@ -194,12 +194,12 @@ typedef struct {
   BOOLEAN    EnableNagle;
   ///
   /// Set it to TRUE to enable TCP timestamps option as defined in
-  /// RFC1323. Set to FALSE to disable it.
+  /// RFC7323. Set to FALSE to disable it.
   ///
   BOOLEAN    EnableTimeStamp;
   ///
   /// Set it to TRUE to enable TCP window scale option as defined in
-  /// RFC1323. Set it to FALSE to disable it.
+  /// RFC7323. Set it to FALSE to disable it.
   ///
   BOOLEAN    EnableWindowScaling;
   ///
diff --git a/src/include/ipxe/efi/Uefi/UefiSpec.h b/src/include/ipxe/efi/Uefi/UefiSpec.h
index e5a32d88c..cc166fc34 100644
--- a/src/include/ipxe/efi/Uefi/UefiSpec.h
+++ b/src/include/ipxe/efi/Uefi/UefiSpec.h
@@ -112,6 +112,21 @@ typedef enum {
 //
 #define EFI_MEMORY_RUNTIME  0x8000000000000000ULL
 
+//
+// If this flag is set, the memory region is
+// described with additional ISA-specific memory attributes
+// as specified in EFI_MEMORY_ISA_MASK.
+//
+#define EFI_MEMORY_ISA_VALID  0x4000000000000000ULL
+
+//
+// Defines the bits reserved for describing optional ISA-specific cacheability
+// attributes that are not covered by the standard UEFI Memory Attributes cacheability
+// bits (EFI_MEMORY_UC, EFI_MEMORY_WC, EFI_MEMORY_WT, EFI_MEMORY_WB and EFI_MEMORY_UCE).
+// See Calling Conventions for further ISA-specific enumeration of these bits.
+//
+#define EFI_MEMORY_ISA_MASK  0x0FFFF00000000000ULL
+
 //
 // Attributes bitmasks, grouped by type
 //
@@ -307,6 +322,9 @@ EFI_STATUS
                                 map that requires a mapping.
   @retval EFI_NOT_FOUND         A virtual address was supplied for an address that is not found
                                 in the memory map.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -397,11 +415,14 @@ EFI_STATUS
                                      for the new virtual address mappings being applied.
 
   @retval EFI_SUCCESS           The pointer pointed to by Address was modified.
-  @retval EFI_INVALID_PARAMETER 1) Address is NULL.
-                                2) *Address is NULL and DebugDisposition does
-                                not have the EFI_OPTIONAL_PTR bit set.
   @retval EFI_NOT_FOUND         The pointer pointed to by Address was not found to be part
                                 of the current memory map. This is normally fatal.
+  @retval EFI_INVALID_PARAMETER Address is NULL.
+  @retval EFI_INVALID_PARAMETER *Address is NULL and DebugDisposition does
+                                not have the EFI_OPTIONAL_PTR bit set.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -666,6 +687,10 @@ VOID
   @retval EFI_INVALID_PARAMETER  The DataSize is not too small and Data is NULL.
   @retval EFI_DEVICE_ERROR       The variable could not be retrieved due to a hardware error.
   @retval EFI_SECURITY_VIOLATION The variable could not be retrieved due to an authentication failure.
+  @retval EFI_UNSUPPORTED        After ExitBootServices() has been called, this return code may be returned
+                                 if no variable storage is supported. The platform should describe this
+                                 runtime service as unsupported at runtime via an EFI_RT_PROPERTIES_TABLE
+                                 configuration table.
 
 **/
 typedef
@@ -702,6 +727,10 @@ EFI_STATUS
   @retval EFI_INVALID_PARAMETER Null-terminator is not found in the first VariableNameSize bytes of
                                 the input VariableName buffer.
   @retval EFI_DEVICE_ERROR      The variable could not be retrieved due to a hardware error.
+  @retval EFI_UNSUPPORTED       After ExitBootServices() has been called, this return code may be returned
+                                if no variable storage is supported. The platform should describe this
+                                runtime service as unsupported at runtime via an EFI_RT_PROPERTIES_TABLE
+                                configuration table.
 
 **/
 typedef
@@ -744,6 +773,9 @@ EFI_STATUS
                                  but the AuthInfo does NOT pass the validation check carried out by the firmware.
 
   @retval EFI_NOT_FOUND          The variable trying to be updated or deleted was not found.
+  @retval EFI_UNSUPPORTED        This call is not supported by this platform at the time the call is made.
+                                 The platform should describe this runtime service as unsupported at runtime
+                                 via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -796,6 +828,9 @@ typedef struct {
   @retval EFI_SUCCESS           The operation completed successfully.
   @retval EFI_INVALID_PARAMETER Time is NULL.
   @retval EFI_DEVICE_ERROR      The time could not be retrieved due to hardware error.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -813,6 +848,9 @@ EFI_STATUS
   @retval EFI_SUCCESS           The operation completed successfully.
   @retval EFI_INVALID_PARAMETER A time field is out of range.
   @retval EFI_DEVICE_ERROR      The time could not be set due due to hardware error.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -833,7 +871,9 @@ EFI_STATUS
   @retval EFI_INVALID_PARAMETER Pending is NULL.
   @retval EFI_INVALID_PARAMETER Time is NULL.
   @retval EFI_DEVICE_ERROR      The wakeup time could not be retrieved due to a hardware error.
-  @retval EFI_UNSUPPORTED       A wakeup timer is not supported on this platform.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -855,7 +895,9 @@ EFI_STATUS
                                 Enable is FALSE, then the wakeup alarm was disabled.
   @retval EFI_INVALID_PARAMETER A time field is out of range.
   @retval EFI_DEVICE_ERROR      The wakeup time could not be set due to a hardware error.
-  @retval EFI_UNSUPPORTED       A wakeup timer is not supported on this platform.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -900,7 +942,7 @@ EFI_STATUS
 (EFIAPI *EFI_IMAGE_LOAD)(
   IN  BOOLEAN                      BootPolicy,
   IN  EFI_HANDLE                   ParentImageHandle,
-  IN  EFI_DEVICE_PATH_PROTOCOL     *DevicePath,
+  IN  EFI_DEVICE_PATH_PROTOCOL     *DevicePath   OPTIONAL,
   IN  VOID                         *SourceBuffer OPTIONAL,
   IN  UINTN                        SourceSize,
   OUT EFI_HANDLE                   *ImageHandle
@@ -1077,6 +1119,9 @@ EFI_STATUS
   @retval EFI_SUCCESS           The next high monotonic count was returned.
   @retval EFI_INVALID_PARAMETER HighCount is NULL.
   @retval EFI_DEVICE_ERROR      The device is not functioning properly.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -1650,7 +1695,7 @@ typedef struct {
   ///
   UINT32      Flags;
   ///
-  /// Size in bytes of the capsule.
+  /// Size in bytes of the capsule (including capsule header).
   ///
   UINT32      CapsuleImageSize;
 } EFI_CAPSULE_HEADER;
@@ -1703,6 +1748,9 @@ typedef struct {
                                 in runtime. The caller may resubmit the capsule prior to ExitBootServices().
   @retval EFI_OUT_OF_RESOURCES  When ExitBootServices() has not been previously called then this error indicates
                                 the capsule is compatible with this platform but there are insufficient resources to process.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -1734,6 +1782,9 @@ EFI_STATUS
                                 in runtime. The caller may resubmit the capsule prior to ExitBootServices().
   @retval EFI_OUT_OF_RESOURCES  When ExitBootServices() has not been previously called then this error indicates
                                 the capsule is compatible with this platform but there are insufficient resources to process.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef

From 1ab4d3079d29e9ebee0c85f1aec14a3b1df8f679 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 13 Mar 2024 15:08:10 +0000
Subject: [PATCH 104/237] [mp] Define an API for multiprocessor functions

Define an API for executing very limited functions on application
processors in a multiprocessor system, along with an x86-only
implementation.

The normal iPXE runtime environment is effectively non-existent on
application processors.  There is no ability to make firmware calls
(e.g. to write to a console), and there may be no stack space
available.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm/include/bits/mp.h     |  12 ++
 src/arch/loong64/include/bits/mp.h |  12 ++
 src/arch/x86/core/mpcall.S         | 197 +++++++++++++++++++++++++++++
 src/arch/x86/include/bits/mp.h     |  12 ++
 src/config/defaults/efi.h          |   1 +
 src/config/defaults/linux.h        |   1 +
 src/config/defaults/pcbios.h       |   1 +
 src/core/mp.c                      |  67 ++++++++++
 src/core/null_mp.c                 |  37 ++++++
 src/include/ipxe/mp.h              | 154 ++++++++++++++++++++++
 src/include/ipxe/null_mp.h         |  36 ++++++
 11 files changed, 530 insertions(+)
 create mode 100644 src/arch/arm/include/bits/mp.h
 create mode 100644 src/arch/loong64/include/bits/mp.h
 create mode 100644 src/arch/x86/core/mpcall.S
 create mode 100644 src/arch/x86/include/bits/mp.h
 create mode 100644 src/core/mp.c
 create mode 100644 src/core/null_mp.c
 create mode 100644 src/include/ipxe/mp.h
 create mode 100644 src/include/ipxe/null_mp.h

diff --git a/src/arch/arm/include/bits/mp.h b/src/arch/arm/include/bits/mp.h
new file mode 100644
index 000000000..e7d4c0c16
--- /dev/null
+++ b/src/arch/arm/include/bits/mp.h
@@ -0,0 +1,12 @@
+#ifndef _BITS_MP_H
+#define _BITS_MP_H
+
+/** @file
+ *
+ * ARM-specific multiprocessor API implementation
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_MP_H */
diff --git a/src/arch/loong64/include/bits/mp.h b/src/arch/loong64/include/bits/mp.h
new file mode 100644
index 000000000..fef2fd59a
--- /dev/null
+++ b/src/arch/loong64/include/bits/mp.h
@@ -0,0 +1,12 @@
+#ifndef _BITS_MP_H
+#define _BITS_MP_H
+
+/** @file
+ *
+ * LoongArch64-specific multiprocessor API implementation
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_MP_H */
diff --git a/src/arch/x86/core/mpcall.S b/src/arch/x86/core/mpcall.S
new file mode 100644
index 000000000..f2a3bf90f
--- /dev/null
+++ b/src/arch/x86/core/mpcall.S
@@ -0,0 +1,197 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * Multiprocessor functions
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+/* Selectively assemble code for 32-bit/64-bit builds */
+#if defined ( __x86_64__ ) && ! defined ( PLATFORM_pcbios )
+#define codemp code64
+#define DI rdi
+#define SP rsp
+#define if32 if 0
+#define if64 if 1
+#else
+#define codemp code32
+#define DI edi
+#define SP esp
+#define if32 if 1
+#define if64 if 0
+#endif
+
+/* Standard features CPUID leaf */
+#define CPUID_FEATURES 0x00000001
+
+/* x2APIC is supported */
+#define CPUID_FEATURES_ECX_X2APIC 0x00200000
+
+/* Extended topology enumeration CPUID leaf */
+#define CPUID_XT_ENUM 0x0000000b
+
+/*
+ * Call multiprocessor function from C code
+ *
+ * Parameters:
+ *	4(%esp)/%rdi	Multiprocessor function
+ *	8(%esp)/%rsi	Opaque data pointer
+ */
+	.section ".text.mp_call", "ax", @progbits
+	.codemp
+	.globl	mp_call
+mp_call:
+.if64	/* Preserve registers, load incoming parameters into registers */
+	pushq	%rax
+	pushq	%rcx
+	pushq	%rdx
+	pushq	%rbx
+	pushq	%rsp
+	pushq	%rbp
+	pushq	%rsi
+	pushq	%rdi
+	pushq	%r8
+	pushq	%r9
+	pushq	%r10
+	pushq	%r11
+	pushq	%r12
+	pushq	%r13
+	pushq	%r14
+	pushq	%r15
+.else
+	pushal
+	movl	36(%esp), %eax
+	movl	40(%esp), %edx
+.endif
+	/* Call multiprocessor function */
+	call	mp_jump
+
+.if64	/* Restore registers and return */
+	popq	%r15
+	popq	%r14
+	popq	%r13
+	popq	%r12
+	popq	%r11
+	popq	%r10
+	popq	%r9
+	popq	%r8
+	popq	%rdi
+	popq	%rsi
+	popq	%rbp
+	leaq	8(%rsp), %rsp /* discard */
+	popq	%rbx
+	popq	%rdx
+	popq	%rcx
+	popq	%rax
+.else
+	popal
+.endif
+	ret
+	.size	mp_call, . - mp_call
+
+/*
+ * Jump to multiprocessor function
+ *
+ * Parameters:
+ *	%eax/%rdi	Multiprocessor function
+ *	%edx/%rsi	Opaque data pointer
+ *	%esp/%rsp	Stack, or NULL to halt AP upon completion
+ *
+ * Obtain the CPU identifier (i.e. the APIC ID) and perform a tail
+ * call into the specified multiprocessor function.
+ *
+ * This code may run with no stack on an application processor.
+ */
+	.section ".text.mp_jump", "ax", @progbits
+	.codemp
+	.globl	mp_jump
+mp_jump:
+.if32	/* Move function parameters to available registers */
+	movl	%eax, %edi
+	movl	%edx, %esi
+.endif
+
+	/* Get 8-bit APIC ID and x2APIC feature bit */
+	movl	$CPUID_FEATURES, %eax
+	cpuid
+	shrl	$24, %ebx
+	movl	%ebx, %edx
+
+	/* Get 32-bit x2APIC ID if applicable */
+	testl	$CPUID_FEATURES_ECX_X2APIC, %ecx
+	jz	1f
+	movl	$CPUID_XT_ENUM, %eax
+	xorl	%ecx, %ecx
+	cpuid
+1:
+
+.if64	/* Tail call to function */
+	movq	%rdi, %rax
+	movq	%rsi, %rdi
+	movl	%edx, %esi
+	jmp	*%rax
+.else
+	movl	%esi, %eax
+	jmp	*%edi
+.endif
+	.size	mp_jump, . - mp_jump
+
+/*
+ * Update maximum CPU identifier
+ *
+ * Parameters:
+ *	%eax/%rdi	Pointer to shared maximum APIC ID
+ *	%edx/%rsi	CPU identifier (APIC ID)
+ *	%esp/%rsp	Stack, or NULL to halt AP upon completion
+ *
+ * This code may run with no stack on an application processor.
+ */
+	.section ".text.mp_update_max_cpuid", "ax", @progbits
+	.codemp
+	.globl	mp_update_max_cpuid
+mp_update_max_cpuid:
+.if32	/* Move function parameters to available registers */
+	movl	%eax, %edi
+	movl	%edx, %esi
+.endif
+	/* Update maximum APIC ID (atomically) */
+	movl	(%DI), %eax
+1:	cmpl	%esi, %eax
+	jae	2f
+	lock cmpxchgl %esi, (%DI)
+	jnz	1b
+2:
+	/* Return to caller (if stack exists), or halt application processor */
+	test	%SP, %SP
+	jz	3f
+	ret
+3:	cli
+	hlt
+	jmp	3b
+	.size	mp_update_max_cpuid, . - mp_update_max_cpuid
diff --git a/src/arch/x86/include/bits/mp.h b/src/arch/x86/include/bits/mp.h
new file mode 100644
index 000000000..fe466b099
--- /dev/null
+++ b/src/arch/x86/include/bits/mp.h
@@ -0,0 +1,12 @@
+#ifndef _BITS_MP_H
+#define _BITS_MP_H
+
+/** @file
+ *
+ * x86-specific multiprocessor API implementation
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_MP_H */
diff --git a/src/config/defaults/efi.h b/src/config/defaults/efi.h
index e39d475b7..9c0f238ab 100644
--- a/src/config/defaults/efi.h
+++ b/src/config/defaults/efi.h
@@ -25,6 +25,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define REBOOT_EFI
 #define ACPI_EFI
 #define FDT_EFI
+#define MPAPI_NULL
 
 #define	NET_PROTO_IPV6		/* IPv6 protocol */
 #define	NET_PROTO_LLDP		/* Link Layer Discovery protocol */
diff --git a/src/config/defaults/linux.h b/src/config/defaults/linux.h
index 21de2a2e2..fae144b3d 100644
--- a/src/config/defaults/linux.h
+++ b/src/config/defaults/linux.h
@@ -22,6 +22,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define PCIAPI_LINUX
 #define DMAAPI_FLAT
 #define ACPI_LINUX
+#define MPAPI_NULL
 
 #define DRIVERS_LINUX
 
diff --git a/src/config/defaults/pcbios.h b/src/config/defaults/pcbios.h
index ee342d41b..968bd82a6 100644
--- a/src/config/defaults/pcbios.h
+++ b/src/config/defaults/pcbios.h
@@ -24,6 +24,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define TIME_RTC
 #define REBOOT_PCBIOS
 #define ACPI_RSDP
+#define MPAPI_NULL
 
 #ifdef __x86_64__
 #define IOMAP_PAGES
diff --git a/src/core/mp.c b/src/core/mp.c
new file mode 100644
index 000000000..146d70a68
--- /dev/null
+++ b/src/core/mp.c
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Multiprocessor functions
+ *
+ */
+
+#include <ipxe/timer.h>
+#include <ipxe/mp.h>
+
+/** Time to wait for application processors */
+#define MP_MAX_CPUID_WAIT_MS 10
+
+/**
+ * Get boot CPU identifier
+ *
+ * @ret id		Boot CPU identifier
+ */
+unsigned int mp_boot_cpuid ( void ) {
+	unsigned int max = 0;
+
+	/* Update maximum to accommodate boot processor */
+	mp_exec_boot ( mp_update_max_cpuid, &max );
+	DBGC ( &mp_call, "MP boot processor ID is %#x\n", max );
+
+	return max;
+}
+
+/**
+ * Get maximum CPU identifier
+ *
+ * @ret max		Maximum CPU identifier
+ */
+unsigned int mp_max_cpuid ( void ) {
+	unsigned int max = mp_boot_cpuid();
+
+	/* Update maximum to accommodate application processors */
+	mp_start_all ( mp_update_max_cpuid, &max );
+	mdelay ( MP_MAX_CPUID_WAIT_MS );
+	DBGC ( &mp_call, "MP observed maximum CPU ID is %#x\n", max );
+
+	return max;
+}
diff --git a/src/core/null_mp.c b/src/core/null_mp.c
new file mode 100644
index 000000000..0fa69303d
--- /dev/null
+++ b/src/core/null_mp.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Null multiprocessor API
+ *
+ */
+
+#include <ipxe/mp.h>
+
+PROVIDE_MPAPI_INLINE ( null, mp_address );
+PROVIDE_MPAPI_INLINE ( null, mp_exec_boot );
+PROVIDE_MPAPI_INLINE ( null, mp_start_all );
diff --git a/src/include/ipxe/mp.h b/src/include/ipxe/mp.h
new file mode 100644
index 000000000..a36574909
--- /dev/null
+++ b/src/include/ipxe/mp.h
@@ -0,0 +1,154 @@
+#ifndef _IPXE_MP_H
+#define _IPXE_MP_H
+
+/** @file
+ *
+ * Multiprocessor functions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/api.h>
+#include <config/defaults.h>
+
+/**
+ * An address within the address space for a multiprocessor function
+ *
+ * Application processors may be started in a different address space
+ * from the normal iPXE runtime environment.  For example: under
+ * legacy BIOS the application processors will use flat 32-bit
+ * physical addressing (with no paging or virtual address offset).
+ */
+typedef unsigned long mp_addr_t;
+
+/** A multiprocessor function
+ *
+ * @v opaque		Opaque data pointer
+ * @v cpuid		CPU identifier
+ *
+ * iPXE does not set up a normal multiprocessor environment.  In
+ * particular, there is no support for dispatching code to individual
+ * processors and there is no per-CPU stack allocation.
+ *
+ * Multiprocessor code must be prepared to run wth no stack space (and
+ * with a zero stack pointer).  Functions may use the CPU identifier
+ * to construct a pointer to per-CPU result storage.
+ *
+ * Multiprocessor functions are permitted to overwrite all registers
+ * apart from the stack pointer.  On exit, the function should check
+ * the stack pointer value: if zero then the function should halt the
+ * CPU, if non-zero then the function should return in the normal way.
+ *
+ * Multiprocessor functions do not have access to any capabilities
+ * typically provided by the firmware: they cannot, for example, write
+ * any console output.
+ *
+ * All parameters are passed in registers, since there may be no stack
+ * available.  These functions cannot be called directly from C code.
+ */
+typedef void ( mp_func_t ) ( mp_addr_t opaque, unsigned int cpuid );
+
+/**
+ * Call a multiprocessor function from C code on the current CPU
+ *
+ * @v func		Multiprocessor function
+ * @v opaque		Opaque data pointer
+ *
+ * This function must be provided for each CPU architecture to bridge
+ * the normal C ABI to the iPXE multiprocessor function ABI.  It must
+ * therefore preserve any necessary registers, determine the CPU
+ * identifier, call the multiprocessor function (which may destroy any
+ * registers other than the stack pointer), restore registers, and
+ * return to the C caller.
+ *
+ * This function must be called from within the multiprocessor address
+ * space (e.g. with flat 32-bit physical addressing for BIOS).  It can
+ * be called directly from C code if the multiprocessor address space
+ * is identical to the address space used for C code (e.g. under EFI,
+ * where everything uses flat physical addresses).
+ */
+extern void __asmcall mp_call ( mp_addr_t func, mp_addr_t opaque );
+
+/**
+ * Calculate static inline multiprocessor API function name
+ *
+ * @v _prefix		Subsystem prefix
+ * @v _api_func		API function
+ * @ret _subsys_func	Subsystem API function
+ */
+#define MPAPI_INLINE( _subsys, _api_func ) \
+	SINGLE_API_INLINE ( MPAPI_PREFIX_ ## _subsys, _api_func )
+
+/**
+ * Provide a multiprocessor API implementation
+ *
+ * @v _prefix		Subsystem prefix
+ * @v _api_func		API function
+ * @v _func		Implementing function
+ */
+#define PROVIDE_MPAPI( _subsys, _api_func, _func ) \
+	PROVIDE_SINGLE_API ( MPAPI_PREFIX_ ## _subsys, _api_func, _func )
+
+/**
+ * Provide a static inline multiprocessor API implementation
+ *
+ * @v _prefix		Subsystem prefix
+ * @v _api_func		API function
+ */
+#define PROVIDE_MPAPI_INLINE( _subsys, _api_func ) \
+	PROVIDE_SINGLE_API_INLINE ( MPAPI_PREFIX_ ## _subsys, _api_func )
+
+/* Include all architecture-independent multiprocessor API headers */
+#include <ipxe/null_mp.h>
+
+/* Include all architecture-dependent multiprocessor API headers */
+#include <bits/mp.h>
+
+/**
+ * Calculate address as seen by a multiprocessor function
+ *
+ * @v address		Address in normal iPXE address space
+ * @ret address		Address in application processor address space
+ */
+mp_addr_t mp_address ( void *address );
+
+/**
+ * Execute a multiprocessor function on the boot processor
+ *
+ * @v func		Multiprocessor function
+ * @v opaque		Opaque data pointer
+ *
+ * This is a blocking operation: the call will return only when the
+ * multiprocessor function exits.
+ */
+void mp_exec_boot ( mp_func_t func, void *opaque );
+
+/**
+ * Start a multiprocessor function on all application processors
+ *
+ * @v func		Multiprocessor function
+ * @v opaque		Opaque data pointer
+ *
+ * This is a non-blocking operation: it is the caller's responsibility
+ * to provide a way to determine when the multiprocessor function has
+ * finished executing and halted its CPU.
+ */
+void mp_start_all ( mp_func_t func, void *opaque );
+
+/**
+ * Update maximum observed CPU identifier
+ *
+ * @v opaque		Opaque data pointer
+ * @v cpuid		CPU identifier
+ *
+ * This may be invoked on each processor to update a shared maximum
+ * CPU identifier value.
+ */
+extern mp_func_t mp_update_max_cpuid;
+
+extern unsigned int mp_boot_cpuid ( void );
+extern unsigned int mp_max_cpuid ( void );
+
+#endif /* _IPXE_MP_H */
diff --git a/src/include/ipxe/null_mp.h b/src/include/ipxe/null_mp.h
new file mode 100644
index 000000000..f89da5d96
--- /dev/null
+++ b/src/include/ipxe/null_mp.h
@@ -0,0 +1,36 @@
+#ifndef _IPXE_NULL_MP_H
+#define _IPXE_NULL_MP_H
+
+/** @file
+ *
+ * Null multiprocessor API implementation
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#ifdef MPAPI_NULL
+#define MPAPI_PREFIX_null
+#else
+#define MPAPI_PREFIX_null __null_
+#endif
+
+static inline __attribute__ (( always_inline )) mp_addr_t
+MPAPI_INLINE ( null, mp_address ) ( void *address ) {
+
+	return ( ( mp_addr_t ) address );
+}
+
+static inline __attribute__ (( always_inline )) void
+MPAPI_INLINE ( null, mp_exec_boot ) ( mp_func_t func __unused,
+				      void *opaque __unused ) {
+	/* Do nothing */
+}
+
+static inline __attribute__ (( always_inline )) void
+MPAPI_INLINE ( null, mp_start_all ) ( mp_func_t func __unused,
+				      void *opaque __unused ) {
+	/* Do nothing */
+}
+
+#endif /* _IPXE_NULL_MP_H */

From 89bb926a041b03c3926bf21266cbdf735d9aee66 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 13 Mar 2024 15:16:47 +0000
Subject: [PATCH 105/237] [efi] Provide a multiprocessor API for EFI

Provide an implementation of the iPXE multiprocessor API for EFI,
based on using EFI_MP_SERVICES to start up a wrapper function on all
application processors.

Note that the processor numbers used by EFI_MP_SERVICES are opaque
integers that bear no relation to the underlying CPU identity
(e.g. the APIC ID), and so we must rely on our own (architecture-
specific) implementation to determine the relevant CPU identifiers.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/defaults/efi.h                 |   2 +-
 src/include/ipxe/efi/Protocol/MpService.h | 676 ++++++++++++++++++++++
 src/include/ipxe/efi/efi_mp.h             |  30 +
 src/include/ipxe/errfile.h                |   1 +
 src/include/ipxe/mp.h                     |   1 +
 src/interface/efi/efi_mp.c                | 112 ++++
 6 files changed, 821 insertions(+), 1 deletion(-)
 create mode 100644 src/include/ipxe/efi/Protocol/MpService.h
 create mode 100644 src/include/ipxe/efi/efi_mp.h
 create mode 100644 src/interface/efi/efi_mp.c

diff --git a/src/config/defaults/efi.h b/src/config/defaults/efi.h
index 9c0f238ab..b62ddb46a 100644
--- a/src/config/defaults/efi.h
+++ b/src/config/defaults/efi.h
@@ -25,7 +25,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define REBOOT_EFI
 #define ACPI_EFI
 #define FDT_EFI
-#define MPAPI_NULL
+#define MPAPI_EFI
 
 #define	NET_PROTO_IPV6		/* IPv6 protocol */
 #define	NET_PROTO_LLDP		/* Link Layer Discovery protocol */
diff --git a/src/include/ipxe/efi/Protocol/MpService.h b/src/include/ipxe/efi/Protocol/MpService.h
new file mode 100644
index 000000000..cd1bb27fe
--- /dev/null
+++ b/src/include/ipxe/efi/Protocol/MpService.h
@@ -0,0 +1,676 @@
+/** @file
+  When installed, the MP Services Protocol produces a collection of services
+  that are needed for MP management.
+
+  The MP Services Protocol provides a generalized way of performing following tasks:
+    - Retrieving information of multi-processor environment and MP-related status of
+      specific processors.
+    - Dispatching user-provided function to APs.
+    - Maintain MP-related processor status.
+
+  The MP Services Protocol must be produced on any system with more than one logical
+  processor.
+
+  The Protocol is available only during boot time.
+
+  MP Services Protocol is hardware-independent. Most of the logic of this protocol
+  is architecturally neutral. It abstracts the multi-processor environment and
+  status of processors, and provides interfaces to retrieve information, maintain,
+  and dispatch.
+
+  MP Services Protocol may be consumed by ACPI module. The ACPI module may use this
+  protocol to retrieve data that are needed for an MP platform and report them to OS.
+  MP Services Protocol may also be used to program and configure processors, such
+  as MTRR synchronization for memory space attributes setting in DXE Services.
+  MP Services Protocol may be used by non-CPU DXE drivers to speed up platform boot
+  by taking advantage of the processing capabilities of the APs, for example, using
+  APs to help test system memory in parallel with other device initialization.
+  Diagnostics applications may also use this protocol for multi-processor.
+
+Copyright (c) 2006 - 2017, Intel Corporation. All rights reserved.<BR>
+SPDX-License-Identifier: BSD-2-Clause-Patent
+
+  @par Revision Reference:
+  This Protocol is defined in the UEFI Platform Initialization Specification 1.2,
+  Volume 2:Driver Execution Environment Core Interface.
+
+**/
+
+#ifndef _MP_SERVICE_PROTOCOL_H_
+#define _MP_SERVICE_PROTOCOL_H_
+
+FILE_LICENCE ( BSD2_PATENT );
+
+///
+/// Global ID for the EFI_MP_SERVICES_PROTOCOL.
+///
+#define EFI_MP_SERVICES_PROTOCOL_GUID \
+  { \
+    0x3fdda605, 0xa76e, 0x4f46, {0xad, 0x29, 0x12, 0xf4, 0x53, 0x1b, 0x3d, 0x08} \
+  }
+
+///
+/// Value used in the NumberProcessors parameter of the GetProcessorInfo function
+///
+#define CPU_V2_EXTENDED_TOPOLOGY  BIT24
+
+///
+/// Forward declaration for the EFI_MP_SERVICES_PROTOCOL.
+///
+typedef struct _EFI_MP_SERVICES_PROTOCOL EFI_MP_SERVICES_PROTOCOL;
+
+///
+/// Terminator for a list of failed CPUs returned by StartAllAPs().
+///
+#define END_OF_CPU_LIST  0xffffffff
+
+///
+/// This bit is used in the StatusFlag field of EFI_PROCESSOR_INFORMATION and
+/// indicates whether the processor is playing the role of BSP. If the bit is 1,
+/// then the processor is BSP. Otherwise, it is AP.
+///
+#define PROCESSOR_AS_BSP_BIT  0x00000001
+
+///
+/// This bit is used in the StatusFlag field of EFI_PROCESSOR_INFORMATION and
+/// indicates whether the processor is enabled. If the bit is 1, then the
+/// processor is enabled. Otherwise, it is disabled.
+///
+#define PROCESSOR_ENABLED_BIT  0x00000002
+
+///
+/// This bit is used in the StatusFlag field of EFI_PROCESSOR_INFORMATION and
+/// indicates whether the processor is healthy. If the bit is 1, then the
+/// processor is healthy. Otherwise, some fault has been detected for the processor.
+///
+#define PROCESSOR_HEALTH_STATUS_BIT  0x00000004
+
+///
+/// Structure that describes the pyhiscal location of a logical CPU.
+///
+typedef struct {
+  ///
+  /// Zero-based physical package number that identifies the cartridge of the processor.
+  ///
+  UINT32    Package;
+  ///
+  /// Zero-based physical core number within package of the processor.
+  ///
+  UINT32    Core;
+  ///
+  /// Zero-based logical thread number within core of the processor.
+  ///
+  UINT32    Thread;
+} EFI_CPU_PHYSICAL_LOCATION;
+
+///
+///  Structure that defines the 6-level physical location of the processor
+///
+typedef struct {
+  ///
+  ///    Package     Zero-based physical package number that identifies the cartridge of the processor.
+  ///
+  UINT32    Package;
+  ///
+  ///    Module      Zero-based physical module number within package of the processor.
+  ///
+  UINT32    Module;
+  ///
+  ///    Tile        Zero-based physical tile number within module of the processor.
+  ///
+  UINT32    Tile;
+  ///
+  ///    Die         Zero-based physical die number within tile of the processor.
+  ///
+  UINT32    Die;
+  ///
+  ///     Core        Zero-based physical core number within die of the processor.
+  ///
+  UINT32    Core;
+  ///
+  ///     Thread      Zero-based logical thread number within core of the processor.
+  ///
+  UINT32    Thread;
+} EFI_CPU_PHYSICAL_LOCATION2;
+
+typedef union {
+  /// The 6-level physical location of the processor, including the
+  /// physical package number that identifies the cartridge, the physical
+  /// module number within package, the physical tile number within the module,
+  /// the physical die number within the tile, the physical core number within
+  /// package, and logical thread number within core.
+  EFI_CPU_PHYSICAL_LOCATION2    Location2;
+} EXTENDED_PROCESSOR_INFORMATION;
+
+///
+/// Structure that describes information about a logical CPU.
+///
+typedef struct {
+  ///
+  /// The unique processor ID determined by system hardware.  For IA32 and X64,
+  /// the processor ID is the same as the Local APIC ID. Only the lower 8 bits
+  /// are used, and higher bits are reserved.  For IPF, the lower 16 bits contains
+  /// id/eid, and higher bits are reserved.
+  ///
+  UINT64    ProcessorId;
+  ///
+  /// Flags indicating if the processor is BSP or AP, if the processor is enabled
+  /// or disabled, and if the processor is healthy. Bits 3..31 are reserved and
+  /// must be 0.
+  ///
+  /// <pre>
+  /// BSP  ENABLED  HEALTH  Description
+  /// ===  =======  ======  ===================================================
+  ///  0      0       0     Unhealthy Disabled AP.
+  ///  0      0       1     Healthy Disabled AP.
+  ///  0      1       0     Unhealthy Enabled AP.
+  ///  0      1       1     Healthy Enabled AP.
+  ///  1      0       0     Invalid. The BSP can never be in the disabled state.
+  ///  1      0       1     Invalid. The BSP can never be in the disabled state.
+  ///  1      1       0     Unhealthy Enabled BSP.
+  ///  1      1       1     Healthy Enabled BSP.
+  /// </pre>
+  ///
+  UINT32                            StatusFlag;
+  ///
+  /// The physical location of the processor, including the physical package number
+  /// that identifies the cartridge, the physical core number within package, and
+  /// logical thread number within core.
+  ///
+  EFI_CPU_PHYSICAL_LOCATION         Location;
+  ///
+  /// The extended information of the processor. This field is filled only when
+  /// CPU_V2_EXTENDED_TOPOLOGY is set in parameter ProcessorNumber.
+  EXTENDED_PROCESSOR_INFORMATION    ExtendedInformation;
+} EFI_PROCESSOR_INFORMATION;
+
+/**
+  This service retrieves the number of logical processor in the platform
+  and the number of those logical processors that are enabled on this boot.
+  This service may only be called from the BSP.
+
+  This function is used to retrieve the following information:
+    - The number of logical processors that are present in the system.
+    - The number of enabled logical processors in the system at the instant
+      this call is made.
+
+  Because MP Service Protocol provides services to enable and disable processors
+  dynamically, the number of enabled logical processors may vary during the
+  course of a boot session.
+
+  If this service is called from an AP, then EFI_DEVICE_ERROR is returned.
+  If NumberOfProcessors or NumberOfEnabledProcessors is NULL, then
+  EFI_INVALID_PARAMETER is returned. Otherwise, the total number of processors
+  is returned in NumberOfProcessors, the number of currently enabled processor
+  is returned in NumberOfEnabledProcessors, and EFI_SUCCESS is returned.
+
+  @param[in]  This                        A pointer to the EFI_MP_SERVICES_PROTOCOL
+                                          instance.
+  @param[out] NumberOfProcessors          Pointer to the total number of logical
+                                          processors in the system, including the BSP
+                                          and disabled APs.
+  @param[out] NumberOfEnabledProcessors   Pointer to the number of enabled logical
+                                          processors that exist in system, including
+                                          the BSP.
+
+  @retval EFI_SUCCESS             The number of logical processors and enabled
+                                  logical processors was retrieved.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_INVALID_PARAMETER   NumberOfProcessors is NULL.
+  @retval EFI_INVALID_PARAMETER   NumberOfEnabledProcessors is NULL.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MP_SERVICES_GET_NUMBER_OF_PROCESSORS)(
+  IN  EFI_MP_SERVICES_PROTOCOL  *This,
+  OUT UINTN                     *NumberOfProcessors,
+  OUT UINTN                     *NumberOfEnabledProcessors
+  );
+
+/**
+  Gets detailed MP-related information on the requested processor at the
+  instant this call is made. This service may only be called from the BSP.
+
+  This service retrieves detailed MP-related information about any processor
+  on the platform. Note the following:
+    - The processor information may change during the course of a boot session.
+    - The information presented here is entirely MP related.
+
+  Information regarding the number of caches and their sizes, frequency of operation,
+  slot numbers is all considered platform-related information and is not provided
+  by this service.
+
+  @param[in]  This                  A pointer to the EFI_MP_SERVICES_PROTOCOL
+                                    instance.
+  @param[in]  ProcessorNumber       The handle number of processor.
+  @param[out] ProcessorInfoBuffer   A pointer to the buffer where information for
+                                    the requested processor is deposited.
+
+  @retval EFI_SUCCESS             Processor information was returned.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_INVALID_PARAMETER   ProcessorInfoBuffer is NULL.
+  @retval EFI_NOT_FOUND           The processor with the handle specified by
+                                  ProcessorNumber does not exist in the platform.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MP_SERVICES_GET_PROCESSOR_INFO)(
+  IN  EFI_MP_SERVICES_PROTOCOL   *This,
+  IN  UINTN                      ProcessorNumber,
+  OUT EFI_PROCESSOR_INFORMATION  *ProcessorInfoBuffer
+  );
+
+/**
+  This service executes a caller provided function on all enabled APs. APs can
+  run either simultaneously or one at a time in sequence. This service supports
+  both blocking and non-blocking requests. The non-blocking requests use EFI
+  events so the BSP can detect when the APs have finished. This service may only
+  be called from the BSP.
+
+  This function is used to dispatch all the enabled APs to the function specified
+  by Procedure.  If any enabled AP is busy, then EFI_NOT_READY is returned
+  immediately and Procedure is not started on any AP.
+
+  If SingleThread is TRUE, all the enabled APs execute the function specified by
+  Procedure one by one, in ascending order of processor handle number. Otherwise,
+  all the enabled APs execute the function specified by Procedure simultaneously.
+
+  If WaitEvent is NULL, execution is in blocking mode. The BSP waits until all
+  APs finish or TimeoutInMicroSecs expires. Otherwise, execution is in non-blocking
+  mode, and the BSP returns from this service without waiting for APs. If a
+  non-blocking mode is requested after the UEFI Event EFI_EVENT_GROUP_READY_TO_BOOT
+  is signaled, then EFI_UNSUPPORTED must be returned.
+
+  If the timeout specified by TimeoutInMicroseconds expires before all APs return
+  from Procedure, then Procedure on the failed APs is terminated. All enabled APs
+  are always available for further calls to EFI_MP_SERVICES_PROTOCOL.StartupAllAPs()
+  and EFI_MP_SERVICES_PROTOCOL.StartupThisAP(). If FailedCpuList is not NULL, its
+  content points to the list of processor handle numbers in which Procedure was
+  terminated.
+
+  Note: It is the responsibility of the consumer of the EFI_MP_SERVICES_PROTOCOL.StartupAllAPs()
+  to make sure that the nature of the code that is executed on the BSP and the
+  dispatched APs is well controlled. The MP Services Protocol does not guarantee
+  that the Procedure function is MP-safe. Hence, the tasks that can be run in
+  parallel are limited to certain independent tasks and well-controlled exclusive
+  code. EFI services and protocols may not be called by APs unless otherwise
+  specified.
+
+  In blocking execution mode, BSP waits until all APs finish or
+  TimeoutInMicroSeconds expires.
+
+  In non-blocking execution mode, BSP is freed to return to the caller and then
+  proceed to the next task without having to wait for APs. The following
+  sequence needs to occur in a non-blocking execution mode:
+
+    -# The caller that intends to use this MP Services Protocol in non-blocking
+       mode creates WaitEvent by calling the EFI CreateEvent() service.  The caller
+       invokes EFI_MP_SERVICES_PROTOCOL.StartupAllAPs(). If the parameter WaitEvent
+       is not NULL, then StartupAllAPs() executes in non-blocking mode. It requests
+       the function specified by Procedure to be started on all the enabled APs,
+       and releases the BSP to continue with other tasks.
+    -# The caller can use the CheckEvent() and WaitForEvent() services to check
+       the state of the WaitEvent created in step 1.
+    -# When the APs complete their task or TimeoutInMicroSecondss expires, the MP
+       Service signals WaitEvent by calling the EFI SignalEvent() function. If
+       FailedCpuList is not NULL, its content is available when WaitEvent is
+       signaled. If all APs returned from Procedure prior to the timeout, then
+       FailedCpuList is set to NULL. If not all APs return from Procedure before
+       the timeout, then FailedCpuList is filled in with the list of the failed
+       APs. The buffer is allocated by MP Service Protocol using AllocatePool().
+       It is the caller's responsibility to free the buffer with FreePool() service.
+    -# This invocation of SignalEvent() function informs the caller that invoked
+       EFI_MP_SERVICES_PROTOCOL.StartupAllAPs() that either all the APs completed
+       the specified task or a timeout occurred. The contents of FailedCpuList
+       can be examined to determine which APs did not complete the specified task
+       prior to the timeout.
+
+  @param[in]  This                    A pointer to the EFI_MP_SERVICES_PROTOCOL
+                                      instance.
+  @param[in]  Procedure               A pointer to the function to be run on
+                                      enabled APs of the system. See type
+                                      EFI_AP_PROCEDURE.
+  @param[in]  SingleThread            If TRUE, then all the enabled APs execute
+                                      the function specified by Procedure one by
+                                      one, in ascending order of processor handle
+                                      number.  If FALSE, then all the enabled APs
+                                      execute the function specified by Procedure
+                                      simultaneously.
+  @param[in]  WaitEvent               The event created by the caller with CreateEvent()
+                                      service.  If it is NULL, then execute in
+                                      blocking mode. BSP waits until all APs finish
+                                      or TimeoutInMicroSeconds expires.  If it's
+                                      not NULL, then execute in non-blocking mode.
+                                      BSP requests the function specified by
+                                      Procedure to be started on all the enabled
+                                      APs, and go on executing immediately. If
+                                      all return from Procedure, or TimeoutInMicroSeconds
+                                      expires, this event is signaled. The BSP
+                                      can use the CheckEvent() or WaitForEvent()
+                                      services to check the state of event.  Type
+                                      EFI_EVENT is defined in CreateEvent() in
+                                      the Unified Extensible Firmware Interface
+                                      Specification.
+  @param[in]  TimeoutInMicrosecsond   Indicates the time limit in microseconds for
+                                      APs to return from Procedure, either for
+                                      blocking or non-blocking mode. Zero means
+                                      infinity.  If the timeout expires before
+                                      all APs return from Procedure, then Procedure
+                                      on the failed APs is terminated. All enabled
+                                      APs are available for next function assigned
+                                      by EFI_MP_SERVICES_PROTOCOL.StartupAllAPs()
+                                      or EFI_MP_SERVICES_PROTOCOL.StartupThisAP().
+                                      If the timeout expires in blocking mode,
+                                      BSP returns EFI_TIMEOUT.  If the timeout
+                                      expires in non-blocking mode, WaitEvent
+                                      is signaled with SignalEvent().
+  @param[in]  ProcedureArgument       The parameter passed into Procedure for
+                                      all APs.
+  @param[out] FailedCpuList           If NULL, this parameter is ignored. Otherwise,
+                                      if all APs finish successfully, then its
+                                      content is set to NULL. If not all APs
+                                      finish before timeout expires, then its
+                                      content is set to address of the buffer
+                                      holding handle numbers of the failed APs.
+                                      The buffer is allocated by MP Service Protocol,
+                                      and it's the caller's responsibility to
+                                      free the buffer with FreePool() service.
+                                      In blocking mode, it is ready for consumption
+                                      when the call returns. In non-blocking mode,
+                                      it is ready when WaitEvent is signaled.  The
+                                      list of failed CPU is terminated by
+                                      END_OF_CPU_LIST.
+
+  @retval EFI_SUCCESS             In blocking mode, all APs have finished before
+                                  the timeout expired.
+  @retval EFI_SUCCESS             In non-blocking mode, function has been dispatched
+                                  to all enabled APs.
+  @retval EFI_UNSUPPORTED         A non-blocking mode request was made after the
+                                  UEFI event EFI_EVENT_GROUP_READY_TO_BOOT was
+                                  signaled.
+  @retval EFI_DEVICE_ERROR        Caller processor is AP.
+  @retval EFI_NOT_STARTED         No enabled APs exist in the system.
+  @retval EFI_NOT_READY           Any enabled APs are busy.
+  @retval EFI_TIMEOUT             In blocking mode, the timeout expired before
+                                  all enabled APs have finished.
+  @retval EFI_INVALID_PARAMETER   Procedure is NULL.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MP_SERVICES_STARTUP_ALL_APS)(
+  IN  EFI_MP_SERVICES_PROTOCOL  *This,
+  IN  EFI_AP_PROCEDURE          Procedure,
+  IN  BOOLEAN                   SingleThread,
+  IN  EFI_EVENT                 WaitEvent               OPTIONAL,
+  IN  UINTN                     TimeoutInMicroSeconds,
+  IN  VOID                      *ProcedureArgument      OPTIONAL,
+  OUT UINTN                     **FailedCpuList         OPTIONAL
+  );
+
+/**
+  This service lets the caller get one enabled AP to execute a caller-provided
+  function. The caller can request the BSP to either wait for the completion
+  of the AP or just proceed with the next task by using the EFI event mechanism.
+  See EFI_MP_SERVICES_PROTOCOL.StartupAllAPs() for more details on non-blocking
+  execution support.  This service may only be called from the BSP.
+
+  This function is used to dispatch one enabled AP to the function specified by
+  Procedure passing in the argument specified by ProcedureArgument.  If WaitEvent
+  is NULL, execution is in blocking mode. The BSP waits until the AP finishes or
+  TimeoutInMicroSecondss expires. Otherwise, execution is in non-blocking mode.
+  BSP proceeds to the next task without waiting for the AP. If a non-blocking mode
+  is requested after the UEFI Event EFI_EVENT_GROUP_READY_TO_BOOT is signaled,
+  then EFI_UNSUPPORTED must be returned.
+
+  If the timeout specified by TimeoutInMicroseconds expires before the AP returns
+  from Procedure, then execution of Procedure by the AP is terminated. The AP is
+  available for subsequent calls to EFI_MP_SERVICES_PROTOCOL.StartupAllAPs() and
+  EFI_MP_SERVICES_PROTOCOL.StartupThisAP().
+
+  @param[in]  This                    A pointer to the EFI_MP_SERVICES_PROTOCOL
+                                      instance.
+  @param[in]  Procedure               A pointer to the function to be run on the
+                                      designated AP of the system. See type
+                                      EFI_AP_PROCEDURE.
+  @param[in]  ProcessorNumber         The handle number of the AP. The range is
+                                      from 0 to the total number of logical
+                                      processors minus 1. The total number of
+                                      logical processors can be retrieved by
+                                      EFI_MP_SERVICES_PROTOCOL.GetNumberOfProcessors().
+  @param[in]  WaitEvent               The event created by the caller with CreateEvent()
+                                      service.  If it is NULL, then execute in
+                                      blocking mode. BSP waits until this AP finish
+                                      or TimeoutInMicroSeconds expires.  If it's
+                                      not NULL, then execute in non-blocking mode.
+                                      BSP requests the function specified by
+                                      Procedure to be started on this AP,
+                                      and go on executing immediately. If this AP
+                                      return from Procedure or TimeoutInMicroSeconds
+                                      expires, this event is signaled. The BSP
+                                      can use the CheckEvent() or WaitForEvent()
+                                      services to check the state of event.  Type
+                                      EFI_EVENT is defined in CreateEvent() in
+                                      the Unified Extensible Firmware Interface
+                                      Specification.
+  @param[in]  TimeoutInMicrosecsond   Indicates the time limit in microseconds for
+                                      this AP to finish this Procedure, either for
+                                      blocking or non-blocking mode. Zero means
+                                      infinity.  If the timeout expires before
+                                      this AP returns from Procedure, then Procedure
+                                      on the AP is terminated. The
+                                      AP is available for next function assigned
+                                      by EFI_MP_SERVICES_PROTOCOL.StartupAllAPs()
+                                      or EFI_MP_SERVICES_PROTOCOL.StartupThisAP().
+                                      If the timeout expires in blocking mode,
+                                      BSP returns EFI_TIMEOUT.  If the timeout
+                                      expires in non-blocking mode, WaitEvent
+                                      is signaled with SignalEvent().
+  @param[in]  ProcedureArgument       The parameter passed into Procedure on the
+                                      specified AP.
+  @param[out] Finished                If NULL, this parameter is ignored.  In
+                                      blocking mode, this parameter is ignored.
+                                      In non-blocking mode, if AP returns from
+                                      Procedure before the timeout expires, its
+                                      content is set to TRUE. Otherwise, the
+                                      value is set to FALSE. The caller can
+                                      determine if the AP returned from Procedure
+                                      by evaluating this value.
+
+  @retval EFI_SUCCESS             In blocking mode, specified AP finished before
+                                  the timeout expires.
+  @retval EFI_SUCCESS             In non-blocking mode, the function has been
+                                  dispatched to specified AP.
+  @retval EFI_UNSUPPORTED         A non-blocking mode request was made after the
+                                  UEFI event EFI_EVENT_GROUP_READY_TO_BOOT was
+                                  signaled.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_TIMEOUT             In blocking mode, the timeout expired before
+                                  the specified AP has finished.
+  @retval EFI_NOT_READY           The specified AP is busy.
+  @retval EFI_NOT_FOUND           The processor with the handle specified by
+                                  ProcessorNumber does not exist.
+  @retval EFI_INVALID_PARAMETER   ProcessorNumber specifies the BSP or disabled AP.
+  @retval EFI_INVALID_PARAMETER   Procedure is NULL.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MP_SERVICES_STARTUP_THIS_AP)(
+  IN  EFI_MP_SERVICES_PROTOCOL  *This,
+  IN  EFI_AP_PROCEDURE          Procedure,
+  IN  UINTN                     ProcessorNumber,
+  IN  EFI_EVENT                 WaitEvent               OPTIONAL,
+  IN  UINTN                     TimeoutInMicroseconds,
+  IN  VOID                      *ProcedureArgument      OPTIONAL,
+  OUT BOOLEAN                   *Finished               OPTIONAL
+  );
+
+/**
+  This service switches the requested AP to be the BSP from that point onward.
+  This service changes the BSP for all purposes.   This call can only be performed
+  by the current BSP.
+
+  This service switches the requested AP to be the BSP from that point onward.
+  This service changes the BSP for all purposes. The new BSP can take over the
+  execution of the old BSP and continue seamlessly from where the old one left
+  off. This service may not be supported after the UEFI Event EFI_EVENT_GROUP_READY_TO_BOOT
+  is signaled.
+
+  If the BSP cannot be switched prior to the return from this service, then
+  EFI_UNSUPPORTED must be returned.
+
+  @param[in] This              A pointer to the EFI_MP_SERVICES_PROTOCOL instance.
+  @param[in] ProcessorNumber   The handle number of AP that is to become the new
+                               BSP. The range is from 0 to the total number of
+                               logical processors minus 1. The total number of
+                               logical processors can be retrieved by
+                               EFI_MP_SERVICES_PROTOCOL.GetNumberOfProcessors().
+  @param[in] EnableOldBSP      If TRUE, then the old BSP will be listed as an
+                               enabled AP. Otherwise, it will be disabled.
+
+  @retval EFI_SUCCESS             BSP successfully switched.
+  @retval EFI_UNSUPPORTED         Switching the BSP cannot be completed prior to
+                                  this service returning.
+  @retval EFI_UNSUPPORTED         Switching the BSP is not supported.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_NOT_FOUND           The processor with the handle specified by
+                                  ProcessorNumber does not exist.
+  @retval EFI_INVALID_PARAMETER   ProcessorNumber specifies the current BSP or
+                                  a disabled AP.
+  @retval EFI_NOT_READY           The specified AP is busy.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MP_SERVICES_SWITCH_BSP)(
+  IN EFI_MP_SERVICES_PROTOCOL  *This,
+  IN  UINTN                    ProcessorNumber,
+  IN  BOOLEAN                  EnableOldBSP
+  );
+
+/**
+  This service lets the caller enable or disable an AP from this point onward.
+  This service may only be called from the BSP.
+
+  This service allows the caller enable or disable an AP from this point onward.
+  The caller can optionally specify the health status of the AP by Health. If
+  an AP is being disabled, then the state of the disabled AP is implementation
+  dependent. If an AP is enabled, then the implementation must guarantee that a
+  complete initialization sequence is performed on the AP, so the AP is in a state
+  that is compatible with an MP operating system. This service may not be supported
+  after the UEFI Event EFI_EVENT_GROUP_READY_TO_BOOT is signaled.
+
+  If the enable or disable AP operation cannot be completed prior to the return
+  from this service, then EFI_UNSUPPORTED must be returned.
+
+  @param[in] This              A pointer to the EFI_MP_SERVICES_PROTOCOL instance.
+  @param[in] ProcessorNumber   The handle number of AP.
+                               The range is from 0 to the total number of
+                               logical processors minus 1. The total number of
+                               logical processors can be retrieved by
+                               EFI_MP_SERVICES_PROTOCOL.GetNumberOfProcessors().
+  @param[in] EnableAP          Specifies the new state for the processor for
+                               enabled, FALSE for disabled.
+  @param[in] HealthFlag        If not NULL, a pointer to a value that specifies
+                               the new health status of the AP. This flag
+                               corresponds to StatusFlag defined in
+                               EFI_MP_SERVICES_PROTOCOL.GetProcessorInfo(). Only
+                               the PROCESSOR_HEALTH_STATUS_BIT is used. All other
+                               bits are ignored.  If it is NULL, this parameter
+                               is ignored.
+
+  @retval EFI_SUCCESS             The specified AP was enabled or disabled successfully.
+  @retval EFI_UNSUPPORTED         Enabling or disabling an AP cannot be completed
+                                  prior to this service returning.
+  @retval EFI_UNSUPPORTED         Enabling or disabling an AP is not supported.
+  @retval EFI_DEVICE_ERROR        The calling processor is an AP.
+  @retval EFI_NOT_FOUND           Processor with the handle specified by ProcessorNumber
+                                  does not exist.
+  @retval EFI_INVALID_PARAMETER   ProcessorNumber specifies the BSP.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MP_SERVICES_ENABLEDISABLEAP)(
+  IN  EFI_MP_SERVICES_PROTOCOL  *This,
+  IN  UINTN                     ProcessorNumber,
+  IN  BOOLEAN                   EnableAP,
+  IN  UINT32                    *HealthFlag OPTIONAL
+  );
+
+/**
+  This return the handle number for the calling processor.  This service may be
+  called from the BSP and APs.
+
+  This service returns the processor handle number for the calling processor.
+  The returned value is in the range from 0 to the total number of logical
+  processors minus 1. The total number of logical processors can be retrieved
+  with EFI_MP_SERVICES_PROTOCOL.GetNumberOfProcessors(). This service may be
+  called from the BSP and APs. If ProcessorNumber is NULL, then EFI_INVALID_PARAMETER
+  is returned. Otherwise, the current processors handle number is returned in
+  ProcessorNumber, and EFI_SUCCESS is returned.
+
+  @param[in] This              A pointer to the EFI_MP_SERVICES_PROTOCOL instance.
+  @param[in] ProcessorNumber   Pointer to the handle number of AP.
+                               The range is from 0 to the total number of
+                               logical processors minus 1. The total number of
+                               logical processors can be retrieved by
+                               EFI_MP_SERVICES_PROTOCOL.GetNumberOfProcessors().
+
+  @retval EFI_SUCCESS             The current processor handle number was returned
+                                  in ProcessorNumber.
+  @retval EFI_INVALID_PARAMETER   ProcessorNumber is NULL.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_MP_SERVICES_WHOAMI)(
+  IN EFI_MP_SERVICES_PROTOCOL  *This,
+  OUT UINTN                    *ProcessorNumber
+  );
+
+///
+/// When installed, the MP Services Protocol produces a collection of services
+/// that are needed for MP management.
+///
+/// Before the UEFI event EFI_EVENT_GROUP_READY_TO_BOOT is signaled, the module
+/// that produces this protocol is required to place all APs into an idle state
+/// whenever the APs are disabled or the APs are not executing code as requested
+/// through the StartupAllAPs() or StartupThisAP() services. The idle state of
+/// an AP before the UEFI event EFI_EVENT_GROUP_READY_TO_BOOT is signaled is
+/// implementation dependent.
+///
+/// After the UEFI event EFI_EVENT_GROUP_READY_TO_BOOT is signaled, all the APs
+/// must be placed in the OS compatible CPU state as defined by the UEFI
+/// Specification. Implementations of this protocol may use the UEFI event
+/// EFI_EVENT_GROUP_READY_TO_BOOT to force APs into the OS compatible state as
+/// defined by the UEFI Specification. Modules that use this protocol must
+/// guarantee that all non-blocking mode requests on all APs have been completed
+/// before the UEFI event EFI_EVENT_GROUP_READY_TO_BOOT is signaled. Since the
+/// order that event notification functions in the same event group are executed
+/// is not deterministic, an event of type EFI_EVENT_GROUP_READY_TO_BOOT cannot
+/// be used to guarantee that APs have completed their non-blocking mode requests.
+///
+/// When the UEFI event EFI_EVENT_GROUP_READY_TO_BOOT is signaled, the StartAllAPs()
+/// and StartupThisAp() services must no longer support non-blocking mode requests.
+/// The support for SwitchBSP() and EnableDisableAP() may no longer be supported
+/// after this event is signaled. Since UEFI Applications and UEFI OS Loaders
+/// execute after the UEFI event EFI_EVENT_GROUP_READY_TO_BOOT is signaled, these
+/// UEFI images must be aware that the functionality of this protocol may be reduced.
+///
+struct _EFI_MP_SERVICES_PROTOCOL {
+  EFI_MP_SERVICES_GET_NUMBER_OF_PROCESSORS    GetNumberOfProcessors;
+  EFI_MP_SERVICES_GET_PROCESSOR_INFO          GetProcessorInfo;
+  EFI_MP_SERVICES_STARTUP_ALL_APS             StartupAllAPs;
+  EFI_MP_SERVICES_STARTUP_THIS_AP             StartupThisAP;
+  EFI_MP_SERVICES_SWITCH_BSP                  SwitchBSP;
+  EFI_MP_SERVICES_ENABLEDISABLEAP             EnableDisableAP;
+  EFI_MP_SERVICES_WHOAMI                      WhoAmI;
+};
+
+extern EFI_GUID  gEfiMpServiceProtocolGuid;
+
+#endif
diff --git a/src/include/ipxe/efi/efi_mp.h b/src/include/ipxe/efi/efi_mp.h
new file mode 100644
index 000000000..8dc4243e4
--- /dev/null
+++ b/src/include/ipxe/efi/efi_mp.h
@@ -0,0 +1,30 @@
+#ifndef _IPXE_EFI_MP_H
+#define _IPXE_EFI_MP_H
+
+/** @file
+ *
+ * EFI multiprocessor API implementation
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#ifdef MPAPI_EFI
+#define MPAPI_PREFIX_efi
+#else
+#define MPAPI_PREFIX_efi __efi_
+#endif
+
+/**
+ * Calculate address as seen by a multiprocessor function
+ *
+ * @v address		Address in boot processor address space
+ * @ret address		Address in application processor address space
+ */
+static inline __attribute__ (( always_inline )) mp_addr_t
+MPAPI_INLINE ( efi, mp_address ) ( void *address ) {
+
+	return ( ( mp_addr_t ) address );
+}
+
+#endif /* _IPXE_EFI_MP_H */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 1fe34113d..c673b9a40 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -81,6 +81,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_efi_strings	       ( ERRFILE_CORE | 0x00290000 )
 #define ERRFILE_uuid		       ( ERRFILE_CORE | 0x002a0000 )
 #define ERRFILE_efi_path	       ( ERRFILE_CORE | 0x002b0000 )
+#define ERRFILE_efi_mp		       ( ERRFILE_CORE | 0x002c0000 )
 
 #define ERRFILE_eisa		     ( ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_isa		     ( ERRFILE_DRIVER | 0x00010000 )
diff --git a/src/include/ipxe/mp.h b/src/include/ipxe/mp.h
index a36574909..9670dea55 100644
--- a/src/include/ipxe/mp.h
+++ b/src/include/ipxe/mp.h
@@ -102,6 +102,7 @@ extern void __asmcall mp_call ( mp_addr_t func, mp_addr_t opaque );
 
 /* Include all architecture-independent multiprocessor API headers */
 #include <ipxe/null_mp.h>
+#include <ipxe/efi/efi_mp.h>
 
 /* Include all architecture-dependent multiprocessor API headers */
 #include <bits/mp.h>
diff --git a/src/interface/efi/efi_mp.c b/src/interface/efi/efi_mp.c
new file mode 100644
index 000000000..fdbbc9ae8
--- /dev/null
+++ b/src/interface/efi/efi_mp.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * EFI multiprocessor API implementation
+ *
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/mp.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/Protocol/MpService.h>
+
+/** EFI multiprocessor function call data */
+struct efi_mp_func_data {
+	/** Multiprocessor function */
+	mp_addr_t func;
+	/** Opaque data pointer */
+	mp_addr_t opaque;
+};
+
+/** Multiprocessor services protocol */
+static EFI_MP_SERVICES_PROTOCOL *efimp;
+EFI_REQUEST_PROTOCOL ( EFI_MP_SERVICES_PROTOCOL, &efimp );
+
+/**
+ * Call multiprocessor function on current CPU
+ *
+ * @v buffer		Multiprocessor function call data
+ */
+static EFIAPI VOID efi_mp_call ( VOID *buffer ) {
+	struct efi_mp_func_data *data = buffer;
+
+	/* Call multiprocessor function */
+	mp_call ( data->func, data->opaque );
+}
+
+/**
+ * Execute a multiprocessor function on the boot processor
+ *
+ * @v func		Multiprocessor function
+ * @v opaque		Opaque data pointer
+ */
+static void efi_mp_exec_boot ( mp_func_t func, void *opaque ) {
+	struct efi_mp_func_data data;
+
+	/* Construct call data */
+	data.func = mp_address ( func );
+	data.opaque = mp_address ( opaque );
+
+	/* Call multiprocesor function */
+	efi_mp_call ( &data );
+}
+
+/**
+ * Start a multiprocessor function on all application processors
+ *
+ * @v func		Multiprocessor function
+ * @v opaque		Opaque data pointer
+ */
+static void efi_mp_start_all ( mp_func_t func, void *opaque ) {
+	struct efi_mp_func_data data;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Do nothing if MP services is not present */
+	if ( ! efimp ) {
+		DBGC ( func, "EFIMP has no multiprocessor services\n" );
+		return;
+	}
+
+	/* Construct call data */
+	data.func = mp_address ( func );
+	data.opaque = mp_address ( opaque );
+
+	/* Start up all application processors */
+	if ( ( efirc = efimp->StartupAllAPs ( efimp, efi_mp_call, FALSE, NULL,
+					      0, &data, NULL ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( func, "EFIMP could not start APs: %s\n",
+		       strerror ( rc ) );
+		return;
+	}
+}
+
+PROVIDE_MPAPI_INLINE ( efi, mp_address );
+PROVIDE_MPAPI ( efi, mp_exec_boot, efi_mp_exec_boot );
+PROVIDE_MPAPI ( efi, mp_start_all, efi_mp_start_all );

From a67f913d6635772af746fc215066e3d510ae25d1 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 15 Mar 2024 13:12:22 +0000
Subject: [PATCH 106/237] [librm] Add support for installing a startup IPI
 handler

Application processors are started via INIT and SIPI interprocessor
interrupts: the INIT places the processor into a "wait for SIPI"
state, and the SIPI then starts the processor in real mode at a
page-aligned address derived from the SIPI vector number.

Add support for installing a real-mode SIPI handler that will switch
the CPU into protected mode with flat physical addressing, load
initial register contents, and then jump to the address of a
protected-mode SIPI handler.  No stack pointer is set up, to avoid the
need to allocate stack space for each available processor.

We use 32-bit physical addressing in order to minimise the changes
required for a 64-bit build.  The existing long mode transition code
relies on the existence of the stack, so we cannot easily switch the
application processor into long mode.  We could use 32-bit virtual
addressing, but this runtime environment does not currently exist
outside of librm.S itself in a 64-bit build, and using it would
complicate the implementation of the protected-mode SIPI handler.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/include/librm.h          | 20 ++++++++
 src/arch/x86/transitions/librm.S      | 67 +++++++++++++++++++++++++++
 src/arch/x86/transitions/librm_mgmt.c | 26 +++++++++++
 3 files changed, 113 insertions(+)

diff --git a/src/arch/x86/include/librm.h b/src/arch/x86/include/librm.h
index 40f075439..84b345d3a 100644
--- a/src/arch/x86/include/librm.h
+++ b/src/arch/x86/include/librm.h
@@ -474,6 +474,26 @@ extern struct page_table io_pages;
  */
 #define IO_BASE ( ( void * ) 0x100000000ULL )
 
+/** Startup IPI real-mode handler */
+extern char __text16_array ( sipi, [] );
+#define sipi __use_text16 ( sipi )
+
+/** Length of startup IPI real-mode handler */
+extern char sipi_len[];
+
+/** Startup IPI real-mode handler copy of real-mode data segment */
+extern uint16_t __text16 ( sipi_ds );
+#define sipi_ds __use_text16 ( sipi_ds )
+
+/** Startup IPI protected-mode handler (physical address) */
+extern uint32_t sipi_handler;
+
+/** Startup IPI register state */
+extern struct i386_regs sipi_regs;
+
+extern void setup_sipi ( unsigned int vector, uint32_t handler,
+			 struct i386_regs *regs );
+
 #endif /* ASSEMBLY */
 
 #endif /* LIBRM_H */
diff --git a/src/arch/x86/transitions/librm.S b/src/arch/x86/transitions/librm.S
index 394313246..a93b0251f 100644
--- a/src/arch/x86/transitions/librm.S
+++ b/src/arch/x86/transitions/librm.S
@@ -1632,3 +1632,70 @@ init_pages:
 
 	/* Return */
 	ret
+
+/****************************************************************************
+ * sipi (real-mode jump)
+ *
+ * Handle Startup IPI
+ *
+ * This code must be copied to a page-aligned boundary in base memory.
+ * It will be entered with %cs:0000 pointing to the start of the code.
+ * The stack pointer is undefined and so no stack space can be used.
+ *
+ ****************************************************************************
+ */
+	.section ".text16.sipi", "ax", @progbits
+	.code16
+	.globl	sipi
+sipi:
+	/* Retrieve rm_ds from copy */
+	movw	%cs:( sipi_ds - sipi ), %ax
+	movw	%ax, %ds
+
+	/* Load GDT and switch to protected mode */
+	data32 lgdt gdtr
+	movl	%cr0, %eax
+	orb	$CR0_PE, %al
+	movl	%eax, %cr0
+	data32 ljmp $VIRTUAL_CS, $VIRTUAL(1f)
+
+	/* Copy of rm_ds required to access GDT */
+	.globl	sipi_ds
+sipi_ds:
+	.word	0
+
+	/* Length of real-mode SIPI handler to be copied */
+	.globl	sipi_len
+	.equ	sipi_len, . - sipi
+
+	.section ".text.sipi", "ax", @progbits
+	.code32
+1:	/* Set up protected-mode segment registers (with no stack) */
+	movw	$VIRTUAL_DS, %ax
+	movw	%ax, %ds
+	movw	%ax, %ss
+	movw	$PHYSICAL_DS, %ax
+	movw	%ax, %es
+	movw	%ax, %fs
+	movw	%ax, %gs
+
+	/* Load register state and clear stack pointer */
+	movl	$VIRTUAL(sipi_regs), %esp
+	popal
+
+	/* Switch to flat physical addressing */
+	movw	$PHYSICAL_DS, %sp
+	movw	%sp, %ds
+	movw	%sp, %ss
+
+	/* Clear stack pointer */
+	xorl	%esp, %esp
+
+	/* Jump to protected-mode SIPI handler */
+	ljmp	%cs:*VIRTUAL(sipi_handler)
+
+	/* Protected-mode SIPI handler vector */
+	.section ".data.sipi_handler", "aw", @progbits
+	.globl	sipi_handler
+sipi_handler:
+	.long	0, PHYSICAL_CS
diff --git a/src/arch/x86/transitions/librm_mgmt.c b/src/arch/x86/transitions/librm_mgmt.c
index da221e8b1..b3820589c 100644
--- a/src/arch/x86/transitions/librm_mgmt.c
+++ b/src/arch/x86/transitions/librm_mgmt.c
@@ -45,6 +45,9 @@ struct idtr64 idtr64 = {
 	.limit = ( sizeof ( idt64 ) - 1 ),
 };
 
+/** Startup IPI register state */
+struct i386_regs sipi_regs;
+
 /** Length of stack dump */
 #define STACK_DUMP_LEN 128
 
@@ -402,6 +405,29 @@ __asmcall void check_fxsr ( struct i386_all_regs *regs ) {
 	       ( ( regs->flags & CF ) ? " not" : "" ) );
 }
 
+/**
+ * Set up startup IPI handler
+ *
+ * @v vector		Startup IPI vector
+ * @v handler		Protected-mode startup IPI handler physical address
+ * @v regs		Initial register state
+ */
+void setup_sipi ( unsigned int vector, uint32_t handler,
+		  struct i386_regs *regs ) {
+
+	/* Record protected-mode handler */
+	sipi_handler = handler;
+
+	/* Update copy of rm_ds */
+	sipi_ds = rm_ds;
+
+	/* Save register state */
+	memcpy ( &sipi_regs, regs, sizeof ( sipi_regs ) );
+
+	/* Copy real-mode handler */
+	copy_to_real ( ( vector << 8 ), 0, sipi, ( ( size_t ) sipi_len ) );
+}
+
 PROVIDE_UACCESS_INLINE ( librm, phys_to_user );
 PROVIDE_UACCESS_INLINE ( librm, user_to_phys );
 PROVIDE_UACCESS_INLINE ( librm, virt_to_user );

From 1344e13a03cb6ed25372651cae6b057b863c89be Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 15 Mar 2024 17:30:21 +0000
Subject: [PATCH 107/237] [bios] Provide a multiprocessor API for BIOS

Provide an implementation of the iPXE multiprocessor API for BIOS,
based on sending broadcast INIT and SIPI interprocessor interrupts to
start up all application processors.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/include/bits/mp.h          |   2 +
 src/arch/x86/include/ipxe/bios_mp.h     |  32 +++++
 src/arch/x86/interface/pcbios/bios_mp.c | 172 ++++++++++++++++++++++++
 src/config/defaults/pcbios.h            |   2 +-
 4 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 src/arch/x86/include/ipxe/bios_mp.h
 create mode 100644 src/arch/x86/interface/pcbios/bios_mp.c

diff --git a/src/arch/x86/include/bits/mp.h b/src/arch/x86/include/bits/mp.h
index fe466b099..4541aca3e 100644
--- a/src/arch/x86/include/bits/mp.h
+++ b/src/arch/x86/include/bits/mp.h
@@ -9,4 +9,6 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <ipxe/bios_mp.h>
+
 #endif /* _BITS_MP_H */
diff --git a/src/arch/x86/include/ipxe/bios_mp.h b/src/arch/x86/include/ipxe/bios_mp.h
new file mode 100644
index 000000000..e2e83a15f
--- /dev/null
+++ b/src/arch/x86/include/ipxe/bios_mp.h
@@ -0,0 +1,32 @@
+#ifndef _IPXE_BIOS_MP_H
+#define _IPXE_BIOS_MP_H
+
+/** @file
+ *
+ * BIOS multiprocessor API implementation
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/io.h>
+
+#ifdef MPAPI_PCBIOS
+#define MPAPI_PREFIX_pcbios
+#else
+#define MPAPI_PREFIX_pcbios __pcbios_
+#endif
+
+/**
+ * Calculate address as seen by a multiprocessor function
+ *
+ * @v address		Address in boot processor address space
+ * @ret address		Address in application processor address space
+ */
+static inline __attribute__ (( always_inline )) mp_addr_t
+MPAPI_INLINE ( pcbios, mp_address ) ( void *address ) {
+
+	return virt_to_phys ( address );
+}
+
+#endif /* _IPXE_BIOS_MP_H */
diff --git a/src/arch/x86/interface/pcbios/bios_mp.c b/src/arch/x86/interface/pcbios/bios_mp.c
new file mode 100644
index 000000000..914fe5c10
--- /dev/null
+++ b/src/arch/x86/interface/pcbios/bios_mp.c
@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * BIOS multiprocessor API implementation
+ *
+ */
+
+#include <registers.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/timer.h>
+#include <ipxe/msr.h>
+#include <ipxe/mp.h>
+
+/** Local APIC base address MSR */
+#define MSR_APIC_BASE 0x0000001b
+
+/** Local APIC is in x2APIC mode */
+#define MSR_APIC_BASE_X2APIC 0x400
+
+/** Local APIC base address mask */
+#define MSR_APIC_BASE_MASK ( ~0xfffULL )
+
+/** Interrupt command register */
+#define APIC_ICR 0x0300
+
+/** Interrupt command register (x2APIC) */
+#define MSR_X2APIC_ICR 0x830
+
+/** Interrupt command register: send to all excluding self */
+#define APIC_ICR_ALL_NOT_SELF 0x000c0000
+
+/** Interrupt command register: level mode */
+#define APIC_ICR_LEVEL 0x00008000
+
+/** Interrupt command register: level asserted */
+#define APIC_ICR_LEVEL_ASSERT 0x00004000
+
+/** Interrupt command register: INIT */
+#define APIC_ICR_INIT 0x00000500
+
+/** Interrupt command register: SIPI */
+#define APIC_ICR_SIPI( vector ) ( 0x00000600 | (vector) )
+
+/** Time to wait for an IPI to complete */
+#define IPI_WAIT_MS 10
+
+/**
+ * Startup IPI vector
+ *
+ * The real-mode startup IPI code must be copied to a page boundary in
+ * base memory.  We fairly arbitrarily choose to place this at 0x8000.
+ */
+#define SIPI_VECTOR 0x08
+
+/** Protected-mode startup IPI handler */
+extern void __asmcall mp_jump ( mp_addr_t func, mp_addr_t opaque );
+
+/**
+ * Execute a multiprocessor function on the boot processor
+ *
+ * @v func		Multiprocessor function
+ * @v opaque		Opaque data pointer
+ */
+static void bios_mp_exec_boot ( mp_func_t func, void *opaque ) {
+
+	/* Call multiprocessor function with physical addressing */
+	__asm__ __volatile__ ( PHYS_CODE ( "pushl %k2\n\t"
+					   "pushl %k1\n\t"
+					   "call *%k0\n\t"
+					   "addl $8, %%esp\n\t" )
+			       : : "r" ( mp_address ( mp_call ) ),
+				   "r" ( mp_address ( func ) ),
+				   "r" ( mp_address ( opaque ) ) );
+}
+
+/**
+ * Send an interprocessor interrupt
+ *
+ * @v apic		APIC base address
+ * @v x2apic		x2APIC mode enabled
+ * @v icr		Interrupt control register value
+ */
+static void bios_mp_ipi ( void *apic, int x2apic, uint32_t icr ) {
+
+	/* Write ICR according to APIC/x2APIC mode */
+	DBGC ( MSR_APIC_BASE, "BIOSMP sending IPI %#08x\n", icr );
+	if ( x2apic ) {
+		wrmsr ( MSR_X2APIC_ICR, icr );
+	} else {
+		writel ( icr, ( apic + APIC_ICR ) );
+	}
+
+	/* Allow plenty of time for delivery to complete */
+	mdelay ( IPI_WAIT_MS );
+}
+
+/**
+ * Start a multiprocessor function on all application processors
+ *
+ * @v func		Multiprocessor function
+ * @v opaque		Opaque data pointer
+ */
+static void bios_mp_start_all ( mp_func_t func, void *opaque ) {
+	struct i386_regs regs;
+	uint64_t base;
+	uint32_t ipi;
+	void *apic;
+	int x2apic;
+
+	/* Prepare SIPI handler */
+	regs.eax = mp_address ( func );
+	regs.edx = mp_address ( opaque );
+	setup_sipi ( SIPI_VECTOR, virt_to_phys ( mp_jump ), &regs );
+
+	/* Get local APIC base address and mode */
+	base = rdmsr ( MSR_APIC_BASE );
+	x2apic = ( base & MSR_APIC_BASE_X2APIC );
+	DBGC ( MSR_APIC_BASE, "BIOSMP local %sAPIC base %#llx\n",
+	       ( x2apic ? "x2" : "" ), ( ( unsigned long long ) base ) );
+
+	/* Map local APIC */
+	apic = ioremap ( ( base & MSR_APIC_BASE_MASK ), PAGE_SIZE );
+	if ( ! apic )
+		goto err_ioremap;
+
+	/* Assert INIT IPI */
+	ipi = ( APIC_ICR_ALL_NOT_SELF | APIC_ICR_LEVEL |
+		APIC_ICR_LEVEL_ASSERT | APIC_ICR_INIT );
+	bios_mp_ipi ( apic, x2apic, ipi );
+
+	/* Clear INIT IPI */
+	ipi &= ~APIC_ICR_LEVEL_ASSERT;
+	bios_mp_ipi ( apic, x2apic, ipi );
+
+	/* Send SIPI */
+	ipi = ( APIC_ICR_ALL_NOT_SELF | APIC_ICR_SIPI ( SIPI_VECTOR ) );
+	bios_mp_ipi ( apic, x2apic, ipi );
+
+	iounmap ( apic );
+ err_ioremap:
+	/* No way to handle errors: caller must check that
+	 * multiprocessor function executed as expected.
+	 */
+}
+
+PROVIDE_MPAPI_INLINE ( pcbios, mp_address );
+PROVIDE_MPAPI ( pcbios, mp_exec_boot, bios_mp_exec_boot );
+PROVIDE_MPAPI ( pcbios, mp_start_all, bios_mp_start_all );
diff --git a/src/config/defaults/pcbios.h b/src/config/defaults/pcbios.h
index 968bd82a6..fa12a1005 100644
--- a/src/config/defaults/pcbios.h
+++ b/src/config/defaults/pcbios.h
@@ -24,7 +24,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define TIME_RTC
 #define REBOOT_PCBIOS
 #define ACPI_RSDP
-#define MPAPI_NULL
+#define MPAPI_PCBIOS
 
 #ifdef __x86_64__
 #define IOMAP_PAGES

From 17882e76afc0e69a0d4ed142aa33b94017ae4e58 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 15 Mar 2024 17:43:49 +0000
Subject: [PATCH 108/237] [ucode] Add support for updating x86 microcode

Intel and AMD distribute microcode updates, which are typically
applied by the BIOS and/or the booted operating system.

BIOS updates can be difficult to obtain and cumbersome to apply, and
are often neglected.  Operating system updates may be subject to
strict change control processes, particularly for production
workloads.  There is therefore value in being able to update the
microcode at boot time using a freshly downloaded microcode update
file, particularly in scenarios where the physical hardware and the
installed operating system are controlled by different parties (such
as in a public cloud infrastructure).

Add support for parsing Intel and AMD microcode update images, and for
applying the updates to all CPUs in the system.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/core/ucode_mp.S        | 257 +++++++++
 src/arch/x86/image/ucode.c          | 798 ++++++++++++++++++++++++++++
 src/arch/x86/include/bits/errfile.h |   1 +
 src/arch/x86/include/ipxe/ucode.h   | 223 ++++++++
 src/config/config.c                 |   3 +
 src/config/general.h                |   1 +
 6 files changed, 1283 insertions(+)
 create mode 100644 src/arch/x86/core/ucode_mp.S
 create mode 100644 src/arch/x86/image/ucode.c
 create mode 100644 src/arch/x86/include/ipxe/ucode.h

diff --git a/src/arch/x86/core/ucode_mp.S b/src/arch/x86/core/ucode_mp.S
new file mode 100644
index 000000000..808e881e0
--- /dev/null
+++ b/src/arch/x86/core/ucode_mp.S
@@ -0,0 +1,257 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Microcode updates
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+/* Selectively assemble code for 32-bit/64-bit builds */
+#if defined ( __x86_64__ ) && ! defined ( PLATFORM_pcbios )
+#define codemp code64
+#define AX rax
+#define BX rbx
+#define CX rcx
+#define DX rdx
+#define SI rsi
+#define DI rdi
+#define BP rbp
+#define SP rsp
+#define if32 if 0
+#define if64 if 1
+#else
+#define codemp code32
+#define AX eax
+#define BX ebx
+#define CX ecx
+#define DX edx
+#define SI esi
+#define DI edi
+#define BP ebp
+#define SP esp
+#define if32 if 1
+#define if64 if 0
+#endif
+
+/* Standard features CPUID leaf */
+#define CPUID_FEATURES 0x00000001
+
+/* BIOS update signature MSR */
+#define MSR_BIOS_SIGN_ID 0x0000008b
+
+/** Microcode update control layout
+ *
+ * This must match the layout of struct ucode_control.
+ */
+	.struct	0
+CONTROL_DESC:
+	.space	8
+CONTROL_STATUS:
+	.space	8
+CONTROL_TRIGGER_MSR:
+	.space	4
+CONTROL_APIC_MAX:
+	.space	4
+CONTROL_APIC_UNEXPECTED:
+	.space	4
+CONTROL_APIC_MASK:
+	.space	4
+CONTROL_APIC_TEST:
+	.space	4
+CONTROL_VER_CLEAR:
+	.space	1
+CONTROL_VER_HIGH:
+	.space	1
+CONTROL_LEN:
+
+/* We use register %ebp/%rbp to hold the address of the update control */
+#define CONTROL BP
+
+/* Microcode update descriptor layout
+ *
+ * This must match the layout of struct ucode_descriptor.
+ */
+	.struct	0
+DESC_SIGNATURE:
+	.space	4
+DESC_VERSION:
+	.space	4
+DESC_ADDRESS:
+	.space	8
+DESC_LEN:
+
+/* We use register %esi/%rsi to hold the address of the descriptor */
+#define DESC SI
+
+/** Microcode update status report layout
+ *
+ * This must match the layout of struct ucode_status.
+ */
+	.struct	0
+STATUS_SIGNATURE:
+	.space	4
+STATUS_ID:
+	.space	4
+STATUS_BEFORE:
+	.space	4
+STATUS_AFTER:
+	.space	4
+STATUS_LEN:
+	.equ	LOG2_STATUS_LEN, 4
+	.if	( 1 << LOG2_STATUS_LEN ) - STATUS_LEN
+	.error	"LOG2_STATUS_LEN value is incorrect"
+	.endif
+
+/* We use register %edi/%rdi to hold the address of the status report */
+#define STATUS DI
+
+/*
+ * Update microcode
+ *
+ * Parameters:
+ *	%eax/%rdi	Microcode update structure
+ *	%edx/%rsi	CPU identifier (APIC ID)
+ *	%esp/%rsp	Stack, or NULL to halt AP upon completion
+ *
+ * This code may run with no stack on an application processor (AP).
+ * All values must be held in registers, and no subroutine calls are
+ * possible.  No firmware routines may be called.
+ *
+ * Since cpuid/rdmsr/wrmsr require the use of %eax, %ebx, %ecx, and
+ * %edx, we have essentially only three registers available for
+ * long-term state.
+ */
+	.text
+	.globl	ucode_update
+	.codemp
+	.section ".text.ucode_update", "ax", @progbits
+ucode_update:
+
+.if64	/* Get input parameters */
+	movq	%rdi, %CONTROL
+	movl	%esi, %edx
+.else
+	movl	%eax, %CONTROL
+.endif
+	/* Check against maximum expected APIC ID */
+	cmpl	CONTROL_APIC_MAX(%CONTROL), %edx
+	jbe	1f
+	movl	%edx, CONTROL_APIC_UNEXPECTED(%CONTROL)
+	jmp	done
+1:
+	/* Calculate per-CPU status report buffer address */
+	mov	%DX, %STATUS
+	shl	$LOG2_STATUS_LEN, %STATUS
+	add	CONTROL_STATUS(%CONTROL), %STATUS
+
+	/* Report APIC ID */
+	movl	%edx, STATUS_ID(%STATUS)
+
+	/* Get and report CPU signature */
+	movl	$CPUID_FEATURES, %eax
+	cpuid
+	movl	%eax, STATUS_SIGNATURE(%STATUS)
+
+	/* Check APIC ID mask */
+	movl	STATUS_ID(%STATUS), %eax
+	andl	CONTROL_APIC_MASK(%CONTROL), %eax
+	cmpl	CONTROL_APIC_TEST(%CONTROL), %eax
+	jne	done
+
+	/* Clear BIOS_SIGN_ID MSR if applicable */
+	movl	$MSR_BIOS_SIGN_ID, %ecx
+	xorl	%eax, %eax
+	xorl	%edx, %edx
+	testb	$0xff, CONTROL_VER_CLEAR(%CONTROL)
+	jz	1f
+	wrmsr
+1:
+	/* Get CPU signature to repopulate BIOS_SIGN_ID MSR (for Intel) */
+	movl	$CPUID_FEATURES, %eax
+	cpuid
+
+	/* Get initial microcode version */
+	movl	$MSR_BIOS_SIGN_ID, %ecx
+	rdmsr
+	testb	$0xff, CONTROL_VER_HIGH(%CONTROL)
+	jz	1f
+	movl	%edx, %eax
+1:	movl	%eax, STATUS_BEFORE(%STATUS)
+
+	/* Get start of descriptor list */
+	mov	CONTROL_DESC(%CONTROL), %DESC
+	sub	$DESC_LEN, %DESC
+
+1:	/* Walk update descriptor list to find a matching CPU signature */
+	add	$DESC_LEN, %DESC
+	movl	DESC_SIGNATURE(%DESC), %eax
+	testl	%eax, %eax
+	jz	noload
+	cmpl	STATUS_SIGNATURE(%STATUS), %eax
+	jne	1b
+
+	/* Compare (signed) microcode versions */
+	movl	STATUS_BEFORE(%STATUS), %eax
+	cmpl	DESC_VERSION(%DESC), %eax
+	jge	noload
+
+	/* Load microcode update */
+	movl	CONTROL_TRIGGER_MSR(%CONTROL), %ecx
+	movl	(DESC_ADDRESS + 0)(%DESC), %eax
+	movl	(DESC_ADDRESS + 4)(%DESC), %edx
+	wrmsr
+
+noload:	/* Clear BIOS_SIGN_ID MSR if applicable */
+	movl	$MSR_BIOS_SIGN_ID, %ecx
+	xorl	%eax, %eax
+	xorl	%edx, %edx
+	testb	$0xff, CONTROL_VER_CLEAR(%CONTROL)
+	jz	1f
+	wrmsr
+1:
+	/* Get CPU signature to repopulate BIOS_SIGN_ID MSR (for Intel) */
+	movl	$CPUID_FEATURES, %eax
+	cpuid
+
+	/* Get and report final microcode version */
+	movl	$MSR_BIOS_SIGN_ID, %ecx
+	rdmsr
+	testb	$0xff, CONTROL_VER_HIGH(%CONTROL)
+	jz	1f
+	movl	%edx, %eax
+1:	movl	%eax, STATUS_AFTER(%STATUS)
+
+done:	/* Return to caller (if stack exists), or halt application processor */
+	test	%SP, %SP
+	jz	1f
+	ret
+1:	cli
+	hlt
+	jmp	1b
+	.size	ucode_update, . - ucode_update
diff --git a/src/arch/x86/image/ucode.c b/src/arch/x86/image/ucode.c
new file mode 100644
index 000000000..499c0a940
--- /dev/null
+++ b/src/arch/x86/image/ucode.c
@@ -0,0 +1,798 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Microcode updates
+ *
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <errno.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/umalloc.h>
+#include <ipxe/image.h>
+#include <ipxe/cpuid.h>
+#include <ipxe/msr.h>
+#include <ipxe/mp.h>
+#include <ipxe/timer.h>
+#include <ipxe/ucode.h>
+
+/**
+ * Maximum number of hyperthread siblings
+ *
+ * Microcode updates must not be performed on hyperthread siblings at
+ * the same time, since they share microcode storage.
+ *
+ * Hyperthread siblings are always the lowest level of the CPU
+ * topology and correspond to the least significant bits of the APIC
+ * ID.  We may therefore avoid collisions by performing the microcode
+ * updates in batches, with each batch targeting just one value for
+ * the least significant N bits of the APIC ID.
+ *
+ * We assume that no CPUs exist with more than this number of
+ * hyperthread siblings.  (This must be a power of two.)
+ */
+#define UCODE_MAX_HT 8
+
+/** Time to wait for a microcode update to complete */
+#define UCODE_WAIT_MS 10
+
+/** A CPU vendor string */
+union ucode_vendor_id {
+	/** CPUID registers */
+	uint32_t dword[3];
+	/** Human-readable string */
+	uint8_t string[12];
+};
+
+/** A CPU vendor */
+struct ucode_vendor {
+	/** Vendor string */
+	union ucode_vendor_id id;
+	/** Microcode load trigger MSR */
+	uint32_t trigger_msr;
+	/** Microcode version requires manual clear */
+	uint8_t ver_clear;
+	/** Microcode version is reported via high dword */
+	uint8_t ver_high;
+};
+
+/** A microcode update */
+struct ucode_update {
+	/** CPU vendor, if known */
+	struct ucode_vendor *vendor;
+	/** Boot processor CPU signature */
+	uint32_t signature;
+	/** Platform ID */
+	uint32_t platform;
+	/** Number of potentially relevant signatures found */
+	unsigned int count;
+	/** Update descriptors (if being populated) */
+	struct ucode_descriptor *desc;
+};
+
+/** A microcode update summary */
+struct ucode_summary {
+	/** Number of CPUs processed */
+	unsigned int count;
+	/** Lowest observed microcode version */
+	int32_t low;
+	/** Highest observed microcode version */
+	int32_t high;
+};
+
+/** Intel CPU vendor */
+static struct ucode_vendor ucode_intel = {
+	.id = { .string = "GenuineIntel" },
+	.ver_clear = 1,
+	.ver_high = 1,
+	.trigger_msr = MSR_UCODE_TRIGGER_INTEL,
+};
+
+/** AMD CPU vendor */
+static struct ucode_vendor ucode_amd = {
+	.id = { .string = "AuthenticAMD" },
+	.trigger_msr = MSR_UCODE_TRIGGER_AMD,
+};
+
+/** List of known CPU vendors */
+static struct ucode_vendor *ucode_vendors[] = {
+	&ucode_intel,
+	&ucode_amd,
+};
+
+/**
+ * Get CPU vendor name (for debugging)
+ *
+ * @v vendor		CPU vendor
+ * @ret name		Name
+ */
+static const char * ucode_vendor_name ( const union ucode_vendor_id *vendor ) {
+	static union {
+		union ucode_vendor_id vendor;
+		char text[ sizeof ( *vendor ) + 1 /* NUL */ ];
+	} u;
+
+	/* Construct name */
+	memcpy ( &u.vendor, vendor, sizeof ( u.vendor ) );
+	u.text[ sizeof ( u.text ) - 1 ] = '\0';
+	return u.text;
+}
+
+/**
+ * Check status report
+ *
+ * @v update		Microcode update
+ * @v control		Microcode update control
+ * @v summary		Microcode update summary
+ * @v id		APIC ID
+ * @v optional		Status report is optional
+ * @ret rc		Return status code
+ */
+static int ucode_status ( struct ucode_update *update,
+			  struct ucode_control *control,
+			  struct ucode_summary *summary,
+			  unsigned int id, int optional ) {
+	struct ucode_status status;
+	struct ucode_descriptor *desc;
+
+	/* Sanity check */
+	assert ( id <= control->apic_max );
+
+	/* Read status report */
+	copy_from_user ( &status, phys_to_user ( control->status ),
+			 ( id * sizeof ( status ) ), sizeof ( status ) );
+
+	/* Ignore empty optional status reports */
+	if ( optional && ( ! status.signature ) )
+		return 0;
+	DBGC ( update, "UCODE %#08x signature %#08x ucode %#08x->%#08x\n",
+	       id, status.signature, status.before, status.after );
+
+	/* Check CPU signature */
+	if ( ! status.signature ) {
+		DBGC2 ( update, "UCODE %#08x has no signature\n", id );
+		return -ENOENT;
+	}
+
+	/* Check APIC ID is correct */
+	if ( status.id != id ) {
+		DBGC ( update, "UCODE %#08x wrong APIC ID %#08x\n",
+		       id, status.id );
+		return -EINVAL;
+	}
+
+	/* Check that maximum APIC ID was not exceeded */
+	if ( control->apic_unexpected ) {
+		DBGC ( update, "UCODE %#08x saw unexpected APIC ID %#08x\n",
+		       id, control->apic_unexpected );
+		return -ERANGE;
+	}
+
+	/* Check microcode was not downgraded */
+	if ( status.after < status.before ) {
+		DBGC ( update, "UCODE %#08x was downgraded %#08x->%#08x\n",
+		       id, status.before, status.after );
+		return -ENOTTY;
+	}
+
+	/* Check that expected updates (if any) were applied */
+	for ( desc = update->desc ; desc->signature ; desc++ ) {
+		if ( ( desc->signature == status.signature ) &&
+		     ( status.after < desc->version ) ) {
+			DBGC ( update, "UCODE %#08x failed update %#08x->%#08x "
+			       "(wanted %#08x)\n", id, status.before,
+			       status.after, desc->version );
+			return -EIO;
+		}
+	}
+
+	/* Update summary */
+	summary->count++;
+	if ( status.before < summary->low )
+		summary->low = status.before;
+	if ( status.after > summary->high )
+		summary->high = status.after;
+
+	return 0;
+}
+
+/**
+ * Update microcode on all CPUs
+ *
+ * @v image		Microcode image
+ * @v update		Microcode update
+ * @v summary		Microcode update summary to fill in
+ * @ret rc		Return status code
+ */
+static int ucode_update_all ( struct image *image,
+			      struct ucode_update *update,
+			      struct ucode_summary *summary ) {
+	struct ucode_control control;
+	struct ucode_vendor *vendor;
+	userptr_t status;
+	unsigned int max;
+	unsigned int i;
+	size_t len;
+	int rc;
+
+	/* Initialise summary */
+	summary->count = 0;
+	summary->low = UCODE_VERSION_MAX;
+	summary->high = UCODE_VERSION_MIN;
+
+	/* Allocate status reports */
+	max = mp_max_cpuid();
+	len = ( ( max + 1 ) * sizeof ( struct ucode_status ) );
+	status = umalloc ( len );
+	if ( ! status ) {
+		DBGC ( image, "UCODE %s could not allocate %d status reports\n",
+		       image->name, ( max + 1 ) );
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	memset_user ( status, 0, 0, len );
+
+	/* Construct control structure */
+	memset ( &control, 0, sizeof ( control ) );
+	control.desc = virt_to_phys ( update->desc );
+	control.status = user_to_phys ( status, 0 );
+	vendor = update->vendor;
+	if ( vendor ) {
+		control.ver_clear = vendor->ver_clear;
+		control.ver_high = vendor->ver_high;
+		control.trigger_msr = vendor->trigger_msr;
+	} else {
+		assert ( update->count == 0 );
+	}
+	control.apic_max = max;
+
+	/* Update microcode on boot processor */
+	mp_exec_boot ( ucode_update, &control );
+	if ( ( rc = ucode_status ( update, &control, summary,
+				   mp_boot_cpuid(), 0 ) ) != 0 ) {
+		DBGC ( image, "UCODE %s failed on boot processor: %s\n",
+		       image->name, strerror ( rc ) );
+		goto err_boot;
+	}
+
+	/* Update microcode on application processors, avoiding
+	 * simultaneous updates on hyperthread siblings.
+	 */
+	build_assert ( ( UCODE_MAX_HT & ( UCODE_MAX_HT - 1 ) ) == 0 );
+	control.apic_mask = ( UCODE_MAX_HT - 1 );
+	for ( ; control.apic_test <= control.apic_mask ; control.apic_test++ ) {
+		mp_start_all ( ucode_update, &control );
+		mdelay ( UCODE_WAIT_MS );
+	}
+
+	/* Check status reports */
+	summary->count = 0;
+	for ( i = 0 ; i <= max ; i++ ) {
+		if ( ( rc = ucode_status ( update, &control, summary,
+					   i, 1 ) ) != 0 ) {
+			goto err_status;
+		}
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_status:
+ err_boot:
+	ufree ( status );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Add descriptor to list (if applicable)
+ *
+ * @v image		Microcode image
+ * @v start		Starting offset within image
+ * @v vendor		CPU vendor
+ * @v desc		Microcode descriptor
+ * @v platforms		Supported platforms, or 0 for all platforms
+ * @v update		Microcode update
+ */
+static void ucode_describe ( struct image *image, size_t start,
+			     const struct ucode_vendor *vendor,
+			     const struct ucode_descriptor *desc,
+			     uint32_t platforms, struct ucode_update *update ) {
+
+	/* Dump descriptor information */
+	DBGC2 ( image, "UCODE %s+%#04zx %s %#08x", image->name, start,
+		ucode_vendor_name ( &vendor->id ), desc->signature );
+	if ( platforms )
+		DBGC2 ( image, " (%#02x)", platforms );
+	DBGC2 ( image, " version %#08x\n", desc->version );
+
+	/* Check applicability */
+	if ( vendor != update->vendor )
+		return;
+	if ( ( desc->signature ^ update->signature ) & UCODE_SIGNATURE_MASK )
+		return;
+	if ( platforms && ( ! ( platforms & update->platform ) ) )
+		return;
+
+	/* Add descriptor, if applicable */
+	if ( update->desc ) {
+		memcpy ( &update->desc[update->count], desc, sizeof ( *desc ) );
+		DBGC ( image, "UCODE %s+%#04zx found %s %#08x version %#08x\n",
+		       image->name, start, ucode_vendor_name ( &vendor->id ),
+		       desc->signature, desc->version );
+	}
+	update->count++;
+}
+
+/**
+ * Verify checksum
+ *
+ * @v image		Microcode image
+ * @v start		Starting offset
+ * @v len		Length
+ * @ret rc		Return status code
+ */
+static int ucode_verify ( struct image *image, size_t start, size_t len ) {
+	uint32_t checksum = 0;
+	uint32_t dword;
+	size_t offset;
+
+	/* Check length is a multiple of dwords */
+	if ( ( len % sizeof ( dword ) ) != 0 ) {
+		DBGC ( image, "UCODE %s+%#04zx invalid length %#zx\n",
+		       image->name, start, len );
+		return -EINVAL;
+	}
+
+	/* Calculate checksum */
+	for ( offset = start ; len ;
+	      offset += sizeof ( dword ), len -= sizeof ( dword ) ) {
+		copy_from_user ( &dword, image->data, offset,
+				 sizeof ( dword ) );
+		checksum += dword;
+	}
+	if ( checksum != 0 ) {
+		DBGC ( image, "UCODE %s+%#04zx bad checksum %#08x\n",
+		       image->name, start, checksum );
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * Parse Intel microcode image
+ *
+ * @v image		Microcode image
+ * @v start		Starting offset within image
+ * @v update		Microcode update
+ * @ret len		Length consumed, or negative error
+ */
+static int ucode_parse_intel ( struct image *image, size_t start,
+			       struct ucode_update *update ) {
+	struct intel_ucode_header hdr;
+	struct intel_ucode_ext_header exthdr;
+	struct intel_ucode_ext ext;
+	struct ucode_descriptor desc;
+	size_t remaining;
+	size_t offset;
+	size_t data_len;
+	size_t len;
+	unsigned int i;
+	int rc;
+
+	/* Read header */
+	remaining = ( image->len - start );
+	if ( remaining < sizeof ( hdr ) ) {
+		DBGC ( image, "UCODE %s+%#04zx too small for Intel header\n",
+		       image->name, start );
+		return -ENOEXEC;
+	}
+	copy_from_user ( &hdr, image->data, start, sizeof ( hdr ) );
+
+	/* Determine lengths */
+	data_len = hdr.data_len;
+	if ( ! data_len )
+		data_len = INTEL_UCODE_DATA_LEN;
+	len = hdr.len;
+	if ( ! len )
+		len = ( sizeof ( hdr ) + data_len );
+
+	/* Verify a selection of fields */
+	if ( ( hdr.hver != INTEL_UCODE_HVER ) ||
+	     ( hdr.lver != INTEL_UCODE_LVER ) ||
+	     ( len < sizeof ( hdr ) ) ||
+	     ( len > remaining ) ||
+	     ( data_len > ( len - sizeof ( hdr ) ) ) ||
+	     ( ( data_len % sizeof ( uint32_t ) ) != 0 ) ||
+	     ( ( len % INTEL_UCODE_ALIGN ) != 0 ) ) {
+		DBGC2 ( image, "UCODE %s+%#04zx is not an Intel update\n",
+			image->name, start );
+		return -EINVAL;
+	}
+	DBGC2 ( image, "UCODE %s+%#04zx is an Intel update\n",
+		image->name, start );
+
+	/* Verify checksum */
+	if ( ( rc = ucode_verify ( image, start, len ) ) != 0 )
+		return rc;
+
+	/* Populate descriptor */
+	desc.signature = hdr.signature;
+	desc.version = hdr.version;
+	desc.address = user_to_phys ( image->data,
+				      ( start + sizeof ( hdr ) ) );
+
+	/* Add non-extended descriptor, if applicable */
+	ucode_describe ( image, start, &ucode_intel, &desc, hdr.platforms,
+			 update );
+
+	/* Construct extended descriptors, if applicable */
+	offset = ( sizeof ( hdr ) + data_len );
+	if ( offset <= ( len - sizeof ( exthdr ) ) ) {
+
+		/* Read extended header */
+		copy_from_user ( &exthdr, image->data, ( start + offset ),
+				 sizeof ( exthdr ) );
+		offset += sizeof ( exthdr );
+
+		/* Read extended signatures */
+		for ( i = 0 ; i < exthdr.count ; i++ ) {
+
+			/* Read extended signature */
+			if ( offset > ( len - sizeof ( ext ) ) ) {
+				DBGC ( image, "UCODE %s+%#04zx extended "
+				       "signature overrun\n",
+				       image->name, start );
+				return -EINVAL;
+			}
+			copy_from_user ( &ext, image->data, ( start + offset ),
+					 sizeof ( ext ) );
+			offset += sizeof ( ext );
+
+			/* Avoid duplicating non-extended descriptor */
+			if ( ( ext.signature == hdr.signature ) &&
+			     ( ext.platforms == hdr.platforms ) ) {
+				continue;
+			}
+
+			/* Construct descriptor, if applicable */
+			desc.signature = ext.signature;
+			ucode_describe ( image, start, &ucode_intel, &desc,
+					 ext.platforms, update );
+		}
+	}
+
+	return len;
+}
+
+/**
+ * Parse AMD microcode image
+ *
+ * @v image		Microcode image
+ * @v start		Starting offset within image
+ * @v update		Microcode update
+ * @ret len		Length consumed, or negative error
+ */
+static int ucode_parse_amd ( struct image *image, size_t start,
+			     struct ucode_update *update ) {
+	struct amd_ucode_header hdr;
+	struct amd_ucode_equivalence equiv;
+	struct amd_ucode_patch_header phdr;
+	struct amd_ucode_patch patch;
+	struct ucode_descriptor desc;
+	size_t remaining;
+	size_t offset;
+	unsigned int count;
+	unsigned int used;
+	unsigned int i;
+
+	/* Read header */
+	remaining = ( image->len - start );
+	if ( remaining < sizeof ( hdr ) ) {
+		DBGC ( image, "UCODE %s+%#04zx too small for AMD header\n",
+		       image->name, start );
+		return -ENOEXEC;
+	}
+	copy_from_user ( &hdr, image->data, start, sizeof ( hdr ) );
+
+	/* Check header */
+	if ( hdr.magic != AMD_UCODE_MAGIC ) {
+		DBGC2 ( image, "UCODE %s+%#04zx is not an AMD update\n",
+			image->name, start );
+		return -ENOEXEC;
+	}
+	DBGC2 ( image, "UCODE %s+%#04zx is an AMD update\n",
+		image->name, start );
+	if ( hdr.type != AMD_UCODE_EQUIV_TYPE ) {
+		DBGC ( image, "UCODE %s+%#04zx unsupported equivalence table "
+		       "type %d\n", image->name, start, hdr.type );
+		return -ENOTSUP;
+	}
+	if ( hdr.len > ( remaining - sizeof ( hdr ) ) ) {
+		DBGC ( image, "UCODE %s+%#04zx truncated equivalence table\n",
+		       image->name, start );
+		return -EINVAL;
+	}
+
+	/* Count number of equivalence table entries */
+	offset = sizeof ( hdr );
+	for ( count = 0 ; offset < ( sizeof ( hdr ) + hdr.len ) ;
+	      count++, offset += sizeof ( equiv ) ) {
+		copy_from_user ( &equiv, image->data, ( start + offset ),
+				 sizeof ( equiv ) );
+		if ( ! equiv.signature )
+			break;
+	}
+	DBGC2 ( image, "UCODE %s+%#04zx has %d equivalence table entries\n",
+		image->name, start, count );
+
+	/* Parse available updates */
+	offset = ( sizeof ( hdr ) + hdr.len );
+	used = 0;
+	while ( used < count ) {
+
+		/* Read patch header */
+		if ( ( offset + sizeof ( phdr ) ) > remaining ) {
+			DBGC ( image, "UCODE %s+%#04zx truncated patch "
+			       "header\n", image->name, start );
+			return -EINVAL;
+		}
+		copy_from_user ( &phdr, image->data, ( start + offset ),
+				 sizeof ( phdr ) );
+		offset += sizeof ( phdr );
+
+		/* Validate patch header */
+		if ( phdr.type != AMD_UCODE_PATCH_TYPE ) {
+			DBGC ( image, "UCODE %s+%#04zx unsupported patch type "
+			       "%d\n", image->name, start, phdr.type );
+			return -ENOTSUP;
+		}
+		if ( phdr.len < sizeof ( patch ) ) {
+			DBGC ( image, "UCODE %s+%#04zx underlength patch\n",
+			       image->name, start );
+			return -EINVAL;
+		}
+		if ( phdr.len > ( remaining - offset ) ) {
+			DBGC ( image, "UCODE %s+%#04zx truncated patch\n",
+			       image->name, start );
+			return -EINVAL;
+		}
+
+		/* Read patch and construct descriptor */
+		copy_from_user ( &patch, image->data, ( start + offset ),
+				 sizeof ( patch ) );
+		desc.version = patch.version;
+		desc.address = user_to_phys ( image->data, ( start + offset ) );
+		offset += phdr.len;
+
+		/* Parse equivalence table to find matching signatures */
+		for ( i = 0 ; i < count ; i++ ) {
+			copy_from_user ( &equiv, image->data,
+					 ( start + sizeof ( hdr ) +
+					   ( i * ( sizeof ( equiv ) ) ) ),
+					 sizeof ( equiv ) );
+			if ( patch.id == equiv.id ) {
+				desc.signature = equiv.signature;
+				ucode_describe ( image, start, &ucode_amd,
+						 &desc, 0, update );
+				used++;
+			}
+		}
+	}
+
+	return offset;
+}
+
+/**
+ * Parse microcode image
+ *
+ * @v image		Microcode image
+ * @v update		Microcode update
+ * @ret rc		Return status code
+ */
+static int ucode_parse ( struct image *image, struct ucode_update *update ) {
+	size_t start;
+	int len;
+
+	/* Attempt to parse concatenated microcode updates */
+	for ( start = 0 ; start < image->len ; start += len ) {
+
+		/* Attempt to parse as Intel microcode */
+		len = ucode_parse_intel ( image, start, update );
+		if ( len > 0 )
+			continue;
+
+		/* Attempt to parse as AMD microcode */
+		len = ucode_parse_amd ( image, start, update );
+		if ( len > 0 )
+			continue;
+
+		/* Not a recognised microcode format */
+		DBGC ( image, "UCODE %s+%zx not recognised\n",
+		       image->name, start );
+		return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+/**
+ * Execute microcode update
+ *
+ * @v image		Microcode image
+ * @ret rc		Return status code
+ */
+static int ucode_exec ( struct image *image ) {
+	struct ucode_update update;
+	struct ucode_vendor *vendor;
+	struct ucode_summary summary;
+	union ucode_vendor_id id;
+	uint64_t platform_id;
+	uint32_t discard_a;
+	uint32_t discard_b;
+	uint32_t discard_c;
+	uint32_t discard_d;
+	unsigned int check;
+	unsigned int i;
+	size_t len;
+	int rc;
+
+	/* Initialise update */
+	memset ( &update, 0, sizeof ( update ) );
+	cpuid ( CPUID_VENDOR_ID, 0, &discard_a, &id.dword[0], &id.dword[2],
+		&id.dword[1] );
+	cpuid ( CPUID_FEATURES, 0, &update.signature, &discard_b,
+		&discard_c, &discard_d );
+
+	/* Identify CPU vendor, if recognised */
+	for ( i = 0 ; i < ( sizeof ( ucode_vendors ) /
+			    sizeof ( ucode_vendors[0] ) ) ; i++ ) {
+		vendor = ucode_vendors[i];
+		if ( memcmp ( &id, &vendor->id, sizeof ( id ) ) == 0 )
+			update.vendor = vendor;
+	}
+
+	/* Identify platform, if applicable */
+	if ( update.vendor == &ucode_intel ) {
+		platform_id = rdmsr ( MSR_PLATFORM_ID );
+		update.platform =
+			( 1 << MSR_PLATFORM_ID_VALUE ( platform_id ) );
+	}
+
+	/* Count number of matching update descriptors */
+	DBGC ( image, "UCODE %s applying to %s %#08x",
+	       image->name, ucode_vendor_name ( &id ), update.signature );
+	if ( update.platform )
+		DBGC ( image, " (%#02x)", update.platform );
+	DBGC ( image, "\n" );
+	if ( ( rc = ucode_parse ( image, &update ) ) != 0 )
+		goto err_count;
+	DBGC ( image, "UCODE %s found %d matching update(s)\n",
+	       image->name, update.count );
+
+	/* Allocate descriptors */
+	len = ( ( update.count + 1 /* terminator */ ) *
+		sizeof ( update.desc[0] ) );
+	update.desc = zalloc ( len );
+	if ( ! update.desc ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Populate descriptors */
+	check = update.count;
+	update.count = 0;
+	if ( ( rc = ucode_parse ( image, &update ) ) != 0 )
+		goto err_parse;
+	assert ( check == update.count );
+
+	/* Perform update */
+	if ( ( rc = ucode_update_all ( image, &update, &summary ) ) != 0 )
+		goto err_update;
+
+	/* Print summary if directed to do so */
+	if ( image->cmdline && ( strstr ( image->cmdline, "-v" ) ) ) {
+		printf ( "Microcode: " );
+		if ( summary.low == summary.high ) {
+			printf ( "already version %#x", summary.low );
+		} else {
+			printf ( "updated version %#x->%#x",
+				 summary.low, summary.high );
+		}
+		printf ( " (x%d)\n", summary.count );
+	}
+
+ err_update:
+ err_parse:
+	free ( update.desc );
+ err_alloc:
+ err_count:
+	return rc;
+}
+
+/**
+ * Probe microcode update image
+ *
+ * @v image		Microcode image
+ * @ret rc		Return status code
+ */
+static int ucode_probe ( struct image *image ) {
+	union {
+		struct intel_ucode_header intel;
+		struct amd_ucode_header amd;
+	} header;
+
+	/* Sanity check */
+	if ( image->len < sizeof ( header )  ) {
+		DBGC ( image, "UCODE %s too short\n", image->name );
+		return -ENOEXEC;
+	}
+
+	/* Read first microcode image header */
+	copy_from_user ( &header, image->data, 0, sizeof ( header ) );
+
+	/* Check for something that looks like an Intel update
+	 *
+	 * Intel updates unfortunately have no magic signatures or
+	 * other easily verifiable fields.  We check a small selection
+	 * of header fields that can be easily verified.
+	 *
+	 * We do not attempt to fully parse the update, since we want
+	 * errors to be reported at the point of attempting to execute
+	 * the image, and do not want to have a microcode image
+	 * erroneously treated as a PXE boot executable.
+	 */
+	if ( ( header.intel.hver == INTEL_UCODE_HVER ) &&
+	     ( header.intel.lver == INTEL_UCODE_LVER ) &&
+	     ( ( header.intel.date.century == 0x19 ) ||
+	       ( ( header.intel.date.century >= 0x20 ) &&
+		 ( header.intel.date.century <= 0x29 ) ) ) ) {
+		DBGC ( image, "UCODE %s+%#04zx looks like an Intel update\n",
+		       image->name, ( ( size_t ) 0 ) );
+		return 0;
+	}
+
+	/* Check for AMD update signature */
+	if ( ( header.amd.magic == AMD_UCODE_MAGIC ) &&
+	     ( header.amd.type == AMD_UCODE_EQUIV_TYPE ) ) {
+		DBGC ( image, "UCODE %s+%#04zx looks like an AMD update\n",
+		       image->name, ( ( size_t ) 0 ) );
+		return 0;
+	}
+
+	return -ENOEXEC;
+}
+
+/** Microcode update image type */
+struct image_type ucode_image_type __image_type ( PROBE_NORMAL ) = {
+	.name = "ucode",
+	.probe = ucode_probe,
+	.exec = ucode_exec,
+};
diff --git a/src/arch/x86/include/bits/errfile.h b/src/arch/x86/include/bits/errfile.h
index b5316a586..78b3dea1c 100644
--- a/src/arch/x86/include/bits/errfile.h
+++ b/src/arch/x86/include/bits/errfile.h
@@ -44,6 +44,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_sdi	       ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000b0000 )
 #define ERRFILE_initrd	       ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000c0000 )
 #define ERRFILE_pxe_call       ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000d0000 )
+#define ERRFILE_ucode	       ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000e0000 )
 
 #define ERRFILE_undi		 ( ERRFILE_ARCH | ERRFILE_NET | 0x00000000 )
 #define ERRFILE_undiload	 ( ERRFILE_ARCH | ERRFILE_NET | 0x00010000 )
diff --git a/src/arch/x86/include/ipxe/ucode.h b/src/arch/x86/include/ipxe/ucode.h
new file mode 100644
index 000000000..964e8d7b1
--- /dev/null
+++ b/src/arch/x86/include/ipxe/ucode.h
@@ -0,0 +1,223 @@
+#ifndef _IPXE_UCODE_H
+#define _IPXE_UCODE_H
+
+/** @file
+ *
+ * Microcode updates
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/mp.h>
+
+/** Platform ID MSR */
+#define MSR_PLATFORM_ID 0x00000017UL
+
+/** Extract platform ID from MSR value */
+#define MSR_PLATFORM_ID_VALUE( value ) ( ( (value) >> 50 ) & 0x7 )
+
+/** Intel microcode load trigger MSR */
+#define MSR_UCODE_TRIGGER_INTEL 0x00000079UL
+
+/** AMD microcode load trigger MSR */
+#define MSR_UCODE_TRIGGER_AMD 0xc0010020UL
+
+/** CPUID signature applicability mask
+ *
+ * We assume that only steppings may vary between the boot CPU and any
+ * application processors.
+ */
+#define UCODE_SIGNATURE_MASK 0xfffffff0UL
+
+/** Minimum possible microcode version */
+#define UCODE_VERSION_MIN -0x80000000L
+
+/** Maximum possible microcode version */
+#define UCODE_VERSION_MAX 0x7fffffffL
+
+/** A microcode update control
+ *
+ * This must match the layout as used by the assembly code in
+ * ucode_mp.S.
+ */
+struct ucode_control {
+	/** Microcode descriptor list physical address */
+	uint64_t desc;
+	/** Microcode status array physical address */
+	uint64_t status;
+	/** Microcode load trigger MSR */
+	uint32_t trigger_msr;
+	/** Maximum expected APIC ID */
+	uint32_t apic_max;
+	/** Unexpected APIC ID
+	 *
+	 * Any application processor may set this to indicate that its
+	 * APIC ID was higher than the maximum expected APIC ID.
+	 */
+	uint32_t apic_unexpected;
+	/** APIC ID eligibility mask bits */
+	uint32_t apic_mask;
+	/** APIC ID eligibility test bits */
+	uint32_t apic_test;
+	/** Microcode version requires manual clear */
+	uint8_t ver_clear;
+	/** Microcode version is reported via high dword */
+	uint8_t ver_high;
+} __attribute__ (( packed ));
+
+/** A microcode update descriptor
+ *
+ * This must match the layout as used by the assembly code in
+ * ucode_mp.S.
+ */
+struct ucode_descriptor {
+	/** CPUID signature (or 0 to terminate list) */
+	uint32_t signature;
+	/** Microcode version */
+	int32_t version;
+	/** Microcode physical address */
+	uint64_t address;
+} __attribute__ (( packed ));
+
+/** A microcode update status report
+ *
+ * This must match the layout as used by the assembly code in
+ * ucode_mp.S.
+ */
+struct ucode_status {
+	/** CPU signature */
+	uint32_t signature;
+	/** APIC ID (for sanity checking) */
+	uint32_t id;
+	/** Initial microcode version */
+	int32_t before;
+	/** Final microcode version */
+	int32_t after;
+} __attribute__ (( packed ));
+
+/** A microcode date */
+struct ucode_date {
+	/** Year (BCD) */
+	uint8_t year;
+	/** Century (BCD) */
+	uint8_t century;
+	/** Day (BCD) */
+	uint8_t day;
+	/** Month (BCD) */
+	uint8_t month;
+} __attribute__ (( packed ));
+
+/** An Intel microcode update file header */
+struct intel_ucode_header {
+	/** Header version number */
+	uint32_t hver;
+	/** Microcode version */
+	int32_t version;
+	/** Date */
+	struct ucode_date date;
+	/** CPUID signature */
+	uint32_t signature;
+	/** Checksum */
+	uint32_t checksum;
+	/** Loader version */
+	uint32_t lver;
+	/** Supported platforms */
+	uint32_t platforms;
+	/** Microcode data size (or 0 to indicate 2000 bytes) */
+	uint32_t data_len;
+	/** Total size (or 0 to indicate 2048 bytes) */
+	uint32_t len;
+	/** Reserved */
+	uint8_t reserved[12];
+} __attribute__ (( packed ));
+
+/** Intel microcode header version number */
+#define INTEL_UCODE_HVER 0x00000001UL
+
+/** Intel microcode loader version number */
+#define INTEL_UCODE_LVER 0x00000001UL
+
+/** Intel microcode default data length */
+#define INTEL_UCODE_DATA_LEN 2000
+
+/** Intel microcode file alignment */
+#define INTEL_UCODE_ALIGN 1024
+
+/** An Intel microcode update file extended header */
+struct intel_ucode_ext_header {
+	/** Extended signature count */
+	uint32_t count;
+	/** Extended checksum */
+	uint32_t checksum;
+	/** Reserved */
+	uint8_t reserved[12];
+} __attribute__ (( packed ));
+
+/** An Intel microcode extended signature */
+struct intel_ucode_ext {
+	/** CPUID signature */
+	uint32_t signature;
+	/** Supported platforms */
+	uint32_t platforms;
+	/** Checksum */
+	uint32_t checksum;
+} __attribute__ (( packed ));
+
+/** An AMD microcode update file header */
+struct amd_ucode_header {
+	/** Magic signature */
+	uint32_t magic;
+	/** Equivalence table type */
+	uint32_t type;
+	/** Equivalence table length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** AMD microcode magic signature */
+#define AMD_UCODE_MAGIC ( ( 'A' << 16 ) | ( 'M' << 8 ) | ( 'D' << 0 ) )
+
+/** AMD microcode equivalence table type */
+#define AMD_UCODE_EQUIV_TYPE 0x00000000UL
+
+/** An AMD microcode equivalence table entry */
+struct amd_ucode_equivalence {
+	/** CPU signature */
+	uint32_t signature;
+	/** Reserved */
+	uint8_t reserved_a[8];
+	/** Equivalence ID */
+	uint16_t id;
+	/** Reserved */
+	uint8_t reserved_b[2];
+} __attribute__ (( packed ));
+
+/** An AMD microcode patch header */
+struct amd_ucode_patch_header {
+	/** Patch type */
+	uint32_t type;
+	/** Patch length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** An AMD microcode patch */
+struct amd_ucode_patch {
+	/** Date */
+	struct ucode_date date;
+	/** Microcode version */
+	int32_t version;
+	/** Reserved */
+	uint8_t reserved_a[16];
+	/** Equivalence ID */
+	uint16_t id;
+	/** Reserved */
+	uint8_t reserved_b[14];
+} __attribute__ (( packed ));
+
+/** AMD patch type */
+#define AMD_UCODE_PATCH_TYPE 0x00000001UL
+
+extern mp_func_t ucode_update;
+
+#endif /* _IPXE_UCODE_H */
diff --git a/src/config/config.c b/src/config/config.c
index abb7d16a2..209336c25 100644
--- a/src/config/config.c
+++ b/src/config/config.c
@@ -188,6 +188,9 @@ REQUIRE_OBJECT ( zlib );
 #ifdef IMAGE_GZIP
 REQUIRE_OBJECT ( gzip );
 #endif
+#ifdef IMAGE_UCODE
+REQUIRE_OBJECT ( ucode );
+#endif
 
 /*
  * Drag in all requested commands
diff --git a/src/config/general.h b/src/config/general.h
index c9cdb3dd4..e883e072f 100644
--- a/src/config/general.h
+++ b/src/config/general.h
@@ -127,6 +127,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define	IMAGE_PEM		/* PEM image support */
 //#define	IMAGE_ZLIB		/* ZLIB image support */
 //#define	IMAGE_GZIP		/* GZIP image support */
+//#define	IMAGE_UCODE		/* Microcode update image support */
 
 /*
  * Command-line commands to include

From bac967d51a8851ef274da6aa7a3a9711de0bb056 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sat, 16 Mar 2024 23:25:07 +0000
Subject: [PATCH 109/237] [snp] Allocate additional padding for receive buffers

Some SNP implementations (observed with a wifi adapter in a Dell
Latitude 3440 laptop) seem to require additional space in the
allocated receive buffers, otherwise full-length packets will be
silently dropped.

The EDK2 MnpDxe driver happens to allocate an additional 8 bytes of
padding (4 for a VLAN tag, 4 for the Ethernet frame checksum).  Match
this behaviour since drivers are very likely to have been tested
against MnpDxe.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/efi/snpnet.c | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/drivers/net/efi/snpnet.c b/src/drivers/net/efi/snpnet.c
index 3b09d4918..c66aa7d28 100644
--- a/src/drivers/net/efi/snpnet.c
+++ b/src/drivers/net/efi/snpnet.c
@@ -71,6 +71,19 @@ struct snp_nic {
 /** Delay between each initialisation retry */
 #define SNP_INITIALIZE_RETRY_DELAY_MS 10
 
+/** Additional padding for receive buffers
+ *
+ * Some SNP implementations seem to require additional space in the
+ * allocated receive buffers, otherwise full-length packets will be
+ * silently dropped.
+ *
+ * The EDK2 MnpDxe driver happens to allocate an additional 8 bytes of
+ * padding (4 for a VLAN tag, 4 for the Ethernet frame checksum).
+ * Match this behaviour since drivers are very likely to have been
+ * tested against MnpDxe.
+ */
+#define SNP_RX_PAD 8
+
 /**
  * Format SNP MAC address (for debugging)
  *
@@ -246,7 +259,7 @@ static void snpnet_poll_rx ( struct net_device *netdev ) {
 
 		/* Allocate buffer, if required */
 		if ( ! snp->rxbuf ) {
-			snp->rxbuf = alloc_iob ( snp->mtu );
+			snp->rxbuf = alloc_iob ( snp->mtu + SNP_RX_PAD );
 			if ( ! snp->rxbuf ) {
 				/* Leave for next poll */
 				break;

From fa4bda617d5d2a714d0f8abfba451fd87f11a33e Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 17 Mar 2024 17:49:05 +0000
Subject: [PATCH 110/237] [build] Fix building on older versions of gcc

Older versions of gcc (observed with gcc 4.8.5 on CentOS 7) complain
about having the label "err_ioremap" at the end of a compound
statement in bios_mp_start_all().  The label is correctly placed,
since it immediately follows the iounmap() that would be required to
undo a successful ioremap() in the non-error case.

Fix by adding an explicit "return" immediately after the label.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/interface/pcbios/bios_mp.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/arch/x86/interface/pcbios/bios_mp.c b/src/arch/x86/interface/pcbios/bios_mp.c
index 914fe5c10..9e1179ccd 100644
--- a/src/arch/x86/interface/pcbios/bios_mp.c
+++ b/src/arch/x86/interface/pcbios/bios_mp.c
@@ -165,6 +165,7 @@ static void bios_mp_start_all ( mp_func_t func, void *opaque ) {
 	/* No way to handle errors: caller must check that
 	 * multiprocessor function executed as expected.
 	 */
+	return;
 }
 
 PROVIDE_MPAPI_INLINE ( pcbios, mp_address );

From c11734eee0fcaaf49f6f3f0342f928e1e5232560 Mon Sep 17 00:00:00 2001
From: Rabia Manaa <rabiam@nvidia.com>
Date: Sun, 17 Mar 2024 15:58:53 +0200
Subject: [PATCH 111/237] [golan] Use ETH_HLEN for inline header size

The driver does not correctly handle very short transmitted packets
such as EAPoL-Start where the entire DMA content lies within the
current send work queue entry inline header length of 18 bytes.

Fix by reducing the inline header length to the Ethernet frame header
length of 14 bytes.

Modified-by: Michael Brown <mcb30@ipxe.org>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/infiniband/golan.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/drivers/infiniband/golan.c b/src/drivers/infiniband/golan.c
index ce02a867f..68a7c4f5d 100755
--- a/src/drivers/infiniband/golan.c
+++ b/src/drivers/infiniband/golan.c
@@ -2502,7 +2502,7 @@ static mlx_status shomron_fill_eth_send_wqe ( struct ib_device *ibdev,
 	}
 
 #define SHOMRON_GENERATE_CQE 0x3
-#define SHOMRON_INLINE_HEADERS_SIZE 18
+#define SHOMRON_INLINE_HEADERS_SIZE ETH_HLEN
 #define SHOMRON_INLINE_HEADERS_OFFSET 32
 	MLX_FILL_2 ( &eth_wqe->ctrl, 0, opcode, FLEXBOOT_NODNIC_OPCODE_SEND,
 			wqe_index, wqe_index & 0xFFFF);

From 926816c58fca5641b17c17379b52203458081668 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 18 Mar 2024 15:21:04 +0000
Subject: [PATCH 112/237] [efi] Pad transmit buffer length to work around
 vendor driver bugs

The Mellanox/Nvidia UEFI driver is built from the same codebase as the
iPXE driver, and appears to contain the bug that was fixed in commit
c11734e ("[golan] Use ETH_HLEN for inline header size").  This results
in identical failures when using the SNP or NII interface (via
e.g. snponly.efi) to drive a Mellanox card while EAPoL is enabled.

Work around the underlying UEFI driver bug by padding transmit I/O
buffers to the minimum Ethernet frame length before passing them to
the underlying driver's transmit function.

This padding is not technically necessary, since almost all modern
hardware will insert transmit padding as necessary (and where the
hardware does not support doing so, the underlying UEFI driver is
responsible for adding any necessary padding).  However, it is
guaranteed to be harmless (other than a miniscule performance impact):
the Ethernet specification requires zero padding up to the minimum
frame length for packets that are transmitted onto the wire, and so
the receiver will see the same packet whether or not we manually
insert this padding in software.

The additional padding causes the underlying Mellanox driver to avoid
its faulty code path, since it will never be asked to transmit a very
short packet.

Tested-by: Eric Hagberg <ehagberg@janestreet.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/efi/nii.c    | 7 +++++++
 src/drivers/net/efi/snpnet.c | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/src/drivers/net/efi/nii.c b/src/drivers/net/efi/nii.c
index 8dd17e4b5..16e9e10df 100644
--- a/src/drivers/net/efi/nii.c
+++ b/src/drivers/net/efi/nii.c
@@ -30,6 +30,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <errno.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
 #include <ipxe/umalloc.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_driver.h>
@@ -998,6 +999,12 @@ static int nii_transmit ( struct net_device *netdev,
 		return 0;
 	}
 
+	/* Pad to minimum Ethernet length, to work around underlying
+	 * drivers that do not correctly handle frame padding
+	 * themselves.
+	 */
+	iob_pad ( iobuf, ETH_ZLEN );
+
 	/* Construct parameter block */
 	memset ( &cpb, 0, sizeof ( cpb ) );
 	cpb.FrameAddr = ( ( intptr_t ) iobuf->data );
diff --git a/src/drivers/net/efi/snpnet.c b/src/drivers/net/efi/snpnet.c
index c66aa7d28..b8bf963e9 100644
--- a/src/drivers/net/efi/snpnet.c
+++ b/src/drivers/net/efi/snpnet.c
@@ -26,6 +26,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/iobuf.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
 #include <ipxe/vsprintf.h>
 #include <ipxe/timer.h>
 #include <ipxe/efi/efi.h>
@@ -187,6 +188,12 @@ static int snpnet_transmit ( struct net_device *netdev,
 		return 0;
 	}
 
+	/* Pad to minimum Ethernet length, to work around underlying
+	 * drivers that do not correctly handle frame padding
+	 * themselves.
+	 */
+	iob_pad ( iobuf, ETH_ZLEN );
+
 	/* Transmit packet */
 	if ( ( efirc = snp->snp->Transmit ( snp->snp, 0, iob_len ( iobuf ),
 					    iobuf->data, NULL, NULL,

From 88c2a01e1aeb56335f893ac865d36f0fa843864c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 19 Mar 2024 13:22:57 +0000
Subject: [PATCH 113/237] [settings] Expose current working URI and directory
 URI via settings

iPXE maintains a concept of a current working URI, which is used when
resolving relative URIs and allows scripts to download files using
URIs relative to the script itself.

There are situations in which it is valuable for a script to be able
to access the URI explicitly as a string, not just implicitly as a
base URI for subsequent downloads.  For example, when booting a Fedora
installer, the "inst.repo" command-line parameter may be used to pass
the URI of the repository to the installer.

Expose the current working URI as ${cwuri}.  Since relative URIs may
be constructed as strings only from a directory URI (not from a full
URI), also expose the current working directory URI as ${cwduri}.

This feature may be used as e.g.

  #!ipxe
  echo Booting from ${cwuri}
  prompt -k 0x197e -t 2000 Press F12 to install Fedora... || exit
  kernel images/pxeboot/vmlinux inst.repo=${cwduri}
  initrd images/pxeboot/initrd.img
  boot

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/settings.c  | 107 +++++++++++++++++++++++++++++++++++++++++++
 src/tests/uri_test.c |  16 +++++++
 2 files changed, 123 insertions(+)

diff --git a/src/core/settings.c b/src/core/settings.c
index 4593876f0..9fbf753ab 100644
--- a/src/core/settings.c
+++ b/src/core/settings.c
@@ -2650,6 +2650,113 @@ struct builtin_setting unixtime_builtin_setting __builtin_setting = {
 	.fetch = unixtime_fetch,
 };
 
+/**
+ * Fetch current working URI-related setting
+ *
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @v rel		Relative URI string
+ * @ret len		Length of setting data, or negative error
+ */
+static int cwuri_fetch_uri ( void *data, size_t len, const char *rel ) {
+	struct uri *reluri;
+	struct uri *uri;
+	char *uristring;
+	int ret;
+
+	/* Check that current working URI is set */
+	if ( ! cwuri ) {
+		ret = -ENOENT;
+		goto err_unset;
+	}
+
+	/* Construct relative URI */
+	reluri = parse_uri ( rel );
+	if ( ! reluri ) {
+		ret = -ENOMEM;
+		goto err_parse;
+	}
+
+	/* Construct resolved URI */
+	uri = resolve_uri ( cwuri, reluri );
+	if ( ! uri ) {
+		ret = -ENOMEM;
+		goto err_resolve;
+	}
+
+	/* Format URI string into allocated buffer (with NUL) */
+	uristring = format_uri_alloc ( uri );
+	if ( ! uristring ) {
+		ret = -ENOMEM;
+		goto err_format;
+	}
+
+	/* Copy URI string to buffer */
+	strncpy ( data, uristring, len );
+	ret = strlen ( uristring );
+
+	free ( uristring );
+ err_format:
+	uri_put ( uri );
+ err_resolve:
+	uri_put ( reluri );
+ err_parse:
+ err_unset:
+	return ret;
+}
+
+/**
+ * Fetch current working URI setting
+ *
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int cwuri_fetch ( void *data, size_t len ) {
+
+	return cwuri_fetch_uri ( data, len, "" );
+}
+
+/**
+ * Fetch current working directory URI setting
+ *
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int cwduri_fetch ( void *data, size_t len ) {
+
+	return cwuri_fetch_uri ( data, len, "." );
+}
+
+/** Current working URI setting */
+const struct setting cwuri_setting __setting ( SETTING_MISC, cwuri ) = {
+	.name = "cwuri",
+	.description = "Current working URI",
+	.type = &setting_type_string,
+	.scope = &builtin_scope,
+};
+
+/** Current working directory URI setting */
+const struct setting cwduri_setting __setting ( SETTING_MISC, cwduri ) = {
+	.name = "cwduri",
+	.description = "Current working directory URI",
+	.type = &setting_type_string,
+	.scope = &builtin_scope,
+};
+
+/** Current working URI built-in setting */
+struct builtin_setting cwuri_builtin_setting __builtin_setting = {
+	.setting = &cwuri_setting,
+	.fetch = cwuri_fetch,
+};
+
+/** Current working directory URI built-in setting */
+struct builtin_setting cwduri_builtin_setting __builtin_setting = {
+	.setting = &cwduri_setting,
+	.fetch = cwduri_fetch,
+};
+
 /**
  * Fetch built-in setting
  *
diff --git a/src/tests/uri_test.c b/src/tests/uri_test.c
index 9d2f6dba5..7ce87a208 100644
--- a/src/tests/uri_test.c
+++ b/src/tests/uri_test.c
@@ -754,6 +754,20 @@ static struct uri_resolve_test uri_fragment = {
 	"http://192.168.0.254/test#bar",
 };
 
+/** Empty relative URI resolution test */
+static struct uri_resolve_test uri_self = {
+	"http://192.168.0.1/path/to/me",
+	"",
+	"http://192.168.0.1/path/to/me",
+};
+
+/** Current directory URI resolution test */
+static struct uri_resolve_test uri_cwd = {
+	"http://192.168.0.1/path/to/me",
+	".",
+	"http://192.168.0.1/path/to/",
+};
+
 /** PXE URI with absolute URI */
 static struct uri_pxe_test uri_pxe_absolute = {
 	{
@@ -996,6 +1010,8 @@ static void uri_test_exec ( void ) {
 	uri_resolve_ok ( &uri_absolute_uri_path );
 	uri_resolve_ok ( &uri_query );
 	uri_resolve_ok ( &uri_fragment );
+	uri_resolve_ok ( &uri_self );
+	uri_resolve_ok ( &uri_cwd );
 
 	/* PXE URI construction tests */
 	uri_pxe_ok ( &uri_pxe_absolute );

From 1a84facf12b07a5e5375822015b56cf320821055 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 19 Mar 2024 15:01:25 +0000
Subject: [PATCH 114/237] [efi] Add efi_path_uri() to parse a URI from an EFI
 device path

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/efi_path.h |  1 +
 src/interface/efi/efi_path.c    | 40 +++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/src/include/ipxe/efi/efi_path.h b/src/include/ipxe/efi/efi_path.h
index 20ff43f64..503bd4347 100644
--- a/src/include/ipxe/efi/efi_path.h
+++ b/src/include/ipxe/efi/efi_path.h
@@ -45,6 +45,7 @@ efi_path_end ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern size_t efi_path_len ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern unsigned int efi_path_vlan ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern int efi_path_guid ( EFI_DEVICE_PATH_PROTOCOL *path, union uuid *uuid );
+extern struct uri * efi_path_uri ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern EFI_DEVICE_PATH_PROTOCOL * efi_paths ( EFI_DEVICE_PATH_PROTOCOL *first,
 					      ... );
 extern EFI_DEVICE_PATH_PROTOCOL * efi_netdev_path ( struct net_device *netdev );
diff --git a/src/interface/efi/efi_path.c b/src/interface/efi/efi_path.c
index d1e22eeaa..4e37d248a 100644
--- a/src/interface/efi/efi_path.c
+++ b/src/interface/efi/efi_path.c
@@ -175,6 +175,46 @@ int efi_path_guid ( EFI_DEVICE_PATH_PROTOCOL *path, union uuid *guid ) {
 	return rc;
 }
 
+/**
+ * Parse URI from device path
+ *
+ * @v path		Device path
+ * @ret uri		URI, or NULL if not a URI
+ */
+struct uri * efi_path_uri ( EFI_DEVICE_PATH_PROTOCOL *path ) {
+	EFI_DEVICE_PATH_PROTOCOL *next;
+	URI_DEVICE_PATH *uripath;
+	char *uristring;
+	struct uri *uri;
+	size_t len;
+
+	/* Search for URI device path */
+	for ( ; ( next = efi_path_next ( path ) ) ; path = next ) {
+		if ( ( path->Type == MESSAGING_DEVICE_PATH ) &&
+		     ( path->SubType == MSG_URI_DP ) ) {
+
+			/* Calculate path length */
+			uripath = container_of ( path, URI_DEVICE_PATH,
+						 Header );
+			len = ( ( ( path->Length[1] << 8 ) | path->Length[0] )
+				- offsetof ( typeof ( *uripath ), Uri ) );
+
+			/* Parse URI */
+			uristring = zalloc ( len + 1 /* NUL */ );
+			if ( ! uristring )
+				return NULL;
+			memcpy ( uristring, uripath->Uri, len );
+			uri = parse_uri ( uristring );
+			free ( uristring );
+
+			return uri;
+		}
+	}
+
+	/* No URI path found */
+	return NULL;
+}
+
 /**
  * Concatenate EFI device paths
  *

From 390bce9516ce3a4adf599762b6c965813332595e Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 19 Mar 2024 15:13:59 +0000
Subject: [PATCH 115/237] [efi] Set current working URI from our own device
 path URI, if present

When booted via HTTP, our loaded image's device path will include the
URI from which we were downloaded.  Set this as the current working
URI, so that an embedded script may perform subsequent downloads
relative to the iPXE binary, or construct explicit relative paths via
the ${cwduri} setting.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efiprefix.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/interface/efi/efiprefix.c b/src/interface/efi/efiprefix.c
index 261160681..f6395b65b 100644
--- a/src/interface/efi/efiprefix.c
+++ b/src/interface/efi/efiprefix.c
@@ -22,6 +22,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <stdlib.h>
 #include <errno.h>
 #include <ipxe/device.h>
+#include <ipxe/uri.h>
 #include <ipxe/init.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_driver.h>
@@ -30,6 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/efi/efi_autoexec.h>
 #include <ipxe/efi/efi_cachedhcp.h>
 #include <ipxe/efi/efi_watchdog.h>
+#include <ipxe/efi/efi_path.h>
 #include <ipxe/efi/efi_veto.h>
 
 /**
@@ -80,6 +82,12 @@ static void efi_init_application ( void ) {
 	EFI_HANDLE device = efi_loaded_image->DeviceHandle;
 	EFI_DEVICE_PATH_PROTOCOL *devpath = efi_loaded_image_path;
 	EFI_DEVICE_PATH_PROTOCOL *filepath = efi_loaded_image->FilePath;
+	struct uri *uri;
+
+	/* Set current working URI from device path, if present */
+	uri = efi_path_uri ( devpath );
+	if ( uri )
+		churi ( uri );
 
 	/* Identify autoboot device, if any */
 	efi_set_autoboot_ll_addr ( device, devpath );
@@ -89,6 +97,9 @@ static void efi_init_application ( void ) {
 
 	/* Load autoexec script, if any */
 	efi_autoexec_load ( device, filepath );
+
+	/* Drop temporary reference to URI */
+	uri_put ( uri );
 }
 
 /** EFI application initialisation function */

From a15ce00182a8b2e0dfd43b81a3b2936cae339838 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 25 Mar 2024 16:24:24 +0000
Subject: [PATCH 116/237] [efi] Match chainloaded device by uppermost matching
 handle

Commit 4c5b794 ("[efi] Use the SNP protocol instance to match the SNP
chainloading device") switched the chainloaded device matching logic
to use a target protocol instance rather than the loaded image's
device handle, on the basis that we want to bind to the parent SNP
device rather than to a duplicate SNP protocol instance installed onto
an IPv4 or IPv6 child device handle.

It is possible that our calls to DisconnectController() and
ConnectController() will cause the target protocol instance to be
uninstalled and reinstalled, which may change the value of the
protocol instance pointer.  Allow for this by identifying and matching
against the uppermost handle that initially has this target protocol
instance installed.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/efi/snponly.c | 110 +++++++++++++++++++++-------------
 1 file changed, 69 insertions(+), 41 deletions(-)

diff --git a/src/drivers/net/efi/snponly.c b/src/drivers/net/efi/snponly.c
index 674e0a050..163704636 100644
--- a/src/drivers/net/efi/snponly.c
+++ b/src/drivers/net/efi/snponly.c
@@ -45,14 +45,23 @@ struct chained_protocol {
 	/** Protocol GUID */
 	EFI_GUID *protocol;
 	/**
-	 * Protocol instance installed on the loaded image's device handle
+	 * Target device handle
+	 *
+	 * This is the uppermost handle on which the same protocol
+	 * instance is installed as we find on the loaded image's
+	 * device handle.
 	 *
 	 * We match against the protocol instance (rather than simply
 	 * matching against the device handle itself) because some
 	 * systems load us via a child of the underlying device, with
 	 * a duplicate protocol installed on the child handle.
+	 *
+	 * We record the handle rather than the protocol instance
+	 * pointer since the calls to DisconnectController() and
+	 * ConnectController() may end up uninstalling and
+	 * reinstalling the protocol instance.
 	 */
-	void *interface;
+	EFI_HANDLE device;
 };
 
 /** Chainloaded SNP protocol */
@@ -66,49 +75,68 @@ static struct chained_protocol chained_nii = {
 };
 
 /**
- * Locate chainloaded protocol instance
+ * Locate chainloaded protocol
  *
  * @v chained		Chainloaded protocol
- * @ret rc		Return status code
  */
-static int chained_locate ( struct chained_protocol *chained ) {
+static void chained_locate ( struct chained_protocol *chained ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	EFI_HANDLE device = efi_loaded_image->DeviceHandle;
-	EFI_HANDLE parent;
+	EFI_HANDLE handle;
+	void *match = NULL;
+	void *interface;
+	unsigned int skip;
 	EFI_STATUS efirc;
 	int rc;
 
-	/* Locate handle supporting this protocol */
-	if ( ( rc = efi_locate_device ( device, chained->protocol,
-					&parent, 0 ) ) != 0 ) {
-		DBGC ( device, "CHAINED %s does not support %s: %s\n",
-		       efi_handle_name ( device ),
-		       efi_guid_ntoa ( chained->protocol ), strerror ( rc ) );
-		goto err_locate_device;
-	}
-	DBGC ( device, "CHAINED %s found %s on ", efi_handle_name ( device ),
-	       efi_guid_ntoa ( chained->protocol ) );
-	DBGC ( device, "%s\n", efi_handle_name ( parent ) );
+	/* Identify target device handle */
+	for ( skip = 0 ; ; skip++ ) {
 
-	/* Get protocol instance */
-	if ( ( efirc = bs->OpenProtocol ( parent, chained->protocol,
-					  &chained->interface, efi_image_handle,
-					  device,
-					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
-		rc = -EEFI ( efirc );
-		DBGC ( device, "CHAINED %s could not open %s on ",
+		/* Locate handle supporting this protocol */
+		if ( ( rc = efi_locate_device ( device, chained->protocol,
+						&handle, skip ) ) != 0 ) {
+			if ( skip == 0 ) {
+				DBGC ( device, "CHAINED %s does not support "
+				       "%s: %s\n", efi_handle_name ( device ),
+				       efi_guid_ntoa ( chained->protocol ),
+				       strerror ( rc ) );
+			}
+			break;
+		}
+
+		/* Get protocol instance */
+		if ( ( efirc = bs->OpenProtocol (
+					handle, chained->protocol, &interface,
+					efi_image_handle, handle,
+					EFI_OPEN_PROTOCOL_GET_PROTOCOL )) != 0){
+			rc = -EEFI ( efirc );
+			DBGC ( device, "CHAINED %s could not open %s on ",
+			       efi_handle_name ( device ),
+			       efi_guid_ntoa ( chained->protocol ) );
+			DBGC ( device, "%s: %s\n",
+			       efi_handle_name ( handle ), strerror ( rc ) );
+			break;
+		}
+		bs->CloseProtocol ( handle, chained->protocol,
+				    efi_image_handle, handle );
+
+		/* Stop if we reach a non-matching protocol instance */
+		if ( match && ( match != interface ) ) {
+			DBGC ( device, "CHAINED %s found non-matching %s on ",
+			       efi_handle_name ( device ),
+			       efi_guid_ntoa ( chained->protocol ) );
+			DBGC ( device, "%s\n", efi_handle_name ( handle ) );
+			break;
+		}
+
+		/* Record this handle */
+		chained->device = handle;
+		match = interface;
+		DBGC ( device, "CHAINED %s found %s on ",
 		       efi_handle_name ( device ),
 		       efi_guid_ntoa ( chained->protocol ) );
-		DBGC ( device, "%s: %s\n",
-		       efi_handle_name ( parent ), strerror ( rc ) );
-		goto err_open_protocol;
+		DBGC ( device, "%s\n", efi_handle_name ( chained->device ) );
 	}
-
- err_locate_device:
-	bs->CloseProtocol ( parent, chained->protocol, efi_image_handle,
-			    device );
- err_open_protocol:
-	return rc;
 }
 
 /**
@@ -121,8 +149,8 @@ static int chained_locate ( struct chained_protocol *chained ) {
 static int chained_supported ( EFI_HANDLE device,
 			       struct chained_protocol *chained ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	EFI_STATUS efirc;
 	void *interface;
+	EFI_STATUS efirc;
 	int rc;
 
 	/* Get protocol */
@@ -136,19 +164,19 @@ static int chained_supported ( EFI_HANDLE device,
 		goto err_open_protocol;
 	}
 
-	/* Test for a match against the chainloading device */
-	if ( interface != chained->interface ) {
-		DBGC ( device, "CHAINED %s %p is not the chainloaded %s\n",
-		       efi_handle_name ( device ), interface,
-		       efi_guid_ntoa ( chained->protocol ) );
+	/* Ignore non-matching handles */
+	if ( device != chained->device ) {
+		DBGC2 ( device, "CHAINED %s is not the chainloaded %s\n",
+			efi_handle_name ( device ),
+			efi_guid_ntoa ( chained->protocol ) );
 		rc = -ENOTTY;
 		goto err_no_match;
 	}
 
 	/* Success */
 	rc = 0;
-	DBGC ( device, "CHAINED %s %p is the chainloaded %s\n",
-	       efi_handle_name ( device ), interface,
+	DBGC ( device, "CHAINED %s is the chainloaded %s\n",
+	       efi_handle_name ( device ),
 	       efi_guid_ntoa ( chained->protocol ) );
 
  err_no_match:

From ca483a196c091c16ea0a426ce5f915b184a34412 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 20 Mar 2024 12:47:25 +0000
Subject: [PATCH 117/237] [efi] Add helper functions for service binding
 protocols

The EFI service binding abstraction is used to add and remove child
handles for multiple different protocols.  Provide a common interface
for doing so.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 .../ipxe/efi/Protocol/ServiceBinding.h        |  90 ++++++++++++
 src/include/ipxe/efi/efi_service.h            |  19 +++
 src/include/ipxe/errfile.h                    |   1 +
 src/interface/efi/efi_service.c               | 138 ++++++++++++++++++
 4 files changed, 248 insertions(+)
 create mode 100644 src/include/ipxe/efi/Protocol/ServiceBinding.h
 create mode 100644 src/include/ipxe/efi/efi_service.h
 create mode 100644 src/interface/efi/efi_service.c

diff --git a/src/include/ipxe/efi/Protocol/ServiceBinding.h b/src/include/ipxe/efi/Protocol/ServiceBinding.h
new file mode 100644
index 000000000..6baf73aa7
--- /dev/null
+++ b/src/include/ipxe/efi/Protocol/ServiceBinding.h
@@ -0,0 +1,90 @@
+/** @file
+  UEFI Service Binding Protocol is defined in UEFI specification.
+
+  The file defines the generic Service Binding Protocol functions.
+  It provides services that are required to create and destroy child
+  handles that support a given set of protocols.
+
+  Copyright (c) 2006 - 2018, Intel Corporation. All rights reserved.<BR>
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef __EFI_SERVICE_BINDING_H__
+#define __EFI_SERVICE_BINDING_H__
+
+FILE_LICENCE ( BSD2_PATENT );
+
+///
+/// Forward reference for pure ANSI compatability
+///
+typedef struct _EFI_SERVICE_BINDING_PROTOCOL EFI_SERVICE_BINDING_PROTOCOL;
+
+/**
+  Creates a child handle and installs a protocol.
+
+  The CreateChild() function installs a protocol on ChildHandle.
+  If ChildHandle is a pointer to NULL, then a new handle is created and returned in ChildHandle.
+  If ChildHandle is not a pointer to NULL, then the protocol installs on the existing ChildHandle.
+
+  @param  This        Pointer to the EFI_SERVICE_BINDING_PROTOCOL instance.
+  @param  ChildHandle Pointer to the handle of the child to create. If it is NULL,
+                      then a new handle is created. If it is a pointer to an existing UEFI handle,
+                      then the protocol is added to the existing UEFI handle.
+
+  @retval EFI_SUCCES            The protocol was added to ChildHandle.
+  @retval EFI_INVALID_PARAMETER ChildHandle is NULL.
+  @retval EFI_OUT_OF_RESOURCES  There are not enough resources available to create
+                                the child
+  @retval other                 The child handle was not created
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_SERVICE_BINDING_CREATE_CHILD)(
+  IN     EFI_SERVICE_BINDING_PROTOCOL  *This,
+  IN OUT EFI_HANDLE                    *ChildHandle
+  );
+
+/**
+  Destroys a child handle with a protocol installed on it.
+
+  The DestroyChild() function does the opposite of CreateChild(). It removes a protocol
+  that was installed by CreateChild() from ChildHandle. If the removed protocol is the
+  last protocol on ChildHandle, then ChildHandle is destroyed.
+
+  @param  This        Pointer to the EFI_SERVICE_BINDING_PROTOCOL instance.
+  @param  ChildHandle Handle of the child to destroy
+
+  @retval EFI_SUCCES            The protocol was removed from ChildHandle.
+  @retval EFI_UNSUPPORTED       ChildHandle does not support the protocol that is being removed.
+  @retval EFI_INVALID_PARAMETER Child handle is NULL.
+  @retval EFI_ACCESS_DENIED     The protocol could not be removed from the ChildHandle
+                                because its services are being used.
+  @retval other                 The child handle was not destroyed
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_SERVICE_BINDING_DESTROY_CHILD)(
+  IN EFI_SERVICE_BINDING_PROTOCOL          *This,
+  IN EFI_HANDLE                            ChildHandle
+  );
+
+///
+/// The EFI_SERVICE_BINDING_PROTOCOL provides member functions to create and destroy
+/// child handles. A driver is responsible for adding protocols to the child handle
+/// in CreateChild() and removing protocols in DestroyChild(). It is also required
+/// that the CreateChild() function opens the parent protocol BY_CHILD_CONTROLLER
+/// to establish the parent-child relationship, and closes the protocol in DestroyChild().
+/// The pseudo code for CreateChild() and DestroyChild() is provided to specify the
+/// required behavior, not to specify the required implementation. Each consumer of
+/// a software protocol is responsible for calling CreateChild() when it requires the
+/// protocol and calling DestroyChild() when it is finished with that protocol.
+///
+struct _EFI_SERVICE_BINDING_PROTOCOL {
+  EFI_SERVICE_BINDING_CREATE_CHILD     CreateChild;
+  EFI_SERVICE_BINDING_DESTROY_CHILD    DestroyChild;
+};
+
+#endif
diff --git a/src/include/ipxe/efi/efi_service.h b/src/include/ipxe/efi/efi_service.h
new file mode 100644
index 000000000..ca4c7b2a4
--- /dev/null
+++ b/src/include/ipxe/efi/efi_service.h
@@ -0,0 +1,19 @@
+#ifndef _IPXE_EFI_SERVICE_H
+#define _IPXE_EFI_SERVICE_H
+
+/** @file
+ *
+ * EFI service binding
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/efi/efi.h>
+
+extern int efi_service_add ( EFI_HANDLE service, EFI_GUID *binding,
+			     EFI_HANDLE *handle );
+extern int efi_service_del ( EFI_HANDLE service, EFI_GUID *binding,
+			     EFI_HANDLE handle );
+
+#endif /* _IPXE_EFI_SERVICE_H */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index c673b9a40..083c77e1f 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -82,6 +82,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_uuid		       ( ERRFILE_CORE | 0x002a0000 )
 #define ERRFILE_efi_path	       ( ERRFILE_CORE | 0x002b0000 )
 #define ERRFILE_efi_mp		       ( ERRFILE_CORE | 0x002c0000 )
+#define ERRFILE_efi_service	       ( ERRFILE_CORE | 0x002d0000 )
 
 #define ERRFILE_eisa		     ( ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_isa		     ( ERRFILE_DRIVER | 0x00010000 )
diff --git a/src/interface/efi/efi_service.c b/src/interface/efi/efi_service.c
new file mode 100644
index 000000000..d4129c0d9
--- /dev/null
+++ b/src/interface/efi/efi_service.c
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * EFI service binding
+ *
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/efi_service.h>
+#include <ipxe/efi/Protocol/ServiceBinding.h>
+
+/**
+ * Add service to child handle
+ *
+ * @v service		Service binding handle
+ * @v binding		Service binding protocol GUID
+ * @v handle		Handle on which to install child
+ * @ret rc		Return status code
+ */
+int efi_service_add ( EFI_HANDLE service, EFI_GUID *binding,
+		      EFI_HANDLE *handle ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	union {
+		EFI_SERVICE_BINDING_PROTOCOL *sb;
+		void *interface;
+	} u;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Open service binding protocol */
+	if ( ( efirc = bs->OpenProtocol ( service, binding, &u.interface,
+					  efi_image_handle, service,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( service, "EFISVC %s cannot open %s binding: %s\n",
+		       efi_handle_name ( service ), efi_guid_ntoa ( binding ),
+		       strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Create child handle */
+	if ( ( efirc = u.sb->CreateChild ( u.sb, handle ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( service, "EFISVC %s could not create %s child: %s\n",
+		       efi_handle_name ( service ), efi_guid_ntoa ( binding ),
+		       strerror ( rc ) );
+		goto err_create;
+	}
+
+	/* Success */
+	rc = 0;
+	DBGC ( service, "EFISVC %s created %s child ",
+	       efi_handle_name ( service ), efi_guid_ntoa ( binding ) );
+	DBGC ( service, "%s\n", efi_handle_name ( *handle ) );
+
+ err_create:
+	bs->CloseProtocol ( service, binding, efi_image_handle, service );
+ err_open:
+	return rc;
+}
+
+/**
+ * Remove service from child handle
+ *
+ * @v service		Service binding handle
+ * @v binding		Service binding protocol GUID
+ * @v handle		Child handle
+ * @ret rc		Return status code
+ */
+int efi_service_del ( EFI_HANDLE service, EFI_GUID *binding,
+		      EFI_HANDLE handle ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	union {
+		EFI_SERVICE_BINDING_PROTOCOL *sb;
+		void *interface;
+	} u;
+	EFI_STATUS efirc;
+	int rc;
+
+	DBGC ( service, "EFISVC %s removing %s child ",
+	       efi_handle_name ( service ), efi_guid_ntoa ( binding ) );
+	DBGC ( service, "%s\n", efi_handle_name ( handle ) );
+
+	/* Open service binding protocol */
+	if ( ( efirc = bs->OpenProtocol ( service, binding, &u.interface,
+					  efi_image_handle, service,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( service, "EFISVC %s cannot open %s binding: %s\n",
+		       efi_handle_name ( service ), efi_guid_ntoa ( binding ),
+		       strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Destroy child handle */
+	if ( ( efirc = u.sb->DestroyChild ( u.sb, handle ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( service, "EFISVC %s could not destroy %s child ",
+		       efi_handle_name ( service ), efi_guid_ntoa ( binding ) );
+		DBGC ( service, "%s: %s\n",
+		       efi_handle_name ( handle ), strerror ( rc ) );
+		goto err_destroy;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_destroy:
+	bs->CloseProtocol ( service, binding, efi_image_handle, service );
+ err_open:
+	return rc;
+}

From da5188f3ea73900f1c6a4e44a8345b48320d396f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 22 Mar 2024 16:50:13 +0000
Subject: [PATCH 118/237] [efi] Allow for drivers to be located via child
 handles

When using a service binding protocol, CreateChild() will create a new
protocol instance (and optionally a new handle).  The caller will then
typically open this new protocol instance with BY_DRIVER attributes,
since the service binding mechanism has no equivalent of the driver
binding protocol's Stop() method, and there is therefore no other way
for the caller to be informed if the protocol instance is about to
become invalid (e.g. because the service driver wants to remove the
child).

The caller cannot ask CreateChild() to install the new protocol
instance on the original handle (i.e. the service binding handle),
since the whole point of the service binding protocol is to allow for
the existence of multiple children, and UEFI does not permit multiple
instances of the same protocol to be installed on a handle.

Our current drivers all open the original handle (as passed to our
driver binding's Start() method) with BY_DRIVER attributes, and so the
same handle will be passed to our Stop() method.  This changes when
our driver must use a separate handle, as described above.

Add an optional "child handle" field to struct efi_device (on the
assumption that we will not have any drivers that need to create
multiple children), and generalise efidev_find() to match on either
the original handle or the child handle.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/efi_driver.h |  2 ++
 src/interface/efi/efi_driver.c    | 10 ++++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/include/ipxe/efi/efi_driver.h b/src/include/ipxe/efi/efi_driver.h
index 74ece90db..411e93642 100644
--- a/src/include/ipxe/efi/efi_driver.h
+++ b/src/include/ipxe/efi/efi_driver.h
@@ -19,6 +19,8 @@ struct efi_device {
 	struct device dev;
 	/** EFI device handle */
 	EFI_HANDLE device;
+	/** EFI child device handle (if present) */
+	EFI_HANDLE child;
 	/** EFI device path copy */
 	EFI_DEVICE_PATH_PROTOCOL *path;
 	/** Driver for this device */
diff --git a/src/interface/efi/efi_driver.c b/src/interface/efi/efi_driver.c
index 8e537d535..8f8c9f3da 100644
--- a/src/interface/efi/efi_driver.c
+++ b/src/interface/efi/efi_driver.c
@@ -64,16 +64,22 @@ static int efi_driver_disconnecting;
 /**
  * Find EFI device
  *
- * @v device		EFI device handle
+ * @v device		EFI device handle (or child handle)
  * @ret efidev		EFI device, or NULL if not found
  */
 static struct efi_device * efidev_find ( EFI_HANDLE device ) {
 	struct efi_device *efidev;
 
+	/* Avoid false positive matches against NULL children */
+	if ( ! device )
+		return NULL;
+
 	/* Look for an existing EFI device */
 	list_for_each_entry ( efidev, &efi_devices, dev.siblings ) {
-		if ( efidev->device == device )
+		if ( ( device == efidev->device ) ||
+		     ( device == efidev->child ) ) {
 			return efidev;
+		}
 	}
 
 	return NULL;

From dcad73ca5ad3e1fe011c52a24036f67ad69fadc1 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 22 Mar 2024 15:30:45 +0000
Subject: [PATCH 119/237] [efi] Add support for driving
 EFI_MANAGED_NETWORK_PROTOCOL devices

We want exclusive access to the network device, both for performance
reasons and because we perform operations such as EAPoL that affect
the entire link.  We currently drive the network card via either a
native hardware driver or via the SNP or NII/UNDI interfaces, both of
which grant us this exclusive access.

Add an alternative driver that drives the network card non-exclusively
via the EFI_MANAGED_NETWORK_PROTOCOL interface.  This can function as
a fallback for situations where neither SNP nor NII/UNDI interfaces
are functional, and also opens up the possibility of non-destructively
installing a temporary network device over which to download the
autoexec.ipxe script.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/Makefile.efi              |   4 +-
 src/drivers/net/efi/mnp.c     |  56 ++++
 src/drivers/net/efi/mnpnet.c  | 504 ++++++++++++++++++++++++++++++++++
 src/drivers/net/efi/mnpnet.h  |  17 ++
 src/drivers/net/efi/snp.c     |  54 +---
 src/drivers/net/efi/snpnet.c  |  48 ++++
 src/drivers/net/efi/snpnet.h  |   1 +
 src/drivers/net/efi/snponly.c |  26 ++
 src/include/ipxe/errfile.h    |   1 +
 9 files changed, 657 insertions(+), 54 deletions(-)
 create mode 100644 src/drivers/net/efi/mnp.c
 create mode 100644 src/drivers/net/efi/mnpnet.c
 create mode 100644 src/drivers/net/efi/mnpnet.h

diff --git a/src/Makefile.efi b/src/Makefile.efi
index 6e8ad46bc..95ecf3863 100644
--- a/src/Makefile.efi
+++ b/src/Makefile.efi
@@ -23,9 +23,9 @@ NON_AUTO_MEDIA	+= efidrv
 NON_AUTO_MEDIA	+= drv.efi
 NON_AUTO_MEDIA	+= efirom
 
-# Include SNP driver in the all-drivers build
+# Include SNP and MNP drivers in the all-drivers build
 #
-DRIVERS_net += snp
+DRIVERS_net += snp mnp
 
 # Rules for building EFI files
 #
diff --git a/src/drivers/net/efi/mnp.c b/src/drivers/net/efi/mnp.c
new file mode 100644
index 000000000..b79fe188a
--- /dev/null
+++ b/src/drivers/net/efi/mnp.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * MNP driver
+ *
+ */
+
+#include <errno.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/efi_driver.h>
+#include "snpnet.h"
+#include "mnpnet.h"
+
+/**
+ * Check to see if driver supports a device
+ *
+ * @v device		EFI device handle
+ * @ret rc		Return status code
+ */
+static int mnp_supported ( EFI_HANDLE device ) {
+	EFI_GUID *binding = &efi_managed_network_service_binding_protocol_guid;
+
+	return snpnet_supported ( device, binding );
+}
+
+/** EFI MNP driver */
+struct efi_driver mnp_driver __efi_driver ( EFI_DRIVER_NORMAL ) = {
+	.name = "MNP",
+	.supported = mnp_supported,
+	.start = mnpnet_start,
+	.stop = mnpnet_stop,
+};
diff --git a/src/drivers/net/efi/mnpnet.c b/src/drivers/net/efi/mnpnet.c
new file mode 100644
index 000000000..a07eae54e
--- /dev/null
+++ b/src/drivers/net/efi/mnpnet.c
@@ -0,0 +1,504 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * MNP NIC driver
+ *
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/efi_driver.h>
+#include <ipxe/efi/efi_service.h>
+#include <ipxe/efi/efi_utils.h>
+#include <ipxe/efi/Protocol/ManagedNetwork.h>
+#include "mnpnet.h"
+
+/** An MNP transmit or receive token */
+struct mnp_token {
+	/** MNP completion token */
+	EFI_MANAGED_NETWORK_COMPLETION_TOKEN token;
+	/** Token is owned by MNP */
+	int busy;
+};
+
+/** An MNP NIC */
+struct mnp_nic {
+	/** EFI device */
+	struct efi_device *efidev;
+	/** Managed network protocol */
+	EFI_MANAGED_NETWORK_PROTOCOL *mnp;
+	/** Generic device */
+	struct device dev;
+
+	/** Transmit token */
+	struct mnp_token tx;
+	/** Transmit descriptor */
+	EFI_MANAGED_NETWORK_TRANSMIT_DATA txdata;
+	/** Transmit I/O buffer */
+	struct io_buffer *txbuf;
+
+	/** Receive token */
+	struct mnp_token rx;
+};
+
+/**
+ * Transmit or receive token event
+ *
+ * @v event		Event
+ * @v context		Event context
+ */
+static VOID EFIAPI mnpnet_event ( EFI_EVENT event __unused, VOID *context ) {
+	struct mnp_token *token = context;
+
+	/* Sanity check */
+	assert ( token->busy );
+
+	/* Mark token as no longer owned by MNP */
+	token->busy = 0;
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int mnpnet_transmit ( struct net_device *netdev,
+			     struct io_buffer *iobuf ) {
+	struct mnp_nic *mnp = netdev->priv;
+	struct ll_protocol *ll_protocol = netdev->ll_protocol;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Do nothing if shutdown is in progress */
+	if ( efi_shutdown_in_progress )
+		return -ECANCELED;
+
+	/* Defer the packet if there is already a transmission in progress */
+	if ( mnp->txbuf ) {
+		netdev_tx_defer ( netdev, iobuf );
+		return 0;
+	}
+
+	/* Construct transmit token */
+	mnp->txdata.DataLength =
+		( iob_len ( iobuf ) - ll_protocol->ll_header_len );
+	mnp->txdata.HeaderLength = ll_protocol->ll_header_len;
+	mnp->txdata.FragmentCount = 1;
+	mnp->txdata.FragmentTable[0].FragmentLength = iob_len ( iobuf );
+	mnp->txdata.FragmentTable[0].FragmentBuffer = iobuf->data;
+	mnp->tx.token.Packet.TxData = &mnp->txdata;
+
+	/* Record as in use */
+	mnp->tx.busy = 1;
+
+	/* Transmit packet */
+	if ( ( efirc = mnp->mnp->Transmit ( mnp->mnp, &mnp->tx.token ) ) != 0 ){
+		rc = -EEFI ( efirc );
+		DBGC ( mnp, "MNP %s could not transmit: %s\n",
+		       netdev->name, strerror ( rc ) );
+		mnp->tx.busy = 0;
+		return rc;
+	}
+
+	/* Record I/O buffer */
+	mnp->txbuf = iobuf;
+
+	return 0;
+}
+
+/**
+ * Refill receive token
+ *
+ * @v netdev		Network device
+ */
+static void mnpnet_refill_rx ( struct net_device *netdev ) {
+	struct mnp_nic *mnp = netdev->priv;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Do nothing if receive token is still in use */
+	if ( mnp->rx.busy )
+		return;
+
+	/* Mark as in use */
+	mnp->rx.busy = 1;
+
+	/* Queue receive token */
+	if ( ( efirc = mnp->mnp->Receive ( mnp->mnp, &mnp->rx.token ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( mnp, "MNP %s could not receive: %s\n",
+		       netdev->name, strerror ( rc ) );
+		/* Wait for next refill */
+		mnp->rx.busy = 0;
+		return;
+	}
+}
+
+/**
+ * Poll for completed packets
+ *
+ * @v netdev		Network device
+ */
+static void mnpnet_poll_tx ( struct net_device *netdev ) {
+	struct mnp_nic *mnp = netdev->priv;
+	struct io_buffer *iobuf;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Do nothing if transmit token is still in use */
+	if ( mnp->tx.busy )
+		return;
+
+	/* Do nothing unless we have a completion */
+	if ( ! mnp->txbuf )
+		return;
+
+	/* Get completion status */
+	efirc = mnp->tx.token.Status;
+	rc = ( efirc ? -EEFI ( efirc ) : 0 );
+
+	/* Complete transmission */
+	iobuf = mnp->txbuf;
+	mnp->txbuf = NULL;
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/**
+ * Poll for received packets
+ *
+ * @v netdev		Network device
+ */
+static void mnpnet_poll_rx ( struct net_device *netdev ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	struct mnp_nic *mnp = netdev->priv;
+	EFI_MANAGED_NETWORK_RECEIVE_DATA *rxdata;
+	struct io_buffer *iobuf;
+	size_t len;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Do nothing unless we have a completion */
+	if ( mnp->rx.busy )
+		return;
+	rxdata = mnp->rx.token.Packet.RxData;
+
+	/* Get completion status */
+	if ( ( efirc = mnp->rx.token.Status ) != 0 ) {
+		rc = -EEFI ( efirc );
+		netdev_rx_err ( netdev, NULL, rc );
+		goto recycle;
+	}
+
+	/* Allocate and fill I/O buffer */
+	len = rxdata->PacketLength;
+	iobuf = alloc_iob ( len );
+	if ( ! iobuf ) {
+		netdev_rx_err ( netdev, NULL, -ENOMEM );
+		goto recycle;
+	}
+	memcpy ( iob_put ( iobuf, len ), rxdata->MediaHeader, len );
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iobuf );
+
+ recycle:
+	/* Recycle token */
+	bs->SignalEvent ( rxdata->RecycleEvent );
+}
+
+/**
+ * Poll for completed packets
+ *
+ * @v netdev		Network device
+ */
+static void mnpnet_poll ( struct net_device *netdev ) {
+	struct mnp_nic *mnp = netdev->priv;
+
+	/* Do nothing if shutdown is in progress */
+	if ( efi_shutdown_in_progress )
+		return;
+
+	/* Poll interface */
+	mnp->mnp->Poll ( mnp->mnp );
+
+	/* Process any transmit completions */
+	mnpnet_poll_tx ( netdev );
+
+	/* Process any receive completions */
+	mnpnet_poll_rx ( netdev );
+
+	/* Refill receive token */
+	mnpnet_refill_rx ( netdev );
+}
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int mnpnet_open ( struct net_device *netdev ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	static EFI_MANAGED_NETWORK_CONFIG_DATA config = {
+		.EnableUnicastReceive = TRUE,
+		.EnableMulticastReceive = TRUE,
+		.EnableBroadcastReceive = TRUE,
+		.EnablePromiscuousReceive = TRUE,
+		.FlushQueuesOnReset = TRUE,
+		.DisableBackgroundPolling = TRUE,
+	};
+	struct mnp_nic *mnp = netdev->priv;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Create transmit event */
+	if ( ( efirc = bs->CreateEvent ( EVT_NOTIFY_SIGNAL, TPL_NOTIFY,
+					 mnpnet_event, &mnp->tx,
+					 &mnp->tx.token.Event ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( mnp, "MNP %s could not create TX event: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_tx_event;
+	}
+
+	/* Create receive event */
+	if ( ( efirc = bs->CreateEvent ( EVT_NOTIFY_SIGNAL, TPL_NOTIFY,
+					 mnpnet_event, &mnp->rx,
+					 &mnp->rx.token.Event ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( mnp, "MNP %s could not create RX event: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_rx_event;
+	}
+
+	/* Configure MNP */
+	if ( ( efirc = mnp->mnp->Configure ( mnp->mnp, &config ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( mnp, "MNP %s could not configure: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_configure;
+	}
+
+	/* Refill receive token */
+	mnpnet_refill_rx ( netdev );
+
+	return 0;
+
+	mnp->mnp->Configure ( mnp->mnp, NULL );
+ err_configure:
+	bs->CloseEvent ( mnp->rx.token.Event );
+ err_rx_event:
+	bs->CloseEvent ( mnp->tx.token.Event );
+ err_tx_event:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void mnpnet_close ( struct net_device *netdev ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	struct mnp_nic *mnp = netdev->priv;
+
+	/* Reset MNP (unless whole system shutdown is in progress) */
+	if ( ! efi_shutdown_in_progress )
+		mnp->mnp->Configure ( mnp->mnp, NULL );
+
+	/* Close events */
+	bs->CloseEvent ( mnp->rx.token.Event );
+	bs->CloseEvent ( mnp->tx.token.Event );
+
+	/* Reset tokens */
+	mnp->tx.busy = 0;
+	mnp->rx.busy = 0;
+
+	/* Discard any incomplete I/O buffer */
+	if ( mnp->txbuf ) {
+		netdev_tx_complete_err ( netdev, mnp->txbuf, -ECANCELED );
+		mnp->txbuf = NULL;
+	}
+}
+
+/** MNP network device operations */
+static struct net_device_operations mnpnet_operations = {
+	.open = mnpnet_open,
+	.close = mnpnet_close,
+	.transmit = mnpnet_transmit,
+	.poll = mnpnet_poll,
+};
+
+/**
+ * Attach driver to device
+ *
+ * @v efidev		EFI device
+ * @ret rc		Return status code
+ */
+int mnpnet_start ( struct efi_device *efidev ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_HANDLE device = efidev->device;
+	EFI_GUID *binding = &efi_managed_network_service_binding_protocol_guid;
+	EFI_SIMPLE_NETWORK_MODE mode;
+	union {
+		EFI_MANAGED_NETWORK_PROTOCOL *mnp;
+		void *interface;
+	} u;
+	struct net_device *netdev;
+	struct mnp_nic *mnp;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Allocate and initalise structure */
+	netdev = alloc_etherdev ( sizeof ( *mnp ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &mnpnet_operations );
+	mnp = netdev->priv;
+	mnp->efidev = efidev;
+	efidev_set_drvdata ( efidev, netdev );
+
+	/* Populate underlying device information */
+	efi_device_info ( device, "MNP", &mnp->dev );
+	mnp->dev.driver_name = "MNP";
+	mnp->dev.parent = &efidev->dev;
+	list_add ( &mnp->dev.siblings, &efidev->dev.children );
+	INIT_LIST_HEAD ( &mnp->dev.children );
+	netdev->dev = &mnp->dev;
+
+	/* Create MNP child */
+	if ( ( rc = efi_service_add ( device, binding,
+				      &efidev->child ) ) != 0 ) {
+		DBGC ( mnp, "MNP %s could not create child: %s\n",
+		       efi_handle_name ( device ), strerror ( rc ) );
+		goto err_service;
+	}
+
+	/* Open MNP protocol */
+	if ( ( efirc = bs->OpenProtocol ( efidev->child,
+					  &efi_managed_network_protocol_guid,
+					  &u.interface, efi_image_handle,
+					  efidev->child,
+					  ( EFI_OPEN_PROTOCOL_BY_DRIVER |
+					    EFI_OPEN_PROTOCOL_EXCLUSIVE )))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( mnp, "MNP %s could not open MNP protocol: %s\n",
+		       efi_handle_name ( device ), strerror ( rc ) );
+		goto err_open;
+	}
+	mnp->mnp = u.mnp;
+
+	/* Get configuration */
+	efirc = mnp->mnp->GetModeData ( mnp->mnp, NULL, &mode );
+	if ( ( efirc != 0 ) && ( efirc != EFI_NOT_STARTED ) ) {
+		rc = -EEFI ( efirc );
+		DBGC ( mnp, "MNP %s could not get mode data: %s\n",
+		       efi_handle_name ( device ), strerror ( rc ) );
+		goto err_mode;
+	}
+
+	/* Populate network device parameters */
+	if ( mode.HwAddressSize != netdev->ll_protocol->hw_addr_len ) {
+		DBGC ( device, "MNP %s has invalid hardware address length "
+		       "%d\n", efi_handle_name ( device ), mode.HwAddressSize );
+		rc = -ENOTSUP;
+		goto err_hw_addr_len;
+	}
+	memcpy ( netdev->hw_addr, &mode.PermanentAddress,
+		 netdev->ll_protocol->hw_addr_len );
+	if ( mode.HwAddressSize != netdev->ll_protocol->ll_addr_len ) {
+		DBGC ( device, "MNP %s has invalid link-layer address length "
+		       "%d\n", efi_handle_name ( device ), mode.HwAddressSize );
+		rc = -ENOTSUP;
+		goto err_ll_addr_len;
+	}
+	memcpy ( netdev->ll_addr, &mode.CurrentAddress,
+		 netdev->ll_protocol->ll_addr_len );
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+	DBGC ( mnp, "MNP %s registered as %s\n",
+	       efi_handle_name ( device ), netdev->name );
+
+	/* Mark as link up: we don't handle link state */
+	netdev_link_up ( netdev );
+
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_ll_addr_len:
+ err_hw_addr_len:
+ err_mode:
+	bs->CloseProtocol ( efidev->child, &efi_managed_network_protocol_guid,
+			    efi_image_handle, efidev->child );
+ err_open:
+	efi_service_del ( device, binding, efidev->child );
+ err_service:
+	list_del ( &mnp->dev.siblings );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Detach driver from device
+ *
+ * @v efidev		EFI device
+  */
+void mnpnet_stop ( struct efi_device *efidev ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_GUID *binding = &efi_managed_network_service_binding_protocol_guid;
+	struct net_device *netdev = efidev_get_drvdata ( efidev );
+	struct mnp_nic *mnp = netdev->priv;
+
+	/* Unregister network device */
+	unregister_netdev ( netdev );
+
+	/* Close MNP protocol */
+	bs->CloseProtocol ( efidev->child, &efi_managed_network_protocol_guid,
+			    efi_image_handle, efidev->child );
+
+	/* Remove MNP child (unless whole system shutdown is in progress) */
+	if ( ! efi_shutdown_in_progress )
+		efi_service_del ( efidev->device, binding, efidev->child );
+
+	/* Free network device */
+	list_del ( &mnp->dev.siblings );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
diff --git a/src/drivers/net/efi/mnpnet.h b/src/drivers/net/efi/mnpnet.h
new file mode 100644
index 000000000..afed62aae
--- /dev/null
+++ b/src/drivers/net/efi/mnpnet.h
@@ -0,0 +1,17 @@
+#ifndef _MNPNET_H
+#define _MNPNET_H
+
+/** @file
+ *
+ * MNP NIC driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+struct efi_device;
+
+extern int mnpnet_start ( struct efi_device *efidev );
+extern void mnpnet_stop ( struct efi_device *efidev );
+
+#endif /* _MNPNET_H */
diff --git a/src/drivers/net/efi/snp.c b/src/drivers/net/efi/snp.c
index 1920cdbc5..cac8b38e2 100644
--- a/src/drivers/net/efi/snp.c
+++ b/src/drivers/net/efi/snp.c
@@ -23,11 +23,8 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#include <errno.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_driver.h>
-#include <ipxe/efi/efi_snp.h>
-#include <ipxe/efi/efi_utils.h>
 #include "snpnet.h"
 #include "nii.h"
 
@@ -37,53 +34,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
-/**
- * Check to see if driver supports a device
- *
- * @v device		EFI device handle
- * @v protocol		Protocol GUID
- * @ret rc		Return status code
- */
-static int snp_nii_supported ( EFI_HANDLE device, EFI_GUID *protocol ) {
-	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	EFI_HANDLE parent;
-	EFI_STATUS efirc;
-	int rc;
-
-	/* Check that this is not a device we are providing ourselves */
-	if ( find_snpdev ( device ) != NULL ) {
-		DBGCP ( device, "HANDLE %s is provided by this binary\n",
-			efi_handle_name ( device ) );
-		return -ENOTTY;
-	}
-
-	/* Test for presence of protocol */
-	if ( ( efirc = bs->OpenProtocol ( device, protocol,
-					  NULL, efi_image_handle, device,
-					  EFI_OPEN_PROTOCOL_TEST_PROTOCOL))!=0){
-		DBGCP ( device, "HANDLE %s is not a %s device\n",
-			efi_handle_name ( device ),
-			efi_guid_ntoa ( protocol ) );
-		return -EEFI ( efirc );
-	}
-
-	/* Check that there are no instances of this protocol further
-	 * up this device path.
-	 */
-	if ( ( rc = efi_locate_device ( device, protocol,
-					&parent, 1 ) ) == 0 ) {
-		DBGC2 ( device, "HANDLE %s has %s-supporting parent ",
-			efi_handle_name ( device ),
-			efi_guid_ntoa ( protocol ) );
-		DBGC2 ( device, "%s\n", efi_handle_name ( parent ) );
-		return -ENOTTY;
-	}
-
-	DBGC ( device, "HANDLE %s is a %s device\n",
-	       efi_handle_name ( device ), efi_guid_ntoa ( protocol ) );
-	return 0;
-}
-
 /**
  * Check to see if driver supports a device
  *
@@ -92,7 +42,7 @@ static int snp_nii_supported ( EFI_HANDLE device, EFI_GUID *protocol ) {
  */
 static int snp_supported ( EFI_HANDLE device ) {
 
-	return snp_nii_supported ( device, &efi_simple_network_protocol_guid );
+	return snpnet_supported ( device, &efi_simple_network_protocol_guid );
 }
 
 /**
@@ -103,7 +53,7 @@ static int snp_supported ( EFI_HANDLE device ) {
  */
 static int nii_supported ( EFI_HANDLE device ) {
 
-	return snp_nii_supported ( device, &efi_nii31_protocol_guid );
+	return snpnet_supported ( device, &efi_nii31_protocol_guid );
 }
 
 /** EFI SNP driver */
diff --git a/src/drivers/net/efi/snpnet.c b/src/drivers/net/efi/snpnet.c
index b8bf963e9..6ce731d78 100644
--- a/src/drivers/net/efi/snpnet.c
+++ b/src/drivers/net/efi/snpnet.c
@@ -33,6 +33,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/efi/Protocol/SimpleNetwork.h>
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_utils.h>
+#include <ipxe/efi/efi_snp.h>
 #include "snpnet.h"
 
 /** @file
@@ -484,6 +485,53 @@ static struct net_device_operations snpnet_operations = {
 	.poll = snpnet_poll,
 };
 
+/**
+ * Check to see if driver supports a device
+ *
+ * @v device		EFI device handle
+ * @v protocol		Protocol GUID
+ * @ret rc		Return status code
+ */
+int snpnet_supported ( EFI_HANDLE device, EFI_GUID *protocol ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_HANDLE parent;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Check that this is not a device we are providing ourselves */
+	if ( find_snpdev ( device ) != NULL ) {
+		DBGCP ( device, "HANDLE %s is provided by this binary\n",
+			efi_handle_name ( device ) );
+		return -ENOTTY;
+	}
+
+	/* Test for presence of protocol */
+	if ( ( efirc = bs->OpenProtocol ( device, protocol,
+					  NULL, efi_image_handle, device,
+					  EFI_OPEN_PROTOCOL_TEST_PROTOCOL))!=0){
+		DBGCP ( device, "HANDLE %s is not a %s device\n",
+			efi_handle_name ( device ),
+			efi_guid_ntoa ( protocol ) );
+		return -EEFI ( efirc );
+	}
+
+	/* Check that there are no instances of this protocol further
+	 * up this device path.
+	 */
+	if ( ( rc = efi_locate_device ( device, protocol,
+					&parent, 1 ) ) == 0 ) {
+		DBGC2 ( device, "HANDLE %s has %s-supporting parent ",
+			efi_handle_name ( device ),
+			efi_guid_ntoa ( protocol ) );
+		DBGC2 ( device, "%s\n", efi_handle_name ( parent ) );
+		return -ENOTTY;
+	}
+
+	DBGC ( device, "HANDLE %s is a %s device\n",
+	       efi_handle_name ( device ), efi_guid_ntoa ( protocol ) );
+	return 0;
+}
+
 /**
  * Attach driver to device
  *
diff --git a/src/drivers/net/efi/snpnet.h b/src/drivers/net/efi/snpnet.h
index e6d31d5e4..4699c7892 100644
--- a/src/drivers/net/efi/snpnet.h
+++ b/src/drivers/net/efi/snpnet.h
@@ -11,6 +11,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 struct efi_device;
 
+extern int snpnet_supported ( EFI_HANDLE device, EFI_GUID *protocol );
 extern int snpnet_start ( struct efi_device *efidev );
 extern void snpnet_stop ( struct efi_device *efidev );
 
diff --git a/src/drivers/net/efi/snponly.c b/src/drivers/net/efi/snponly.c
index 163704636..6786f3e83 100644
--- a/src/drivers/net/efi/snponly.c
+++ b/src/drivers/net/efi/snponly.c
@@ -32,6 +32,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/efi/Protocol/SimpleNetwork.h>
 #include <ipxe/efi/Protocol/NetworkInterfaceIdentifier.h>
 #include "snpnet.h"
+#include "mnpnet.h"
 #include "nii.h"
 
 /** @file
@@ -74,6 +75,11 @@ static struct chained_protocol chained_nii = {
 	.protocol = &efi_nii31_protocol_guid,
 };
 
+/** Chainloaded MNP protocol */
+static struct chained_protocol chained_mnp = {
+	.protocol = &efi_managed_network_service_binding_protocol_guid,
+};
+
 /**
  * Locate chainloaded protocol
  *
@@ -208,6 +214,17 @@ static int niionly_supported ( EFI_HANDLE device ) {
 	return chained_supported ( device, &chained_nii );
 }
 
+/**
+ * Check to see if driver supports a device
+ *
+ * @v device		EFI device handle
+ * @ret rc		Return status code
+ */
+static int mnponly_supported ( EFI_HANDLE device ) {
+
+	return chained_supported ( device, &chained_mnp );
+}
+
 /** EFI SNP chainloading-device-only driver */
 struct efi_driver snponly_driver __efi_driver ( EFI_DRIVER_NORMAL ) = {
 	.name = "SNPONLY",
@@ -224,6 +241,14 @@ struct efi_driver niionly_driver __efi_driver ( EFI_DRIVER_NORMAL ) = {
 	.stop = nii_stop,
 };
 
+/** EFI MNP chainloading-device-only driver */
+struct efi_driver mnponly_driver __efi_driver ( EFI_DRIVER_NORMAL ) = {
+	.name = "MNPONLY",
+	.supported = mnponly_supported,
+	.start = mnpnet_start,
+	.stop = mnpnet_stop,
+};
+
 /**
  * Initialise EFI chainloaded-device-only driver
  *
@@ -232,6 +257,7 @@ static void chained_init ( void ) {
 
 	chained_locate ( &chained_snp );
 	chained_locate ( &chained_nii );
+	chained_locate ( &chained_mnp );
 }
 
 /** EFI chainloaded-device-only initialisation function */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 083c77e1f..0ab37230a 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -223,6 +223,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_ice		     ( ERRFILE_DRIVER | 0x00d20000 )
 #define ERRFILE_ecam		     ( ERRFILE_DRIVER | 0x00d30000 )
 #define ERRFILE_pcibridge	     ( ERRFILE_DRIVER | 0x00d40000 )
+#define ERRFILE_mnpnet		     ( ERRFILE_DRIVER | 0x00d50000 )
 
 #define ERRFILE_aoe			( ERRFILE_NET | 0x00000000 )
 #define ERRFILE_arp			( ERRFILE_NET | 0x00010000 )

From 170bbfd4875b2a3479101b5f5892cdedcf857fd0 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 26 Mar 2024 15:16:33 +0000
Subject: [PATCH 120/237] [efi] Add efi_path_mac() to parse a MAC address from
 an EFI device path

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/efi_path.h |  1 +
 src/interface/efi/efi_path.c    | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/src/include/ipxe/efi/efi_path.h b/src/include/ipxe/efi/efi_path.h
index 503bd4347..57fce4028 100644
--- a/src/include/ipxe/efi/efi_path.h
+++ b/src/include/ipxe/efi/efi_path.h
@@ -43,6 +43,7 @@ efi_path_prev ( EFI_DEVICE_PATH_PROTOCOL *path,
 extern EFI_DEVICE_PATH_PROTOCOL *
 efi_path_end ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern size_t efi_path_len ( EFI_DEVICE_PATH_PROTOCOL *path );
+extern void * efi_path_mac ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern unsigned int efi_path_vlan ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern int efi_path_guid ( EFI_DEVICE_PATH_PROTOCOL *path, union uuid *uuid );
 extern struct uri * efi_path_uri ( EFI_DEVICE_PATH_PROTOCOL *path );
diff --git a/src/interface/efi/efi_path.c b/src/interface/efi/efi_path.c
index 4e37d248a..23f1bb84d 100644
--- a/src/interface/efi/efi_path.c
+++ b/src/interface/efi/efi_path.c
@@ -111,6 +111,30 @@ size_t efi_path_len ( EFI_DEVICE_PATH_PROTOCOL *path ) {
 	return ( ( ( void * ) end ) - ( ( void * ) path ) );
 }
 
+/**
+ * Get MAC address from device path
+ *
+ * @v path		Device path
+ * @ret mac		MAC address, or NULL if not found
+ */
+void * efi_path_mac ( EFI_DEVICE_PATH_PROTOCOL *path ) {
+	EFI_DEVICE_PATH_PROTOCOL *next;
+	MAC_ADDR_DEVICE_PATH *mac;
+
+	/* Search for MAC address path */
+	for ( ; ( next = efi_path_next ( path ) ) ; path = next ) {
+		if ( ( path->Type == MESSAGING_DEVICE_PATH ) &&
+		     ( path->SubType == MSG_MAC_ADDR_DP ) ) {
+			mac = container_of ( path, MAC_ADDR_DEVICE_PATH,
+					     Header );
+			return &mac->MacAddress;
+		}
+	}
+
+	/* No MAC address found */
+	return NULL;
+}
+
 /**
  * Get VLAN tag from device path
  *

From 9bbe77669c6e2b71826449d854f5aa0e2cee7767 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 26 Mar 2024 15:17:23 +0000
Subject: [PATCH 121/237] [efi] Extract basic network settings from loaded
 image device path

The UEFI HTTP boot mechanism is extraordinarily badly designed, even
by the standards of the UEFI specification in general.  It has the
symptoms of a feature that has been designed entirely in terms of user
stories, without any consideration at all being given to the
underlying technical architecture.  It does work, provided that you
are doing precisely and only what was envisioned by the product owner.
If you want to try anything outside the bounds of the product owner's
extremely limited imagination, then you are almost certainly about to
enter a world of pain.

As one very minor example of this: the cached DHCP packet is not
available when using HTTP boot.  The UEFI HTTP boot code does perform
DHCP, but it pointlessly and unhelpfully throws away the DHCP packet
and trashes the network interface configuration before handing over to
the downloaded executable.

Work around this imbecility by parsing and applying the few network
configuration settings that are persisted into the loaded image's
device path.  This is limited to very basic information such as the IP
address, gateway address, and DNS server address, but it does at least
provide enough for a functional routing table.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/settings.h  |   2 +
 src/interface/efi/efi_path.c | 241 +++++++++++++++++++++++++++++++++++
 2 files changed, 243 insertions(+)

diff --git a/src/include/ipxe/settings.h b/src/include/ipxe/settings.h
index e042b9758..424188deb 100644
--- a/src/include/ipxe/settings.h
+++ b/src/include/ipxe/settings.h
@@ -445,6 +445,8 @@ len6_setting __setting ( SETTING_IP6, len6 );
 extern const struct setting
 gateway6_setting __setting ( SETTING_IP6, gateway6 );
 extern const struct setting
+dns6_setting __setting ( SETTING_IP6_EXTRA, dns6 );
+extern const struct setting
 hostname_setting __setting ( SETTING_HOST, hostname );
 extern const struct setting
 domain_setting __setting ( SETTING_IP_EXTRA, domain );
diff --git a/src/interface/efi/efi_path.c b/src/interface/efi/efi_path.c
index 23f1bb84d..ac3c04987 100644
--- a/src/interface/efi/efi_path.c
+++ b/src/interface/efi/efi_path.c
@@ -34,6 +34,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/fcp.h>
 #include <ipxe/ib_srp.h>
 #include <ipxe/usb.h>
+#include <ipxe/settings.h>
+#include <ipxe/dhcp.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_path.h>
@@ -44,6 +46,40 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+/** An EFI device path settings block */
+struct efi_path_settings {
+	/** Settings interface */
+	struct settings settings;
+	/** Device path */
+	EFI_DEVICE_PATH_PROTOCOL *path;
+};
+
+/** An EFI device path setting */
+struct efi_path_setting {
+	/** Setting */
+	const struct setting *setting;
+	/**
+	 * Fetch setting
+	 *
+	 * @v pathset		Path setting
+	 * @v path		Device path
+	 * @v data		Buffer to fill with setting data
+	 * @v len		Length of buffer
+	 * @ret len		Length of setting data, or negative error
+	 */
+	int ( * fetch ) ( struct efi_path_setting *pathset,
+			  EFI_DEVICE_PATH_PROTOCOL *path,
+			  void *data, size_t len );
+	/** Path type */
+	uint8_t type;
+	/** Path subtype */
+	uint8_t subtype;
+	/** Offset within device path */
+	uint8_t offset;
+	/** Length (if fixed) */
+	uint8_t len;
+};
+
 /**
  * Find next element in device path
  *
@@ -657,3 +693,208 @@ EFI_DEVICE_PATH_PROTOCOL * efi_describe ( struct interface *intf ) {
 	intf_put ( dest );
 	return path;
 }
+
+/**
+ * Fetch an EFI device path fixed-size setting
+ *
+ * @v pathset		Path setting
+ * @v path		Device path
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int efi_path_fetch_fixed ( struct efi_path_setting *pathset,
+				  EFI_DEVICE_PATH_PROTOCOL *path,
+				  void *data, size_t len ) {
+
+	/* Copy data */
+	if ( len > pathset->len )
+		len = pathset->len;
+	memcpy ( data, ( ( ( void * ) path ) + pathset->offset ), len );
+
+	return pathset->len;
+}
+
+/**
+ * Fetch an EFI device path DNS setting
+ *
+ * @v pathset		Path setting
+ * @v path		Device path
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int efi_path_fetch_dns ( struct efi_path_setting *pathset,
+				EFI_DEVICE_PATH_PROTOCOL *path,
+				void *data, size_t len ) {
+	DNS_DEVICE_PATH *dns = container_of ( path, DNS_DEVICE_PATH, Header );
+	unsigned int count;
+	unsigned int i;
+	size_t frag_len;
+
+	/* Check applicability */
+	if ( ( !! dns->IsIPv6 ) !=
+	     ( pathset->setting->type == &setting_type_ipv6 ) )
+		return -ENOENT;
+
+	/* Calculate number of addresses */
+	count = ( ( ( ( path->Length[1] << 8 ) | path->Length[0] ) -
+		    pathset->offset ) / sizeof ( dns->DnsServerIp[0] ) );
+
+	/* Copy data */
+	for ( i = 0 ; i < count ; i++ ) {
+		frag_len = len;
+		if ( frag_len > pathset->len )
+			frag_len = pathset->len;
+		memcpy ( data, &dns->DnsServerIp[i], frag_len );
+		data += frag_len;
+		len -= frag_len;
+	}
+
+	return ( count * pathset->len );
+}
+
+/** EFI device path settings */
+static struct efi_path_setting efi_path_settings[] = {
+	{ &ip_setting, efi_path_fetch_fixed, MESSAGING_DEVICE_PATH,
+	  MSG_IPv4_DP, offsetof ( IPv4_DEVICE_PATH, LocalIpAddress ),
+	  sizeof ( struct in_addr ) },
+	{ &netmask_setting, efi_path_fetch_fixed, MESSAGING_DEVICE_PATH,
+	  MSG_IPv4_DP, offsetof ( IPv4_DEVICE_PATH, SubnetMask ),
+	  sizeof ( struct in_addr ) },
+	{ &gateway_setting, efi_path_fetch_fixed, MESSAGING_DEVICE_PATH,
+	  MSG_IPv4_DP, offsetof ( IPv4_DEVICE_PATH, GatewayIpAddress ),
+	  sizeof ( struct in_addr ) },
+	{ &ip6_setting, efi_path_fetch_fixed, MESSAGING_DEVICE_PATH,
+	  MSG_IPv6_DP, offsetof ( IPv6_DEVICE_PATH, LocalIpAddress ),
+	  sizeof ( struct in6_addr ) },
+	{ &len6_setting, efi_path_fetch_fixed, MESSAGING_DEVICE_PATH,
+	  MSG_IPv6_DP, offsetof ( IPv6_DEVICE_PATH, PrefixLength ),
+	  sizeof ( uint8_t ) },
+	{ &gateway6_setting, efi_path_fetch_fixed, MESSAGING_DEVICE_PATH,
+	  MSG_IPv6_DP, offsetof ( IPv6_DEVICE_PATH, GatewayIpAddress ),
+	  sizeof ( struct in6_addr ) },
+	{ &dns_setting, efi_path_fetch_dns, MESSAGING_DEVICE_PATH,
+	  MSG_DNS_DP, offsetof ( DNS_DEVICE_PATH, DnsServerIp ),
+	  sizeof ( struct in_addr ) },
+	{ &dns6_setting, efi_path_fetch_dns, MESSAGING_DEVICE_PATH,
+	  MSG_DNS_DP, offsetof ( DNS_DEVICE_PATH, DnsServerIp ),
+	  sizeof ( struct in6_addr ) },
+};
+
+/**
+ * Fetch value of EFI device path setting
+ *
+ * @v settings		Settings block
+ * @v setting		Setting to fetch
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int efi_path_fetch ( struct settings *settings, struct setting *setting,
+			    void *data, size_t len ) {
+	struct efi_path_settings *pathsets =
+		container_of ( settings, struct efi_path_settings, settings );
+	EFI_DEVICE_PATH_PROTOCOL *path = pathsets->path;
+	EFI_DEVICE_PATH_PROTOCOL *next;
+	struct efi_path_setting *pathset;
+	unsigned int i;
+	int ret;
+
+	/* Find matching path setting, if any */
+	for ( i = 0 ; i < ( sizeof ( efi_path_settings ) /
+			    sizeof ( efi_path_settings[0] ) ) ; i++ ) {
+
+		/* Check for a matching setting */
+		pathset = &efi_path_settings[i];
+		if ( setting_cmp ( setting, pathset->setting ) != 0 )
+			continue;
+
+		/* Find matching device path element, if any */
+		for ( ; ( next = efi_path_next ( path ) ) ; path = next ) {
+
+			/* Check for a matching path type */
+			if ( ( path->Type != pathset->type ) ||
+			     ( path->SubType != pathset->subtype ) )
+				continue;
+
+			/* Fetch value */
+			if ( ( ret = pathset->fetch ( pathset, path,
+						      data, len ) ) < 0 )
+				return ret;
+
+			/* Apply default type, if not already set */
+			if ( ! setting->type )
+				setting->type = pathset->setting->type;
+
+			return ret;
+		}
+		break;
+	}
+
+	return -ENOENT;
+}
+
+/** EFI device path settings operations */
+static struct settings_operations efi_path_settings_operations = {
+	.fetch = efi_path_fetch,
+};
+
+/**
+ * Create per-netdevice EFI path settings
+ *
+ * @v netdev		Network device
+ * @v priv		Private data
+ * @ret rc		Return status code
+ */
+static int efi_path_net_probe ( struct net_device *netdev, void *priv ) {
+	struct efi_path_settings *pathsets = priv;
+	struct settings *settings = &pathsets->settings;
+	EFI_DEVICE_PATH_PROTOCOL *path = efi_loaded_image_path;
+	unsigned int vlan;
+	void *mac;
+	int rc;
+
+	/* Check applicability */
+	pathsets->path = path;
+	mac = efi_path_mac ( path );
+	vlan = efi_path_vlan ( path );
+	if ( ( mac == NULL ) ||
+	     ( memcmp ( mac, netdev->ll_addr,
+			netdev->ll_protocol->ll_addr_len ) != 0 ) ||
+	     ( vlan != vlan_tag ( netdev ) ) ) {
+		DBGC ( settings, "EFI path %s does not apply to %s\n",
+		       efi_devpath_text ( path ), netdev->name );
+		return 0;
+	}
+
+	/* Never override a real DHCP settings block */
+	if ( find_child_settings ( netdev_settings ( netdev ),
+				   DHCP_SETTINGS_NAME ) ) {
+		DBGC ( settings, "EFI path %s not overriding %s DHCP "
+		       "settings\n", efi_devpath_text ( path ), netdev->name );
+		return 0;
+	}
+
+	/* Initialise and register settings */
+	settings_init ( settings, &efi_path_settings_operations,
+			&netdev->refcnt, NULL );
+	if ( ( rc = register_settings ( settings, netdev_settings ( netdev ),
+					DHCP_SETTINGS_NAME ) ) != 0 ) {
+		DBGC ( settings, "EFI path %s could not register for %s: %s\n",
+		       efi_devpath_text ( path ), netdev->name,
+		       strerror ( rc ) );
+		return rc;
+	}
+	DBGC ( settings, "EFI path %s registered for %s\n",
+	       efi_devpath_text ( path ), netdev->name );
+
+	return 0;
+}
+
+/** EFI path settings per-netdevice driver */
+struct net_driver efi_path_net_driver __net_driver = {
+	.name = "EFI path",
+	.priv_len = sizeof ( struct efi_path_settings ),
+	.probe = efi_path_net_probe,
+};

From 37850e0e854292d074c2d35d18d7bb78d8e6ff85 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 27 Mar 2024 14:28:47 +0000
Subject: [PATCH 122/237] [build] Fix build failures with random versions of
 gcc

For unknown reasons, miscellaneous versions of gcc seem to struggle
with the static assertions used to ensure the correct layout of the
GCM structures.

Adjust the assertions to use offsetof() rather than direct pointer
comparison, on the basis that offsetof() must be a compile-time
constant value.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/gcm.c       | 26 +++++++++++++++-----------
 src/include/ipxe/gcm.h |  7 +++----
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/crypto/gcm.c b/src/crypto/gcm.c
index a32890d59..b93925d07 100644
--- a/src/crypto/gcm.c
+++ b/src/crypto/gcm.c
@@ -109,6 +109,9 @@ static union gcm_block gcm_cached_mult[256];
  */
 static uint16_t gcm_cached_reduce[256];
 
+/** Offset of a field within GCM context */
+#define gcm_offset( field ) offsetof ( struct gcm_context, field )
+
 /**
  * Reverse bits in a byte
  *
@@ -470,17 +473,13 @@ int gcm_setkey ( struct gcm_context *context, const void *key, size_t keylen,
  */
 void gcm_setiv ( struct gcm_context *context, const void *iv, size_t ivlen ) {
 
-	/* Sanity check: ensure that memset()s will clear expected state */
-	build_assert ( &context->hash < &context->ctr );
-	build_assert ( &context->len < &context->ctr );
-	build_assert ( &context->ctr < &context->key );
-	build_assert ( ( ( void * ) &context->raw_cipher ) >
-		       ( ( void * ) &context->key ) );
-	build_assert ( ( ( void * ) context->raw_ctx ) >
-		       ( ( void * ) &context->key ) );
-
 	/* Reset non-key state */
-	memset ( context, 0, offsetof ( typeof ( *context ), key ) );
+	memset ( context, 0, gcm_offset ( key ) );
+	build_assert ( gcm_offset ( key ) > gcm_offset ( hash ) );
+	build_assert ( gcm_offset ( key ) > gcm_offset ( len ) );
+	build_assert ( gcm_offset ( key ) > gcm_offset ( ctr ) );
+	build_assert ( gcm_offset ( key ) < gcm_offset ( raw_cipher ) );
+	build_assert ( gcm_offset ( key ) < gcm_offset ( raw_ctx ) );
 
 	/* Reset counter */
 	context->ctr.ctr.value = cpu_to_be32 ( 1 );
@@ -499,7 +498,12 @@ void gcm_setiv ( struct gcm_context *context, const void *iv, size_t ivlen ) {
 		assert ( context->len.len.add == 0 );
 
 		/* Reset non-key, non-counter state */
-		memset ( context, 0, offsetof ( typeof ( *context ), ctr ) );
+		memset ( context, 0, gcm_offset ( ctr ) );
+		build_assert ( gcm_offset ( ctr ) > gcm_offset ( hash ) );
+		build_assert ( gcm_offset ( ctr ) > gcm_offset ( len ) );
+		build_assert ( gcm_offset ( ctr ) < gcm_offset ( key ) );
+		build_assert ( gcm_offset ( ctr ) < gcm_offset ( raw_cipher ) );
+		build_assert ( gcm_offset ( ctr ) < gcm_offset ( raw_ctx ) );
 	}
 
 	DBGC2 ( context, "GCM %p Y[0]:\n", context );
diff --git a/src/include/ipxe/gcm.h b/src/include/ipxe/gcm.h
index 4864445d2..2c785a977 100644
--- a/src/include/ipxe/gcm.h
+++ b/src/include/ipxe/gcm.h
@@ -89,10 +89,9 @@ static int _gcm_name ## _setkey ( void *ctx, const void *key,		\
 				  size_t keylen ) {			\
 	struct _gcm_name ## _context *context = ctx;			\
 	build_assert ( _blocksize == sizeof ( context->gcm.key ) );	\
-	build_assert ( ( ( void * ) &context->gcm ) ==			\
-		       ( ( void * ) context ) );			\
-	build_assert ( ( ( void * ) &context->raw ) ==			\
-		       ( ( void * ) context->gcm.raw_ctx ) );		\
+	build_assert ( offsetof ( typeof ( *context ), gcm ) == 0 );	\
+	build_assert ( offsetof ( typeof ( *context ), raw ) ==		\
+		       offsetof ( typeof ( *context ), gcm.raw_ctx ) );	\
 	return gcm_setkey ( &context->gcm, key, keylen, &_raw_cipher );	\
 }									\
 static void _gcm_name ## _setiv ( void *ctx, const void *iv,		\

From f39b48d5f82793c75700b0787b36f65d33079f3b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Mar 2024 13:45:24 +0000
Subject: [PATCH 123/237] [image] Allow opaque URI component to provide image
 name

Some URI schemes allow for a path name to be specified via the opaque
component of the URI (e.g. "file:/script.ipxe" to specify a path on
the filesystem from which iPXE itself was loaded).  Files loaded from
such paths will currently fail to be assigned an appropriate name,
since only the path component of the URI will be used to construct a
default image name.

Fix by falling back to attempt deriving an image name from the opaque
component of a URI, if no path component is specified.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/image.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/core/image.c b/src/core/image.c
index 3e65b5edf..bf0e4f756 100644
--- a/src/core/image.c
+++ b/src/core/image.c
@@ -134,10 +134,13 @@ int image_set_uri ( struct image *image, struct uri *uri ) {
 	int rc;
 
 	/* Set name, if image does not already have one */
-	if ( uri->path && ( ! ( image->name && image->name[0] ) ) ) {
-		name = basename ( ( char * ) uri->path );
-		if ( ( rc = image_set_name ( image, name ) ) != 0 )
-			return rc;
+	if ( ! ( image->name && image->name[0] ) ) {
+		name = ( uri->path ? uri->path : uri->opaque );
+		if ( name ) {
+			name = basename ( ( char * ) name );
+			if ( ( rc = image_set_name ( image, name ) ) != 0 )
+				return rc;
+		}
 	}
 
 	/* Update image URI */

From 19f39bc07a82ac589cd2fa360b6f32c15d0eb0a8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Mar 2024 13:32:25 +0000
Subject: [PATCH 124/237] [efi] Report local file errors during download,
 rather than on opening

iPXE is designed around fully asynchronous I/O, including asynchronous
connection opening.  Almost all errors are therefore necessarily
reported as occurring during an in-progress download, rather than
occurring at the time that the URI is opened.

Local file access is currently an exception to this: errors such as
nonexistent files will be encountered while opening the URI.  This
results in mildly unexpected error messages of the form "Could not
start download", rather than the usual pattern of showing the URI, the
initial progress dots, and then the error message.

Fix this inconsistency by deferring the local filesystem access until
the local file download process is running.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_local.c | 254 +++++++++++++++++-----------------
 1 file changed, 130 insertions(+), 124 deletions(-)

diff --git a/src/interface/efi/efi_local.c b/src/interface/efi/efi_local.c
index f54f5daea..00cc1147f 100644
--- a/src/interface/efi/efi_local.c
+++ b/src/interface/efi/efi_local.c
@@ -60,6 +60,13 @@ struct efi_local {
 	/** Download process */
 	struct process process;
 
+	/** Download URI */
+	struct uri *uri;
+	/** Volume name, or NULL to use loaded image's device */
+	const char *volume;
+	/** File path */
+	const char *path;
+
 	/** EFI root directory */
 	EFI_FILE_PROTOCOL *root;
 	/** EFI file */
@@ -68,6 +75,19 @@ struct efi_local {
 	size_t len;
 };
 
+/**
+ * Free local file
+ *
+ * @v refcnt		Reference count
+ */
+static void efi_local_free ( struct refcnt *refcnt ) {
+	struct efi_local *local =
+		container_of ( refcnt, struct efi_local, refcnt );
+
+	uri_put ( local->uri );
+	free ( local );
+}
+
 /**
  * Close local file
  *
@@ -95,91 +115,6 @@ static void efi_local_close ( struct efi_local *local, int rc ) {
 	}
 }
 
-/**
- * Local file process
- *
- * @v local		Local file
- */
-static void efi_local_step ( struct efi_local *local ) {
-	EFI_FILE_PROTOCOL *file = local->file;
-	struct io_buffer *iobuf = NULL;
-	size_t remaining;
-	size_t frag_len;
-	UINTN size;
-	EFI_STATUS efirc;
-	int rc;
-
-	/* Wait until data transfer interface is ready */
-	if ( ! xfer_window ( &local->xfer ) )
-		return;
-
-	/* Presize receive buffer */
-	remaining = local->len;
-	xfer_seek ( &local->xfer, remaining );
-	xfer_seek ( &local->xfer, 0 );
-
-	/* Get file contents */
-	while ( remaining ) {
-
-		/* Calculate length for this fragment */
-		frag_len = remaining;
-		if ( frag_len > EFI_LOCAL_BLKSIZE )
-			frag_len = EFI_LOCAL_BLKSIZE;
-
-		/* Allocate I/O buffer */
-		iobuf = xfer_alloc_iob ( &local->xfer, frag_len );
-		if ( ! iobuf ) {
-			rc = -ENOMEM;
-			goto err;
-		}
-
-		/* Read block */
-		size = frag_len;
-		if ( ( efirc = file->Read ( file, &size, iobuf->data ) ) != 0 ){
-			rc = -EEFI ( efirc );
-			DBGC ( local, "LOCAL %p could not read from file: %s\n",
-			       local, strerror ( rc ) );
-			goto err;
-		}
-		assert ( size <= frag_len );
-		iob_put ( iobuf, size );
-
-		/* Deliver data */
-		if ( ( rc = xfer_deliver_iob ( &local->xfer,
-					       iob_disown ( iobuf ) ) ) != 0 ) {
-			DBGC ( local, "LOCAL %p could not deliver data: %s\n",
-			       local, strerror ( rc ) );
-			goto err;
-		}
-
-		/* Move to next block */
-		remaining -= frag_len;
-	}
-
-	/* Close download */
-	efi_local_close ( local, 0 );
-
-	return;
-
- err:
-	free_iob ( iobuf );
-	efi_local_close ( local, rc );
-}
-
-/** Data transfer interface operations */
-static struct interface_operation efi_local_operations[] = {
-	INTF_OP ( xfer_window_changed, struct efi_local *, efi_local_step ),
-	INTF_OP ( intf_close, struct efi_local *, efi_local_close ),
-};
-
-/** Data transfer interface descriptor */
-static struct interface_descriptor efi_local_xfer_desc =
-	INTF_DESC ( struct efi_local, xfer, efi_local_operations );
-
-/** Process descriptor */
-static struct process_descriptor efi_local_process_desc =
-	PROC_DESC_ONCE ( struct efi_local, process, efi_local_step );
-
 /**
  * Check for matching volume name
  *
@@ -298,15 +233,14 @@ static int efi_local_open_root ( struct efi_local *local, EFI_HANDLE device,
  * Open root filesystem of specified volume
  *
  * @v local		Local file
- * @v volume		Volume name, or NULL to use loaded image's device
  * @ret rc		Return status code
  */
-static int efi_local_open_volume ( struct efi_local *local,
-				   const char *volume ) {
+static int efi_local_open_volume ( struct efi_local *local ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	EFI_GUID *protocol = &efi_simple_file_system_protocol_guid;
 	int ( * check ) ( struct efi_local *local, EFI_HANDLE device,
 			  EFI_FILE_PROTOCOL *root, const char *volume );
+	const char *volume = local->volume;
 	EFI_DEVICE_PATH_PROTOCOL *path;
 	EFI_FILE_PROTOCOL *root;
 	EFI_HANDLE *handles;
@@ -419,11 +353,9 @@ static int efi_local_open_resolved ( struct efi_local *local,
  * Open specified path
  *
  * @v local		Local file
- * @v filename		Path to file relative to our own image
  * @ret rc		Return status code
  */
-static int efi_local_open_path ( struct efi_local *local,
-				 const char *filename ) {
+static int efi_local_open_path ( struct efi_local *local ) {
 	EFI_DEVICE_PATH_PROTOCOL *path = efi_loaded_image->FilePath;
 	EFI_DEVICE_PATH_PROTOCOL *next;
 	FILEPATH_DEVICE_PATH *fp;
@@ -454,7 +386,7 @@ static int efi_local_open_path ( struct efi_local *local,
 	}
 
 	/* Resolve path */
-	resolved = resolve_path ( base, filename );
+	resolved = resolve_path ( base, local->path );
 	if ( ! resolved ) {
 		rc = -ENOMEM;
 		goto err_resolve;
@@ -523,6 +455,106 @@ static int efi_local_len ( struct efi_local *local ) {
 	return rc;
 }
 
+/**
+ * Local file process
+ *
+ * @v local		Local file
+ */
+static void efi_local_step ( struct efi_local *local ) {
+	struct io_buffer *iobuf = NULL;
+	size_t remaining;
+	size_t frag_len;
+	UINTN size;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Wait until data transfer interface is ready */
+	if ( ! xfer_window ( &local->xfer ) )
+		return;
+
+	/* Open specified volume root directory, if not yet open */
+	if ( ( ! local->root ) &&
+	     ( ( rc = efi_local_open_volume ( local ) ) != 0 ) )
+		goto err;
+
+	/* Open specified file, if not yet open */
+	if ( ( ! local->file ) &&
+	     ( ( rc = efi_local_open_path ( local ) ) != 0 ) )
+		goto err;
+
+	/* Get file length, if not yet known */
+	if ( ( ! local->len ) &&
+	     ( ( rc = efi_local_len ( local ) ) != 0 ) )
+		goto err;
+
+	/* Presize receive buffer */
+	remaining = local->len;
+	xfer_seek ( &local->xfer, remaining );
+	xfer_seek ( &local->xfer, 0 );
+
+	/* Get file contents */
+	while ( remaining ) {
+
+		/* Calculate length for this fragment */
+		frag_len = remaining;
+		if ( frag_len > EFI_LOCAL_BLKSIZE )
+			frag_len = EFI_LOCAL_BLKSIZE;
+
+		/* Allocate I/O buffer */
+		iobuf = xfer_alloc_iob ( &local->xfer, frag_len );
+		if ( ! iobuf ) {
+			rc = -ENOMEM;
+			goto err;
+		}
+
+		/* Read block */
+		size = frag_len;
+		if ( ( efirc = local->file->Read ( local->file, &size,
+						   iobuf->data ) ) != 0 ) {
+			rc = -EEFI ( efirc );
+			DBGC ( local, "LOCAL %p could not read from file: %s\n",
+			       local, strerror ( rc ) );
+			goto err;
+		}
+		assert ( size <= frag_len );
+		iob_put ( iobuf, size );
+
+		/* Deliver data */
+		if ( ( rc = xfer_deliver_iob ( &local->xfer,
+					       iob_disown ( iobuf ) ) ) != 0 ) {
+			DBGC ( local, "LOCAL %p could not deliver data: %s\n",
+			       local, strerror ( rc ) );
+			goto err;
+		}
+
+		/* Move to next block */
+		remaining -= frag_len;
+	}
+
+	/* Close download */
+	efi_local_close ( local, 0 );
+
+	return;
+
+ err:
+	free_iob ( iobuf );
+	efi_local_close ( local, rc );
+}
+
+/** Data transfer interface operations */
+static struct interface_operation efi_local_operations[] = {
+	INTF_OP ( xfer_window_changed, struct efi_local *, efi_local_step ),
+	INTF_OP ( intf_close, struct efi_local *, efi_local_close ),
+};
+
+/** Data transfer interface descriptor */
+static struct interface_descriptor efi_local_xfer_desc =
+	INTF_DESC ( struct efi_local, xfer, efi_local_operations );
+
+/** Process descriptor */
+static struct process_descriptor efi_local_process_desc =
+	PROC_DESC_ONCE ( struct efi_local, process, efi_local_step );
+
 /**
  * Open local file
  *
@@ -532,36 +564,18 @@ static int efi_local_len ( struct efi_local *local ) {
  */
 static int efi_local_open ( struct interface *xfer, struct uri *uri ) {
 	struct efi_local *local;
-	const char *volume;
-	const char *path;
-	int rc;
-
-	/* Parse URI */
-	volume = ( ( uri->host && uri->host[0] ) ? uri->host : NULL );
-	path = ( uri->opaque ? uri->opaque : uri->path );
 
 	/* Allocate and initialise structure */
 	local = zalloc ( sizeof ( *local ) );
-	if ( ! local ) {
-		rc = -ENOMEM;
-		goto err_alloc;
-	}
-	ref_init ( &local->refcnt, NULL );
+	if ( ! local )
+		return -ENOMEM;
+	ref_init ( &local->refcnt, efi_local_free );
 	intf_init ( &local->xfer, &efi_local_xfer_desc, &local->refcnt );
 	process_init_stopped ( &local->process, &efi_local_process_desc,
 			       &local->refcnt );
-
-	/* Open specified volume */
-	if ( ( rc = efi_local_open_volume ( local, volume ) ) != 0 )
-		goto err_open_root;
-
-	/* Open specified path */
-	if ( ( rc = efi_local_open_path ( local, path ) ) != 0 )
-		goto err_open_file;
-
-	/* Get length of file */
-	if ( ( rc = efi_local_len ( local ) ) != 0 )
-		goto err_len;
+	local->uri = uri_get ( uri );
+	local->volume = ( ( uri->host && uri->host[0] ) ? uri->host : NULL );
+	local->path = ( uri->opaque ? uri->opaque : uri->path );
 
 	/* Start download process */
 	process_add ( &local->process );
@@ -570,14 +584,6 @@ static int efi_local_open ( struct interface *xfer, struct uri *uri ) {
 	intf_plug_plug ( &local->xfer, xfer );
 	ref_put ( &local->refcnt );
 	return 0;
-
- err_len:
- err_open_file:
- err_open_root:
-	efi_local_close ( local, 0 );
-	ref_put ( &local->refcnt );
- err_alloc:
-	return rc;
 }
 
 /** EFI local file URI opener */

From 43deab89c329984366d519ae612b67ee0d5998b6 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Mar 2024 12:41:32 +0000
Subject: [PATCH 125/237] [efi] Add error table entry for local filesystem
 EFI_NOT_FOUND error

Add an abbreviated "Not found" error message for an EFI_NOT_FOUND
error encountered when attempting to open a file on a local
filesystem, so that any automatic attempt to download a non-existent
autoexec.ipxe script produces only a minimal error message.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_local.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/interface/efi/efi_local.c b/src/interface/efi/efi_local.c
index 00cc1147f..ec6d93a1d 100644
--- a/src/interface/efi/efi_local.c
+++ b/src/interface/efi/efi_local.c
@@ -35,6 +35,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/uri.h>
 #include <ipxe/iobuf.h>
 #include <ipxe/process.h>
+#include <ipxe/errortab.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_strings.h>
 #include <ipxe/efi/efi_path.h>
@@ -48,6 +49,17 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
+/* Disambiguate the various error causes */
+#define EINFO_EEFI_OPEN							\
+	__einfo_uniqify ( EINFO_EPLATFORM, 0x01, "Could not open" )
+#define EINFO_EEFI_OPEN_NOT_FOUND					\
+	__einfo_platformify ( EINFO_EEFI_OPEN, EFI_NOT_FOUND,		\
+			      "Not found" )
+#define EEFI_OPEN_NOT_FOUND						\
+	__einfo_error ( EINFO_EEFI_OPEN_NOT_FOUND )
+#define EEFI_OPEN( efirc ) EPLATFORM ( EINFO_EEFI_OPEN, efirc,		\
+				       EEFI_OPEN_NOT_FOUND )
+
 /** Download blocksize */
 #define EFI_LOCAL_BLKSIZE 4096
 
@@ -75,6 +87,11 @@ struct efi_local {
 	size_t len;
 };
 
+/** Human-readable error messages */
+struct errortab efi_local_errors[] __errortab = {
+	__einfo_errortab ( EINFO_EEFI_OPEN_NOT_FOUND ),
+};
+
 /**
  * Free local file
  *
@@ -339,7 +356,7 @@ static int efi_local_open_resolved ( struct efi_local *local,
 	/* Open file */
 	if ( ( efirc = local->root->Open ( local->root, &file, name,
 					   EFI_FILE_MODE_READ, 0 ) ) != 0 ) {
-		rc = -EEFI ( efirc );
+		rc = -EEFI_OPEN ( efirc );
 		DBGC ( local, "LOCAL %p could not open \"%s\": %s\n",
 		       local, resolved, strerror ( rc ) );
 		return rc;

From afae8817826ef8356b89a5e131ff42750ab267d4 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Mar 2024 12:38:09 +0000
Subject: [PATCH 126/237] [tftp] Add error table entry for TFTP "file not
 found" error code

Add an abbreviated "Not found" error message for a TFTP "file not
found" error code, so that any automatic attempt to download a
non-existent autoexec.ipxe script produces only a minimal error
message.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/udp/tftp.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/net/udp/tftp.c b/src/net/udp/tftp.c
index 3073e682f..2ee01862a 100644
--- a/src/net/udp/tftp.c
+++ b/src/net/udp/tftp.c
@@ -44,6 +44,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/dhcp.h>
 #include <ipxe/uri.h>
 #include <ipxe/profile.h>
+#include <ipxe/errortab.h>
 #include <ipxe/tftp.h>
 
 /** @file
@@ -76,6 +77,9 @@ FEATURE ( FEATURE_PROTOCOL, "TFTP", DHCP_EB_FEATURE_TFTP, 1 );
 #define EINVAL_MC_INVALID_PORT __einfo_error ( EINFO_EINVAL_MC_INVALID_PORT )
 #define EINFO_EINVAL_MC_INVALID_PORT __einfo_uniqify \
 	( EINFO_EINVAL, 0x07, "Invalid multicast port" )
+#define ENOENT_NOT_FOUND __einfo_error ( EINFO_ENOENT_NOT_FOUND )
+#define EINFO_ENOENT_NOT_FOUND __einfo_uniqify \
+	( EINFO_ENOENT, 0x01, "Not found" )
 
 /**
  * A TFTP request
@@ -167,6 +171,11 @@ static struct profiler tftp_client_profiler __profiler =
 static struct profiler tftp_server_profiler __profiler =
 	{ .name = "tftp.server" };
 
+/** Human-readable error messages */
+struct errortab tftp_errors[] __errortab = {
+	__einfo_errortab ( EINFO_ENOENT_NOT_FOUND ),
+};
+
 /**
  * Free TFTP request
  *
@@ -872,7 +881,7 @@ static int tftp_rx_data ( struct tftp_request *tftp,
  */
 static int tftp_errcode_to_rc ( unsigned int errcode ) {
 	switch ( errcode ) {
-	case TFTP_ERR_FILE_NOT_FOUND:	return -ENOENT;
+	case TFTP_ERR_FILE_NOT_FOUND:	return -ENOENT_NOT_FOUND;
 	case TFTP_ERR_ACCESS_DENIED:	return -EACCES;
 	case TFTP_ERR_ILLEGAL_OP:	return -ENOTTY;
 	default:			return -ENOTSUP;

From 764e34f15af89cc7c5e46694ac15c5266f13b3d3 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Mar 2024 12:35:12 +0000
Subject: [PATCH 127/237] [http] Add error table entry for HTTP 404 Not Found
 error

Add an abbreviated "Not found" error message for an HTTP 404 status
code, so that any automatic attempt to download a non-existent
autoexec.ipxe script produces only a minimal error message.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/tcp/httpcore.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/net/tcp/httpcore.c b/src/net/tcp/httpcore.c
index 9ad39656d..af2a237cf 100644
--- a/src/net/tcp/httpcore.c
+++ b/src/net/tcp/httpcore.c
@@ -89,7 +89,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	__einfo_uniqify ( EINFO_EIO, 0x05, "HTTP 5xx Server Error" )
 #define ENOENT_404 __einfo_error ( EINFO_ENOENT_404 )
 #define EINFO_ENOENT_404 \
-	__einfo_uniqify ( EINFO_ENOENT, 0x01, "HTTP 404 Not Found" )
+	__einfo_uniqify ( EINFO_ENOENT, 0x01, "Not found" )
 #define ENOTSUP_CONNECTION __einfo_error ( EINFO_ENOTSUP_CONNECTION )
 #define EINFO_ENOTSUP_CONNECTION \
 	__einfo_uniqify ( EINFO_ENOTSUP, 0x01, "Unsupported connection header" )
@@ -114,6 +114,7 @@ static struct profiler http_xfer_profiler __profiler = { .name = "http.xfer" };
 
 /** Human-readable error messages */
 struct errortab http_errors[] __errortab = {
+	__einfo_errortab ( EINFO_ENOENT_404 ),
 	__einfo_errortab ( EINFO_EIO_4XX ),
 	__einfo_errortab ( EINFO_EIO_5XX ),
 };

From b52b4a46d9ee854130db7a8927f33391fc6ba1fe Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Mar 2024 12:43:24 +0000
Subject: [PATCH 128/237] [efi] Allow for allocating EFI devices from arbitrary
 handles

Split out the code that allocates our internal struct efi_device
representations, to allow for the creation of temporary MNP devices in
order to download the autoexec.ipxe script.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/efi_driver.h |   2 +
 src/interface/efi/efi_driver.c    | 107 ++++++++++++++++++------------
 2 files changed, 67 insertions(+), 42 deletions(-)

diff --git a/src/include/ipxe/efi/efi_driver.h b/src/include/ipxe/efi/efi_driver.h
index 411e93642..7b64e1e0b 100644
--- a/src/include/ipxe/efi/efi_driver.h
+++ b/src/include/ipxe/efi/efi_driver.h
@@ -86,6 +86,8 @@ static inline void * efidev_get_drvdata ( struct efi_device *efidev ) {
 	return efidev->priv;
 }
 
+extern struct efi_device * efidev_alloc ( EFI_HANDLE device );
+extern void efidev_free ( struct efi_device *efidev );
 extern struct efi_device * efidev_parent ( struct device *dev );
 extern int efi_driver_install ( void );
 extern void efi_driver_uninstall ( void );
diff --git a/src/interface/efi/efi_driver.c b/src/interface/efi/efi_driver.c
index 8f8c9f3da..fd9be5f51 100644
--- a/src/interface/efi/efi_driver.c
+++ b/src/interface/efi/efi_driver.c
@@ -61,6 +61,67 @@ static LIST_HEAD ( efi_devices );
 /** We are currently disconnecting drivers */
 static int efi_driver_disconnecting;
 
+/**
+ * Allocate new EFI device
+ *
+ * @v device		EFI device handle
+ * @ret efidev		EFI device, or NULL on error
+ */
+struct efi_device * efidev_alloc ( EFI_HANDLE device ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	struct efi_device *efidev = NULL;
+	union {
+		EFI_DEVICE_PATH_PROTOCOL *path;
+		void *interface;
+	} path;
+	EFI_DEVICE_PATH_PROTOCOL *path_end;
+	size_t path_len;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Open device path */
+	if ( ( efirc = bs->OpenProtocol ( device,
+					  &efi_device_path_protocol_guid,
+					  &path.interface, efi_image_handle,
+					  device,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( device, "EFIDRV %s could not open device path: %s\n",
+		       efi_handle_name ( device ), strerror ( rc ) );
+		goto err_open_path;
+	}
+	path_len = ( efi_path_len ( path.path ) + sizeof ( *path_end ) );
+
+	/* Allocate and initialise structure */
+	efidev = zalloc ( sizeof ( *efidev ) + path_len );
+	if ( ! efidev )
+		goto err_alloc;
+	efidev->device = device;
+	efidev->dev.desc.bus_type = BUS_TYPE_EFI;
+	efidev->path = ( ( ( void * ) efidev ) + sizeof ( *efidev ) );
+	memcpy ( efidev->path, path.path, path_len );
+	INIT_LIST_HEAD ( &efidev->dev.children );
+	list_add ( &efidev->dev.siblings, &efi_devices );
+
+ err_alloc:
+	bs->CloseProtocol ( device, &efi_device_path_protocol_guid,
+			    efi_image_handle, device );
+ err_open_path:
+	return efidev;
+}
+
+/**
+ * Free EFI device
+ *
+ * @v efidev		EFI device
+ */
+void efidev_free ( struct efi_device *efidev ) {
+
+	assert ( list_empty ( &efidev->dev.children ) );
+	list_del ( &efidev->dev.siblings );
+	free ( efidev );
+}
+
 /**
  * Find EFI device
  *
@@ -159,16 +220,9 @@ efi_driver_supported ( EFI_DRIVER_BINDING_PROTOCOL *driver __unused,
 static EFI_STATUS EFIAPI
 efi_driver_start ( EFI_DRIVER_BINDING_PROTOCOL *driver __unused,
 		   EFI_HANDLE device, EFI_DEVICE_PATH_PROTOCOL *child ) {
-	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	struct efi_driver *efidrv;
 	struct efi_device *efidev;
 	struct efi_saved_tpl tpl;
-	union {
-		EFI_DEVICE_PATH_PROTOCOL *path;
-		void *interface;
-	} path;
-	EFI_DEVICE_PATH_PROTOCOL *path_end;
-	size_t path_len;
 	EFI_STATUS efirc;
 	int rc;
 
@@ -197,36 +251,12 @@ efi_driver_start ( EFI_DRIVER_BINDING_PROTOCOL *driver __unused,
 		goto err_disconnecting;
 	}
 
-	/* Open device path */
-	if ( ( efirc = bs->OpenProtocol ( device,
-					  &efi_device_path_protocol_guid,
-					  &path.interface, efi_image_handle,
-					  device,
-					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFIDRV %s could not open device path: %s\n",
-		       efi_handle_name ( device ), strerror ( rc ) );
-		goto err_open_path;
-	}
-	path_len = ( efi_path_len ( path.path ) + sizeof ( *path_end ) );
-
-	/* Allocate and initialise structure */
-	efidev = zalloc ( sizeof ( *efidev ) + path_len );
+	/* Add new device */
+	efidev = efidev_alloc ( device );
 	if ( ! efidev ) {
 		efirc = EFI_OUT_OF_RESOURCES;
 		goto err_alloc;
 	}
-	efidev->device = device;
-	efidev->dev.desc.bus_type = BUS_TYPE_EFI;
-	efidev->path = ( ( ( void * ) efidev ) + sizeof ( *efidev ) );
-	memcpy ( efidev->path, path.path, path_len );
-	INIT_LIST_HEAD ( &efidev->dev.children );
-	list_add ( &efidev->dev.siblings, &efi_devices );
-
-	/* Close device path */
-	bs->CloseProtocol ( device, &efi_device_path_protocol_guid,
-			    efi_image_handle, device );
-	path.path = NULL;
 
 	/* Try to start this device */
 	for_each_table_entry ( efidrv, EFI_DRIVERS ) {
@@ -251,14 +281,8 @@ efi_driver_start ( EFI_DRIVER_BINDING_PROTOCOL *driver __unused,
 	}
 	efirc = EFI_UNSUPPORTED;
 
-	list_del ( &efidev->dev.siblings );
-	free ( efidev );
+	efidev_free ( efidev );
  err_alloc:
-	if ( path.path ) {
-		bs->CloseProtocol ( device, &efi_device_path_protocol_guid,
-				    efi_image_handle, device );
-	}
- err_open_path:
  err_disconnecting:
 	efi_restore_tpl ( &tpl );
  err_already_started:
@@ -306,8 +330,7 @@ efi_driver_stop ( EFI_DRIVER_BINDING_PROTOCOL *driver __unused,
 	efidrv = efidev->driver;
 	assert ( efidrv != NULL );
 	efidrv->stop ( efidev );
-	list_del ( &efidev->dev.siblings );
-	free ( efidev );
+	efidev_free ( efidev );
 
 	efi_restore_tpl ( &tpl );
 	return 0;

From b66f6025fa50f929c8e6448c383ffec273d41e6c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Mar 2024 12:58:10 +0000
Subject: [PATCH 129/237] [efi] Add the ability to create a temporary MNP
 network device

An MNP network device may be temporarily and non-destructively
installed on top of an existing UEFI network stack without having to
disconnect existing drivers.

Add the ability to create such a temporary network device.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/efi/mnp.c     |  2 +-
 src/drivers/net/efi/mnpnet.c  | 57 ++++++++++++++++++++++++++++++++++-
 src/drivers/net/efi/mnpnet.h  | 17 -----------
 src/drivers/net/efi/snponly.c |  2 +-
 src/include/ipxe/efi/mnpnet.h | 20 ++++++++++++
 5 files changed, 78 insertions(+), 20 deletions(-)
 delete mode 100644 src/drivers/net/efi/mnpnet.h
 create mode 100644 src/include/ipxe/efi/mnpnet.h

diff --git a/src/drivers/net/efi/mnp.c b/src/drivers/net/efi/mnp.c
index b79fe188a..33218fb10 100644
--- a/src/drivers/net/efi/mnp.c
+++ b/src/drivers/net/efi/mnp.c
@@ -32,8 +32,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <errno.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_driver.h>
+#include <ipxe/efi/mnpnet.h>
 #include "snpnet.h"
-#include "mnpnet.h"
 
 /**
  * Check to see if driver supports a device
diff --git a/src/drivers/net/efi/mnpnet.c b/src/drivers/net/efi/mnpnet.c
index a07eae54e..84f803f45 100644
--- a/src/drivers/net/efi/mnpnet.c
+++ b/src/drivers/net/efi/mnpnet.c
@@ -38,8 +38,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_service.h>
 #include <ipxe/efi/efi_utils.h>
+#include <ipxe/efi/mnpnet.h>
 #include <ipxe/efi/Protocol/ManagedNetwork.h>
-#include "mnpnet.h"
 
 /** An MNP transmit or receive token */
 struct mnp_token {
@@ -502,3 +502,58 @@ void mnpnet_stop ( struct efi_device *efidev ) {
 	netdev_nullify ( netdev );
 	netdev_put ( netdev );
 }
+
+/**
+ * Create temporary MNP network device
+ *
+ * @v handle		MNP service binding handle
+ * @v netdev		Network device to fill in
+ * @ret rc		Return status code
+ */
+int mnptemp_create ( EFI_HANDLE handle, struct net_device **netdev ) {
+	struct efi_device *efidev;
+	int rc;
+
+	/* Create temporary EFI device */
+	efidev = efidev_alloc ( handle );
+	if ( ! efidev ) {
+		DBGC ( handle, "MNP %s could not create temporary device\n",
+		       efi_handle_name ( handle ) );
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Start temporary network device */
+	if ( ( rc = mnpnet_start ( efidev ) ) != 0 ) {
+		DBGC ( handle, "MNP %s could not start MNP: %s\n",
+		       efi_handle_name ( handle ), strerror ( rc ) );
+		goto err_start;
+	}
+
+	/* Fill in network device */
+	*netdev = efidev_get_drvdata ( efidev );
+
+	return 0;
+
+	mnpnet_stop ( efidev );
+ err_start:
+	efidev_free ( efidev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Destroy temporary MNP network device
+ *
+ * @v netdev		Network device
+ */
+void mnptemp_destroy ( struct net_device *netdev ) {
+	struct mnp_nic *mnp = netdev->priv;
+	struct efi_device *efidev = mnp->efidev;
+
+	/* Stop temporary network device */
+	mnpnet_stop ( efidev );
+
+	/* Free temporary EFI device */
+	efidev_free ( efidev );
+}
diff --git a/src/drivers/net/efi/mnpnet.h b/src/drivers/net/efi/mnpnet.h
deleted file mode 100644
index afed62aae..000000000
--- a/src/drivers/net/efi/mnpnet.h
+++ /dev/null
@@ -1,17 +0,0 @@
-#ifndef _MNPNET_H
-#define _MNPNET_H
-
-/** @file
- *
- * MNP NIC driver
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-struct efi_device;
-
-extern int mnpnet_start ( struct efi_device *efidev );
-extern void mnpnet_stop ( struct efi_device *efidev );
-
-#endif /* _MNPNET_H */
diff --git a/src/drivers/net/efi/snponly.c b/src/drivers/net/efi/snponly.c
index 6786f3e83..2ae63fc06 100644
--- a/src/drivers/net/efi/snponly.c
+++ b/src/drivers/net/efi/snponly.c
@@ -29,10 +29,10 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_utils.h>
+#include <ipxe/efi/mnpnet.h>
 #include <ipxe/efi/Protocol/SimpleNetwork.h>
 #include <ipxe/efi/Protocol/NetworkInterfaceIdentifier.h>
 #include "snpnet.h"
-#include "mnpnet.h"
 #include "nii.h"
 
 /** @file
diff --git a/src/include/ipxe/efi/mnpnet.h b/src/include/ipxe/efi/mnpnet.h
new file mode 100644
index 000000000..99d6cf083
--- /dev/null
+++ b/src/include/ipxe/efi/mnpnet.h
@@ -0,0 +1,20 @@
+#ifndef _IPXE_EFI_MNPNET_H
+#define _IPXE_EFI_MNPNET_H
+
+/** @file
+ *
+ * MNP NIC driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+struct efi_device;
+struct net_device;
+
+extern int mnpnet_start ( struct efi_device *efidev );
+extern void mnpnet_stop ( struct efi_device *efidev );
+extern int mnptemp_create ( EFI_HANDLE handle, struct net_device **netdev );
+extern void mnptemp_destroy ( struct net_device *netdev );
+
+#endif /* _IPXE_EFI_MNPNET_H */

From b940d54235579f2534c12494322876fff555cf36 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 2 Apr 2024 19:36:00 +0100
Subject: [PATCH 130/237] [cachedhcp] Allow cached DHCPACK to apply to
 temporary network devices

Retain a reference to the cached DHCPACK until the late startup phase,
and allow it to be recycled for reuse.  This allows the cached DHCPACK
to be used for a temporary MNP network device and then subsequently
reused for the corresponding real network device.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/cachedhcp.c         | 75 +++++++++++++++++++++++++++++++-----
 src/drivers/net/efi/mnpnet.c |  4 ++
 src/include/ipxe/cachedhcp.h |  2 +
 3 files changed, 71 insertions(+), 10 deletions(-)

diff --git a/src/core/cachedhcp.c b/src/core/cachedhcp.c
index 57226e165..04945e646 100644
--- a/src/core/cachedhcp.c
+++ b/src/core/cachedhcp.c
@@ -46,11 +46,20 @@ struct cached_dhcp_packet {
 	struct dhcp_packet *dhcppkt;
 	/** VLAN tag (if applicable) */
 	unsigned int vlan;
+	/** Flags */
+	unsigned int flags;
 };
 
+/** Cached DHCP packet should be retained */
+#define CACHEDHCP_RETAIN 0x0001
+
+/** Cached DHCP packet has been used */
+#define CACHEDHCP_USED 0x0002
+
 /** Cached DHCPACK */
 struct cached_dhcp_packet cached_dhcpack = {
 	.name = DHCP_SETTINGS_NAME,
+	.flags = CACHEDHCP_RETAIN,
 };
 
 /** Cached ProxyDHCPOFFER */
@@ -101,8 +110,8 @@ static int cachedhcp_apply ( struct cached_dhcp_packet *cache,
 	size_t ll_addr_len;
 	int rc;
 
-	/* Do nothing if cache is empty */
-	if ( ! cache->dhcppkt )
+	/* Do nothing if cache is empty or already in use */
+	if ( ( ! cache->dhcppkt ) || ( cache->flags & CACHEDHCP_USED ) )
 		return 0;
 	chaddr = cache->dhcppkt->dhcphdr->chaddr;
 
@@ -169,8 +178,12 @@ static int cachedhcp_apply ( struct cached_dhcp_packet *cache,
 		return rc;
 	}
 
-	/* Free cached DHCP packet */
-	cachedhcp_free ( cache );
+	/* Mark as used */
+	cache->flags |= CACHEDHCP_USED;
+
+	/* Free cached DHCP packet, if applicable */
+	if ( ! ( cache->flags & CACHEDHCP_RETAIN ) )
+		cachedhcp_free ( cache );
 
 	return 0;
 }
@@ -246,10 +259,10 @@ int cachedhcp_record ( struct cached_dhcp_packet *cache, unsigned int vlan,
 }
 
 /**
- * Cached DHCP packet startup function
+ * Cached DHCP packet early startup function
  *
  */
-static void cachedhcp_startup ( void ) {
+static void cachedhcp_startup_early ( void ) {
 
 	/* Apply cached ProxyDHCPOFFER, if any */
 	cachedhcp_apply ( &cached_proxydhcp, NULL );
@@ -258,6 +271,20 @@ static void cachedhcp_startup ( void ) {
 	/* Apply cached PXEBSACK, if any */
 	cachedhcp_apply ( &cached_pxebs, NULL );
 	cachedhcp_free ( &cached_pxebs );
+}
+
+/**
+ * Cache DHCP packet late startup function
+ *
+ */
+static void cachedhcp_startup_late ( void ) {
+
+	/* Clear retention flag */
+	cached_dhcpack.flags &= ~CACHEDHCP_RETAIN;
+
+	/* Free cached DHCPACK, if used by a network device */
+	if ( cached_dhcpack.flags & CACHEDHCP_USED )
+		cachedhcp_free ( &cached_dhcpack );
 
 	/* Report unclaimed DHCPACK, if any.  Do not free yet, since
 	 * it may still be claimed by a dynamically created device
@@ -284,10 +311,16 @@ static void cachedhcp_shutdown ( int booting __unused ) {
 	cachedhcp_free ( &cached_dhcpack );
 }
 
-/** Cached DHCPACK startup function */
-struct startup_fn cachedhcp_startup_fn __startup_fn ( STARTUP_LATE ) = {
-	.name = "cachedhcp",
-	.startup = cachedhcp_startup,
+/** Cached DHCP packet early startup function */
+struct startup_fn cachedhcp_early_fn __startup_fn ( STARTUP_EARLY ) = {
+	.name = "cachedhcp1",
+	.startup = cachedhcp_startup_early,
+};
+
+/** Cached DHCP packet late startup function */
+struct startup_fn cachedhcp_late_fn __startup_fn ( STARTUP_LATE ) = {
+	.name = "cachedhcp2",
+	.startup = cachedhcp_startup_late,
 	.shutdown = cachedhcp_shutdown,
 };
 
@@ -309,3 +342,25 @@ struct net_driver cachedhcp_driver __net_driver = {
 	.name = "cachedhcp",
 	.probe = cachedhcp_probe,
 };
+
+/**
+ * Recycle cached DHCPACK
+ *
+ * @v netdev		Network device
+ * @v priv		Private data
+ */
+void cachedhcp_recycle ( struct net_device *netdev ) {
+	struct cached_dhcp_packet *cache = &cached_dhcpack;
+	struct settings *settings;
+
+	/* Return DHCPACK to cache, if applicable */
+	settings = find_child_settings ( netdev_settings ( netdev ),
+					 cache->name );
+	if ( cache->dhcppkt && ( settings == &cache->dhcppkt->settings ) ) {
+		DBGC ( colour, "CACHEDHCP %s recycled from %s\n",
+		       cache->name, netdev->name );
+		assert ( cache->flags & CACHEDHCP_USED );
+		unregister_settings ( settings );
+		cache->flags &= ~CACHEDHCP_USED;
+	}
+}
diff --git a/src/drivers/net/efi/mnpnet.c b/src/drivers/net/efi/mnpnet.c
index 84f803f45..eb4b129c7 100644
--- a/src/drivers/net/efi/mnpnet.c
+++ b/src/drivers/net/efi/mnpnet.c
@@ -34,6 +34,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/iobuf.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/ethernet.h>
+#include <ipxe/cachedhcp.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_service.h>
@@ -551,6 +552,9 @@ void mnptemp_destroy ( struct net_device *netdev ) {
 	struct mnp_nic *mnp = netdev->priv;
 	struct efi_device *efidev = mnp->efidev;
 
+	/* Recycle any cached DHCP packet */
+	cachedhcp_recycle ( netdev );
+
 	/* Stop temporary network device */
 	mnpnet_stop ( efidev );
 
diff --git a/src/include/ipxe/cachedhcp.h b/src/include/ipxe/cachedhcp.h
index 4ce4a9f2a..8ebee3b7b 100644
--- a/src/include/ipxe/cachedhcp.h
+++ b/src/include/ipxe/cachedhcp.h
@@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <stddef.h>
 #include <ipxe/uaccess.h>
 
+struct net_device;
 struct cached_dhcp_packet;
 
 extern struct cached_dhcp_packet cached_dhcpack;
@@ -21,5 +22,6 @@ extern struct cached_dhcp_packet cached_pxebs;
 extern int cachedhcp_record ( struct cached_dhcp_packet *cache,
 			      unsigned int vlan, userptr_t data,
 			      size_t max_len );
+extern void cachedhcp_recycle ( struct net_device *netdev );
 
 #endif /* _IPXE_CACHEDHCP_H */

From 165995b7e917a94533799fe43d14f5e3b1fef285 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 29 Mar 2024 13:03:38 +0000
Subject: [PATCH 131/237] [efi] Restructure handling of autoexec.ipxe script

We currently attempt to obtain the autoexec.ipxe script via early use
of the EFI_SIMPLE_FILE_SYSTEM_PROTOCOL or EFI_PXE_BASE_CODE_PROTOCOL
interfaces to obtain an opaque block of memory, which is then
registered as an image at an appropriate point during our startup
sequence.  The early use of these existent interfaces allows us to
obtain the script even if our subsequent actions (e.g. disconnecting
drivers in order to connect up our own) may cause the script to become
inaccessible.

This mirrors the approach used under BIOS, where the autoexec.ipxe
script is provided by the prefix (e.g. as an initrd image when using
the .lkrn build of iPXE) and so must be copied into a normally
allocated image from wherever it happens to previously exist in
memory.

We do not currently have support for downloading an autoexec.ipxe
script if we were ourselves downloaded via UEFI HTTP boot.

There is an EFI_HTTP_PROTOCOL defined within the UEFI specification,
but it is so poorly designed as to be unusable for the simple purpose
of downloading an additional file from the same directory.  It
provides almost nothing more than a very slim wrapper around
EFI_TCP4_PROTOCOL (or EFI_TCP6_PROTOCOL).  It will not handle
redirection, content encoding, retries, or even fundamentals such as
the Content-Length header, leaving all of this up to the caller.

The UEFI HTTP Boot driver will install an EFI_LOAD_FILE_PROTOCOL
instance on the loaded image's device handle.  This looks promising at
first since it provides the LoadFile() API call which is specified to
accept an arbitrary filename parameter.  However, experimentation (and
inspection of the code in EDK2) reveals a multitude of problems that
prevent this from being usable.  Calling LoadFile() will idiotically
restart the entire DHCP process (and potentially pop up a UI requiring
input from the user for e.g. a wireless network password).  The
filename provided to LoadFile() will be ignored.  Any downloaded file
will be rejected unless it happens to match one of the limited set of
types expected by the UEFI HTTP Boot driver.  The list of design
failures and conceptual mismatches is fairly impressive.

Choose to bypass every possible aspect of UEFI HTTP support, and
instead use our own HTTP client and network stack to download the
autoexec.ipxe script over a temporary MNP network device.  Since this
approach works for TFTP as well as HTTP, drop the direct use of
EFI_PXE_BASE_CODE_PROTOCOL.  For consistency and simplicity, also drop
the direct use of EFI_SIMPLE_FILE_SYSTEM_PROTOCOL and rely upon our
existing support to access local files via "file:" URIs.

This approach results in console output during the "iPXE initialising
devices...ok" message that appears while startup is in progress.
Remove the trailing "ok" so that this intermediate output appears at a
sensible location on the screen.  The welcome banner that will be
printed immediately afterwards provides an indication that startup has
completed successfully even absent the explicit "ok".

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/main.c                     |   3 +-
 src/include/ipxe/efi/efi_autoexec.h |   5 +-
 src/interface/efi/efi_autoexec.c    | 495 +++++++---------------------
 src/interface/efi/efiprefix.c       |  11 +-
 4 files changed, 127 insertions(+), 387 deletions(-)

diff --git a/src/core/main.c b/src/core/main.c
index 638dea9cf..3db836491 100644
--- a/src/core/main.c
+++ b/src/core/main.c
@@ -32,9 +32,8 @@ __asmcall int main ( void ) {
 	initialise();
 
 	/* Some devices take an unreasonably long time to initialise */
-	printf ( "%s initialising devices...", product_short_name );
+	printf ( "%s initialising devices...\n", product_short_name );
 	startup();
-	printf ( "ok\n" );
 
 	/* Attempt to boot */
 	if ( ( rc = ipxe ( NULL ) ) != 0 )
diff --git a/src/include/ipxe/efi/efi_autoexec.h b/src/include/ipxe/efi/efi_autoexec.h
index 08ddf5836..18bc4200c 100644
--- a/src/include/ipxe/efi/efi_autoexec.h
+++ b/src/include/ipxe/efi/efi_autoexec.h
@@ -9,9 +9,6 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#include <ipxe/efi/efi.h>
-
-extern int efi_autoexec_load ( EFI_HANDLE device,
-			       EFI_DEVICE_PATH_PROTOCOL *path );
+extern int efi_autoexec_load ( void );
 
 #endif /* _IPXE_EFI_AUTOEXEC_H */
diff --git a/src/interface/efi/efi_autoexec.c b/src/interface/efi/efi_autoexec.c
index fb12cef08..d9ad3b990 100644
--- a/src/interface/efi/efi_autoexec.c
+++ b/src/interface/efi/efi_autoexec.c
@@ -24,16 +24,16 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
-#include <stdlib.h>
 #include <errno.h>
+#include <ipxe/timer.h>
 #include <ipxe/image.h>
-#include <ipxe/init.h>
-#include <ipxe/in.h>
+#include <ipxe/netdevice.h>
 #include <ipxe/efi/efi.h>
+#include <ipxe/efi/efi_utils.h>
 #include <ipxe/efi/efi_autoexec.h>
-#include <ipxe/efi/Protocol/PxeBaseCode.h>
-#include <ipxe/efi/Protocol/SimpleFileSystem.h>
-#include <ipxe/efi/Guid/FileInfo.h>
+#include <ipxe/efi/mnpnet.h>
+#include <usr/imgmgmt.h>
+#include <usr/sync.h>
 
 /** @file
  *
@@ -41,413 +41,160 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
-/** Autoexec script filename */
-static wchar_t efi_autoexec_wname[] = L"autoexec.ipxe";
+/** Timeout for autoexec script downloads */
+#define EFI_AUTOEXEC_TIMEOUT ( 2 * TICKS_PER_SEC )
 
 /** Autoexec script image name */
-static char efi_autoexec_name[] = "autoexec.ipxe";
+#define EFI_AUTOEXEC_NAME "autoexec.ipxe"
 
-/** Autoexec script (if any) */
-static void *efi_autoexec;
-
-/** Autoexec script length */
-static size_t efi_autoexec_len;
+/** An EFI autoexec script loader */
+struct efi_autoexec_loader {
+	/** Required protocol GUID */
+	EFI_GUID *protocol;
+	/**
+	 * Load autoexec script
+	 *
+	 * @v handle		Handle on which protocol was found
+	 * @v image		Image to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * load ) ( EFI_HANDLE handle, struct image **image );
+};
 
 /**
- * Load autoexec script from path within filesystem
+ * Load autoexec script from filesystem
  *
- * @v device		Device handle
- * @v path		Relative path to image, or NULL to load from root
+ * @v handle		Simple filesystem protocol handle
+ * @v image		Image to fill in
  * @ret rc		Return status code
  */
-static int efi_autoexec_filesystem ( EFI_HANDLE device,
-				     EFI_DEVICE_PATH_PROTOCOL *path ) {
-	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	union {
-		void *interface;
-		EFI_SIMPLE_FILE_SYSTEM_PROTOCOL *fs;
-	} u;
-	struct {
-		EFI_FILE_INFO info;
-		CHAR16 name[ sizeof ( efi_autoexec_wname ) /
-			     sizeof ( efi_autoexec_wname[0] ) ];
-	} info;
-	FILEPATH_DEVICE_PATH *filepath;
-	EFI_FILE_PROTOCOL *root;
-	EFI_FILE_PROTOCOL *file;
-	UINTN size;
-	VOID *data;
-	unsigned int dirlen;
-	size_t len;
-	CHAR16 *wname;
-	EFI_STATUS efirc;
+static int efi_autoexec_filesystem ( EFI_HANDLE handle, struct image **image ) {
+	EFI_HANDLE device = efi_loaded_image->DeviceHandle;
 	int rc;
 
-	/* Identify directory */
-	if ( path ) {
-
-		/* Check relative device path is a file path */
-		if ( ! ( ( path->Type == MEDIA_DEVICE_PATH ) &&
-			 ( path->SubType == MEDIA_FILEPATH_DP ) ) ) {
-			DBGC ( device, "EFI %s image path ",
-			       efi_handle_name ( device ) );
-			DBGC ( device, " \"%s\" is not a file path\n",
-			       efi_devpath_text ( path ) );
-			rc = -ENOTTY;
-			goto err_not_filepath;
-		}
-		filepath = container_of ( path, FILEPATH_DEVICE_PATH, Header );
-
-		/* Find length of containing directory */
-		dirlen = ( ( ( ( path->Length[1] << 8 ) | path->Length[0] )
-			     - offsetof ( typeof ( *filepath ), PathName ) )
-			   / sizeof ( filepath->PathName[0] ) );
-		for ( ; dirlen ; dirlen-- ) {
-			if ( filepath->PathName[ dirlen - 1 ] == L'\\' )
-				break;
-		}
-
-	} else {
-
-		/* Use root directory */
-		filepath = NULL;
-		dirlen = 0;
+	/* Check that we were loaded from a filesystem */
+	if ( handle != device ) {
+		DBGC ( device, "EFI %s is not the file system handle\n",
+		       efi_handle_name ( device ) );
+		return -ENOTTY;
 	}
 
-	/* Allocate filename */
-	len = ( ( dirlen * sizeof ( wname[0] ) ) + sizeof ( efi_autoexec_wname ) );
-	wname = malloc ( len );
-	if ( ! wname ) {
-		rc = -ENOMEM;
-		goto err_wname;
-	}
-	memcpy ( wname, filepath->PathName, ( dirlen * sizeof ( wname[0] ) ) );
-	memcpy ( &wname[dirlen], efi_autoexec_wname, sizeof ( efi_autoexec_wname ) );
+	/* Try loading from loaded image directory, if supported */
+	if ( ( rc = imgacquire ( "file:" EFI_AUTOEXEC_NAME,
+				 EFI_AUTOEXEC_TIMEOUT, image ) ) == 0 )
+		return 0;
 
-	/* Open simple file system protocol */
-	if ( ( efirc = bs->OpenProtocol ( device,
-					  &efi_simple_file_system_protocol_guid,
-					  &u.interface, efi_image_handle,
-					  device,
-					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s has no filesystem instance: %s\n",
+	/* Try loading from root directory, if supported */
+	if ( ( rc = imgacquire ( "file:/" EFI_AUTOEXEC_NAME,
+				 EFI_AUTOEXEC_TIMEOUT, image ) ) == 0 )
+		return 0;
+
+	return rc;
+}
+
+/**
+ * Load autoexec script via temporary network device
+ *
+ * @v handle		Managed network protocol service binding handle
+ * @v image		Image to fill in
+ * @ret rc		Return status code
+ */
+static int efi_autoexec_network ( EFI_HANDLE handle, struct image **image ) {
+	EFI_HANDLE device = efi_loaded_image->DeviceHandle;
+	struct net_device *netdev;
+	int rc;
+
+	/* Create temporary network device */
+	if ( ( rc = mnptemp_create ( handle, &netdev ) ) != 0 ) {
+		DBGC ( device, "EFI %s could not create net device: %s\n",
 		       efi_handle_name ( device ), strerror ( rc ) );
-		goto err_filesystem;
+		goto err_create;
 	}
 
-	/* Open root directory */
-	if ( ( efirc = u.fs->OpenVolume ( u.fs, &root ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s could not open volume: %s\n",
+	/* Open network device */
+	if ( ( rc = netdev_open ( netdev ) ) != 0 ) {
+		DBGC ( device, "EFI %s could not open net device: %s\n",
 		       efi_handle_name ( device ), strerror ( rc ) );
-		goto err_volume;
-	}
-
-	/* Open autoexec script */
-	if ( ( efirc = root->Open ( root, &file, wname,
-				    EFI_FILE_MODE_READ, 0 ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s has no %ls: %s\n",
-		       efi_handle_name ( device ), wname, strerror ( rc ) );
 		goto err_open;
 	}
 
-	/* Get file information */
-	size = sizeof ( info );
-	if ( ( efirc = file->GetInfo ( file, &efi_file_info_id, &size,
-				       &info ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s could not get %ls info: %s\n",
-		       efi_handle_name ( device ), wname, strerror ( rc ) );
-		goto err_getinfo;
-	}
-	size = info.info.FileSize;
-
-	/* Ignore zero-length files */
-	if ( ! size ) {
-		rc = -EINVAL;
-		DBGC ( device, "EFI %s has zero-length %ls\n",
-		       efi_handle_name ( device ), wname );
-		goto err_empty;
+	/* Attempt download */
+	rc = imgacquire ( EFI_AUTOEXEC_NAME, EFI_AUTOEXEC_TIMEOUT, image );
+	if ( rc != 0 ) {
+		DBGC ( device, "EFI %s could not download %s: %s\n",
+		       efi_handle_name ( device ), EFI_AUTOEXEC_NAME,
+		       strerror ( rc ) );
 	}
 
-	/* Allocate temporary copy */
-	if ( ( efirc = bs->AllocatePool ( EfiBootServicesData, size,
-					  &data ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s could not allocate %ls: %s\n",
-		       efi_handle_name ( device ), wname, strerror ( rc ) );
-		goto err_alloc;
-	}
+	/* Ensure network exchanges have completed */
+	sync ( EFI_AUTOEXEC_TIMEOUT );
 
-	/* Read file */
-	if ( ( efirc = file->Read ( file, &size, data ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s could not read %ls: %s\n",
-		       efi_handle_name ( device ), wname, strerror ( rc ) );
-		goto err_read;
-	}
-
-	/* Record autoexec script */
-	efi_autoexec = data;
-	efi_autoexec_len = size;
-	data = NULL;
-	DBGC ( device, "EFI %s found %ls\n", efi_handle_name ( device ), wname );
-
-	/* Success */
-	rc = 0;
-
- err_read:
-	if ( data )
-		bs->FreePool ( data );
- err_alloc:
- err_empty:
- err_getinfo:
-	file->Close ( file );
  err_open:
-	root->Close ( root );
- err_volume:
-	bs->CloseProtocol ( device, &efi_simple_file_system_protocol_guid,
-			    efi_image_handle, device );
- err_filesystem:
-	free ( wname );
- err_wname:
- err_not_filepath:
+	mnptemp_destroy ( netdev );
+ err_create:
 	return rc;
 }
 
-/**
- * Load autoexec script from TFTP server
- *
- * @v device		Device handle
- * @ret rc		Return status code
- */
-static int efi_autoexec_tftp ( EFI_HANDLE device ) {
-	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	union {
-		void *interface;
-		EFI_PXE_BASE_CODE_PROTOCOL *pxe;
-	} u;
-	EFI_PXE_BASE_CODE_MODE *mode;
-	EFI_PXE_BASE_CODE_PACKET *packet;
-	union {
-		struct in_addr in;
-		EFI_IP_ADDRESS ip;
-	} server;
-	size_t filename_max;
-	char *filename;
-	char *sep;
-	UINT64 size;
-	VOID *data;
-	EFI_STATUS efirc;
-	int rc;
-
-	/* Open PXE base code protocol */
-	if ( ( efirc = bs->OpenProtocol ( device,
-					  &efi_pxe_base_code_protocol_guid,
-					  &u.interface, efi_image_handle,
-					  device,
-					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s has no PXE base code instance: %s\n",
-		       efi_handle_name ( device ), strerror ( rc ) );
-		goto err_pxe;
-	}
-
-	/* Do not attempt to parse DHCPv6 packets */
-	mode = u.pxe->Mode;
-	if ( mode->UsingIpv6 ) {
-		rc = -ENOTSUP;
-		DBGC ( device, "EFI %s has IPv6 PXE base code\n",
-		       efi_handle_name ( device ) );
-		goto err_ipv6;
-	}
-
-	/* Identify relevant reply packet */
-	if ( mode->PxeReplyReceived &&
-	     mode->PxeReply.Dhcpv4.BootpBootFile[0] ) {
-		/* Use boot filename if present in PXE reply */
-		DBGC ( device, "EFI %s using PXE reply filename\n",
-		       efi_handle_name ( device ) );
-		packet = &mode->PxeReply;
-	} else if ( mode->DhcpAckReceived &&
-		    mode->DhcpAck.Dhcpv4.BootpBootFile[0] ) {
-		/* Otherwise, use boot filename if present in DHCPACK */
-		DBGC ( device, "EFI %s using DHCPACK filename\n",
-		       efi_handle_name ( device ) );
-		packet = &mode->DhcpAck;
-	} else if ( mode->ProxyOfferReceived &&
-		    mode->ProxyOffer.Dhcpv4.BootpBootFile[0] ) {
-		/* Otherwise, use boot filename if present in ProxyDHCPOFFER */
-		DBGC ( device, "EFI %s using ProxyDHCPOFFER filename\n",
-		       efi_handle_name ( device ) );
-		packet = &mode->ProxyOffer;
-	} else {
-		/* No boot filename available */
-		rc = -ENOENT;
-		DBGC ( device, "EFI %s has no PXE boot filename\n",
-		       efi_handle_name ( device ) );
-		goto err_packet;
-	}
-
-	/* Allocate filename */
-	filename_max = ( sizeof ( packet->Dhcpv4.BootpBootFile )
-			 + ( sizeof ( efi_autoexec_name ) - 1 /* NUL */ )
-			 + 1 /* NUL */ );
-	filename = zalloc ( filename_max );
-	if ( ! filename ) {
-		rc = -ENOMEM;
-		goto err_filename;
-	}
-
-	/* Extract next-server address and boot filename */
-	memset ( &server, 0, sizeof ( server ) );
-	memcpy ( &server.in, packet->Dhcpv4.BootpSiAddr,
-		 sizeof ( server.in ) );
-	memcpy ( filename, packet->Dhcpv4.BootpBootFile,
-		 sizeof ( packet->Dhcpv4.BootpBootFile ) );
-
-	/* Update filename to autoexec script name */
-	sep = strrchr ( filename, '/' );
-	if ( ! sep )
-		sep = strrchr ( filename, '\\' );
-	if ( ! sep )
-		sep = ( filename - 1 );
-	strcpy ( ( sep + 1 ), efi_autoexec_name );
-
-	/* Get file size */
-	if ( ( efirc = u.pxe->Mtftp ( u.pxe,
-				      EFI_PXE_BASE_CODE_TFTP_GET_FILE_SIZE,
-				      NULL, FALSE, &size, NULL, &server.ip,
-				      ( ( UINT8 * ) filename ), NULL,
-				      FALSE ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s could not get size of %s:%s: %s\n",
-		       efi_handle_name ( device ), inet_ntoa ( server.in ),
-		       filename, strerror ( rc ) );
-		goto err_size;
-	}
-
-	/* Ignore zero-length files */
-	if ( ! size ) {
-		rc = -EINVAL;
-		DBGC ( device, "EFI %s has zero-length %s:%s\n",
-		       efi_handle_name ( device ), inet_ntoa ( server.in ),
-		       filename );
-		goto err_empty;
-	}
-
-	/* Allocate temporary copy */
-	if ( ( efirc = bs->AllocatePool ( EfiBootServicesData, size,
-					  &data ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s could not allocate %s:%s: %s\n",
-		       efi_handle_name ( device ), inet_ntoa ( server.in ),
-		       filename, strerror ( rc ) );
-		goto err_alloc;
-	}
-
-	/* Download file */
-	if ( ( efirc = u.pxe->Mtftp ( u.pxe, EFI_PXE_BASE_CODE_TFTP_READ_FILE,
-				      data, FALSE, &size, NULL, &server.ip,
-				      ( ( UINT8 * ) filename ), NULL,
-				      FALSE ) ) != 0 ) {
-		rc = -EEFI ( efirc );
-		DBGC ( device, "EFI %s could not download %s:%s: %s\n",
-		       efi_handle_name ( device ), inet_ntoa ( server.in ),
-		       filename, strerror ( rc ) );
-		goto err_download;
-	}
-
-	/* Record autoexec script */
-	efi_autoexec = data;
-	efi_autoexec_len = size;
-	data = NULL;
-	DBGC ( device, "EFI %s found %s:%s\n", efi_handle_name ( device ),
-	       inet_ntoa ( server.in ), filename );
-
-	/* Success */
-	rc = 0;
-
- err_download:
-	if ( data )
-		bs->FreePool ( data );
- err_alloc:
- err_empty:
- err_size:
-	free ( filename );
- err_filename:
- err_packet:
- err_ipv6:
-	bs->CloseProtocol ( device, &efi_pxe_base_code_protocol_guid,
-			    efi_image_handle, device );
- err_pxe:
-	return rc;
-}
+/** Autoexec script loaders */
+static struct efi_autoexec_loader efi_autoexec_loaders[] = {
+	{
+		.protocol = &efi_simple_file_system_protocol_guid,
+		.load = efi_autoexec_filesystem,
+	},
+	{
+		.protocol = &efi_managed_network_service_binding_protocol_guid,
+		.load = efi_autoexec_network,
+	},
+};
 
 /**
  * Load autoexec script
  *
- * @v device		Device handle
- * @v path		Image path within device handle
  * @ret rc		Return status code
  */
-int efi_autoexec_load ( EFI_HANDLE device,
-			EFI_DEVICE_PATH_PROTOCOL *path ) {
+int efi_autoexec_load ( void ) {
+	EFI_HANDLE device = efi_loaded_image->DeviceHandle;
+	EFI_HANDLE handle;
+	struct efi_autoexec_loader *loader;
+	struct image *image;
+	unsigned int i;
 	int rc;
 
-	/* Sanity check */
-	assert ( efi_autoexec == NULL );
-	assert ( efi_autoexec_len == 0 );
+	/* Use first applicable loader */
+	for ( i = 0 ; i < ( sizeof ( efi_autoexec_loaders ) /
+			    sizeof ( efi_autoexec_loaders[0] ) ) ; i ++ ) {
 
-	/* Try loading from file system loaded image directory, if supported */
-	if ( ( rc = efi_autoexec_filesystem ( device, path ) ) == 0 )
-		return 0;
+		/* Locate required protocol for this loader */
+		loader = &efi_autoexec_loaders[i];
+		if ( ( rc = efi_locate_device ( device, loader->protocol,
+						&handle, 0 ) ) != 0 ) {
+			DBGC ( device, "EFI %s found no %s: %s\n",
+			       efi_handle_name ( device ),
+			       efi_guid_ntoa ( loader->protocol ),
+			       strerror ( rc ) );
+			continue;
+		}
+		DBGC ( device, "EFI %s found %s on ",
+		       efi_handle_name ( device ),
+		       efi_guid_ntoa ( loader->protocol ) );
+		DBGC ( device, "%s\n", efi_handle_name ( handle ) );
 
-	/* Try loading from file system root directory, if supported */
-	if ( ( rc = efi_autoexec_filesystem ( device, NULL ) ) == 0 )
-		return 0;
+		/* Try loading */
+		if ( ( rc = loader->load ( handle, &image ) ) != 0 )
+			return rc;
 
-	/* Try loading via TFTP, if supported */
-	if ( ( rc = efi_autoexec_tftp ( device ) ) == 0 )
+		/* Discard zero-length images */
+		if ( ! image->len ) {
+			DBGC ( device, "EFI %s discarding zero-length %s\n",
+			       efi_handle_name ( device ), image->name );
+			unregister_image ( image );
+			return -ENOENT;
+		}
+
+		DBGC ( device, "EFI %s loaded %s (%zd bytes)\n",
+		       efi_handle_name ( device ), image->name, image->len );
 		return 0;
+	}
 
 	return -ENOENT;
 }
-
-/**
- * Register autoexec script
- *
- */
-static void efi_autoexec_startup ( void ) {
-	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	EFI_HANDLE device = efi_loaded_image->DeviceHandle;
-	struct image *image;
-
-	/* Do nothing if we have no autoexec script */
-	if ( ! efi_autoexec )
-		return;
-
-	/* Create autoexec image */
-	image = image_memory ( efi_autoexec_name,
-			       virt_to_user ( efi_autoexec ),
-			       efi_autoexec_len );
-	if ( ! image ) {
-		DBGC ( device, "EFI %s could not create %s\n",
-		       efi_handle_name ( device ), efi_autoexec_name );
-		return;
-	}
-	DBGC ( device, "EFI %s registered %s\n",
-	       efi_handle_name ( device ), efi_autoexec_name );
-
-	/* Free temporary copy */
-	bs->FreePool ( efi_autoexec );
-	efi_autoexec = NULL;
-}
-
-/** Autoexec script startup function */
-struct startup_fn efi_autoexec_startup_fn __startup_fn ( STARTUP_NORMAL ) = {
-	.name = "efi_autoexec",
-	.startup = efi_autoexec_startup,
-};
diff --git a/src/interface/efi/efiprefix.c b/src/interface/efi/efiprefix.c
index f6395b65b..10d8f0bf6 100644
--- a/src/interface/efi/efiprefix.c
+++ b/src/interface/efi/efiprefix.c
@@ -81,25 +81,19 @@ EFI_STATUS EFIAPI _efi_start ( EFI_HANDLE image_handle,
 static void efi_init_application ( void ) {
 	EFI_HANDLE device = efi_loaded_image->DeviceHandle;
 	EFI_DEVICE_PATH_PROTOCOL *devpath = efi_loaded_image_path;
-	EFI_DEVICE_PATH_PROTOCOL *filepath = efi_loaded_image->FilePath;
 	struct uri *uri;
 
 	/* Set current working URI from device path, if present */
 	uri = efi_path_uri ( devpath );
 	if ( uri )
 		churi ( uri );
+	uri_put ( uri );
 
 	/* Identify autoboot device, if any */
 	efi_set_autoboot_ll_addr ( device, devpath );
 
 	/* Store cached DHCP packet, if any */
 	efi_cachedhcp_record ( device, devpath );
-
-	/* Load autoexec script, if any */
-	efi_autoexec_load ( device, filepath );
-
-	/* Drop temporary reference to URI */
-	uri_put ( uri );
 }
 
 /** EFI application initialisation function */
@@ -114,6 +108,9 @@ struct init_fn efi_init_application_fn __init_fn ( INIT_NORMAL ) = {
  */
 static int efi_probe ( struct root_device *rootdev __unused ) {
 
+	/* Try loading autoexec script */
+	efi_autoexec_load();
+
 	/* Remove any vetoed drivers */
 	efi_veto();
 

From 59f27d69358efc919b50760f3d6dac0b637b5488 Mon Sep 17 00:00:00 2001
From: Pavel Krotkiy <porsh@nebius.com>
Date: Wed, 3 Apr 2024 12:52:56 +0100
Subject: [PATCH 132/237] [netdevice] Add "linktype" setting

Add a new setting to provide access to the link layer protocol type
from scripts.  This can be useful in order to skip configuring
interfaces based on their link layer protocol or, conversely,
configure only selected interface types (Ethernet, IPoIB, etc.)

Example script:

    set idx:int32 0
    :loop
    isset ${net${idx}/mac} || exit 0
    iseq ${net${idx}/linktype} IPoIB && goto try_next ||
    autoboot net${idx} ||
    :try_next
    inc idx && goto loop

Signed-off-by: Pavel Krotkiy <porsh@nebius.com>
Modified-by: Michael Brown <mcb30@ipxe.org>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/settings.h |  2 ++
 src/net/netdev_settings.c   | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/src/include/ipxe/settings.h b/src/include/ipxe/settings.h
index 424188deb..0301da12e 100644
--- a/src/include/ipxe/settings.h
+++ b/src/include/ipxe/settings.h
@@ -471,6 +471,8 @@ mac_setting __setting ( SETTING_NETDEV, mac );
 extern const struct setting
 busid_setting __setting ( SETTING_NETDEV, busid );
 extern const struct setting
+linktype_setting __setting ( SETTING_NETDEV, linktype );
+extern const struct setting
 user_class_setting __setting ( SETTING_HOST_EXTRA, user-class );
 extern const struct setting
 vendor_class_setting __setting ( SETTING_HOST_EXTRA, vendor-class );
diff --git a/src/net/netdev_settings.c b/src/net/netdev_settings.c
index fb98663ca..080b6d2a5 100644
--- a/src/net/netdev_settings.c
+++ b/src/net/netdev_settings.c
@@ -65,6 +65,11 @@ const struct setting busid_setting __setting ( SETTING_NETDEV, busid ) = {
 	.description = "Bus ID",
 	.type = &setting_type_hex,
 };
+const struct setting linktype_setting __setting ( SETTING_NETDEV, linktype ) = {
+	.name = "linktype",
+	.description = "Link-layer type",
+	.type = &setting_type_string,
+};
 const struct setting chip_setting __setting ( SETTING_NETDEV, chip ) = {
 	.name = "chip",
 	.description = "Chip",
@@ -219,6 +224,22 @@ static int netdev_fetch_busid ( struct net_device *netdev, void *data,
 	return sizeof ( dhcp_desc );
 }
 
+/**
+ * Fetch link layer type setting
+ *
+ * @v netdev		Network device
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int netdev_fetch_linktype ( struct net_device *netdev, void *data,
+				   size_t len ) {
+	const char *linktype = netdev->ll_protocol->name;
+
+	strncpy ( data, linktype, len );
+	return strlen ( linktype );
+}
+
 /**
  * Fetch chip setting
  *
@@ -281,6 +302,7 @@ static struct netdev_setting_operation netdev_setting_operations[] = {
 	{ &bustype_setting, NULL, netdev_fetch_bustype },
 	{ &busloc_setting, NULL, netdev_fetch_busloc },
 	{ &busid_setting, NULL, netdev_fetch_busid },
+	{ &linktype_setting, NULL, netdev_fetch_linktype },
 	{ &chip_setting, NULL, netdev_fetch_chip },
 	{ &ifname_setting, NULL, netdev_fetch_ifname },
 };

From 27ecc36c0bef804d12dbf8c29684c8e8159c8e47 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 15 Apr 2024 14:28:38 +0100
Subject: [PATCH 133/237] [efi] Do not attempt to download autoexec.ipxe
 without a valid base URI

If we do not have a current working URI (after applying the EFI device
path settings and any cached DHCP settings), then an attempt to
download autoexec.ipxe will fail since there is no base URI from which
to resolve the full autoexec.ipxe URI.

Avoid this potentially confusing error message by attempting the
download only if we have successfully obtained a current working URI.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_autoexec.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/interface/efi/efi_autoexec.c b/src/interface/efi/efi_autoexec.c
index d9ad3b990..44b8b645b 100644
--- a/src/interface/efi/efi_autoexec.c
+++ b/src/interface/efi/efi_autoexec.c
@@ -28,6 +28,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/timer.h>
 #include <ipxe/image.h>
 #include <ipxe/netdevice.h>
+#include <ipxe/uri.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_utils.h>
 #include <ipxe/efi/efi_autoexec.h>
@@ -111,6 +112,14 @@ static int efi_autoexec_network ( EFI_HANDLE handle, struct image **image ) {
 		goto err_create;
 	}
 
+	/* Do nothing unless we have a usable current working URI */
+	if ( ! cwuri ) {
+		DBGC ( device, "EFI %s has no current working URI\n",
+		       efi_handle_name ( device ) );
+		rc = -ENOTTY;
+		goto err_cwuri;
+	}
+
 	/* Open network device */
 	if ( ( rc = netdev_open ( netdev ) ) != 0 ) {
 		DBGC ( device, "EFI %s could not open net device: %s\n",
@@ -130,6 +139,7 @@ static int efi_autoexec_network ( EFI_HANDLE handle, struct image **image ) {
 	sync ( EFI_AUTOEXEC_TIMEOUT );
 
  err_open:
+ err_cwuri:
 	mnptemp_destroy ( netdev );
  err_create:
 	return rc;

From 40b51124400df9cb0c57512ff93ac21827d5bac0 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 15 Apr 2024 15:59:49 +0100
Subject: [PATCH 134/237] [hci] Use dynamically allocated buffers for editable
 strings

Editable strings currently require a fixed-size buffer, which is
inelegant and limits the potential for creating interactive forms with
a variable number of edit box widgets.

Remove this limitation by switching to using a dynamically allocated
buffer for editable strings and edit box widgets.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/editstring.c               | 144 ++++++++++++++++++++---------
 src/hci/mucurses/widgets/editbox.c |  14 +--
 src/hci/readline.c                 |  48 ++++------
 src/hci/tui/login_ui.c             |  39 ++++----
 src/hci/tui/settings_ui.c          |  60 ++++++------
 src/include/ipxe/editbox.h         |   2 +-
 src/include/ipxe/editstring.h      |  33 ++++---
 src/include/ipxe/errfile.h         |   1 +
 8 files changed, 199 insertions(+), 142 deletions(-)

diff --git a/src/hci/editstring.c b/src/hci/editstring.c
index 8cbce0767..b83916c6e 100644
--- a/src/hci/editstring.c
+++ b/src/hci/editstring.c
@@ -24,8 +24,10 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <assert.h>
+#include <errno.h>
 #include <string.h>
 #include <ctype.h>
+#include <stdlib.h>
 #include <ipxe/keys.h>
 #include <ipxe/editstring.h>
 
@@ -35,17 +37,17 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
-static void insert_delete ( struct edit_string *string, size_t delete_len,
-                            const char *insert_text ) 
-			    __attribute__ (( nonnull (1) ));
-static void insert_character ( struct edit_string *string,
-                               unsigned int character ) __nonnull;
-static void delete_character ( struct edit_string *string ) __nonnull;
-static void backspace ( struct edit_string *string ) __nonnull;
-static void previous_word ( struct edit_string *string ) __nonnull;
-static void kill_word ( struct edit_string *string ) __nonnull;
-static void kill_sol ( struct edit_string *string ) __nonnull;
-static void kill_eol ( struct edit_string *string ) __nonnull;
+static __attribute__ (( nonnull ( 1 ) )) int
+insert_delete ( struct edit_string *string, size_t delete_len,
+		const char *insert_text );
+static __nonnull int insert_character ( struct edit_string *string,
+					unsigned int character );
+static __nonnull void delete_character ( struct edit_string *string );
+static __nonnull void backspace ( struct edit_string *string );
+static __nonnull void previous_word ( struct edit_string *string );
+static __nonnull void kill_word ( struct edit_string *string );
+static __nonnull void kill_sol ( struct edit_string *string );
+static __nonnull void kill_eol ( struct edit_string *string );
 
 /**
  * Insert and/or delete text within an editable string
@@ -53,35 +55,56 @@ static void kill_eol ( struct edit_string *string ) __nonnull;
  * @v string		Editable string
  * @v delete_len	Length of text to delete from current cursor position
  * @v insert_text	Text to insert at current cursor position, or NULL
+ * @ret rc		Return status code
  */
-static void insert_delete ( struct edit_string *string, size_t delete_len,
-			    const char *insert_text ) {
-	size_t old_len, max_delete_len, insert_len, max_insert_len, new_len;
+static int insert_delete ( struct edit_string *string, size_t delete_len,
+			   const char *insert_text ) {
+	size_t old_len, max_delete_len, insert_len, new_len;
+	char *buf;
+	char *tmp;
 
 	/* Calculate lengths */
-	old_len = strlen ( string->buf );
+	buf = *(string->buf);
+	old_len = ( buf ? strlen ( buf ) : 0 );
 	assert ( string->cursor <= old_len );
 	max_delete_len = ( old_len - string->cursor );
 	if ( delete_len > max_delete_len )
 		delete_len = max_delete_len;
 	insert_len = ( insert_text ? strlen ( insert_text ) : 0 );
-	max_insert_len = ( ( string->len - 1 ) - ( old_len - delete_len ) );
-	if ( insert_len > max_insert_len )
-		insert_len = max_insert_len;
 	new_len = ( old_len - delete_len + insert_len );
 
+	/* Reallocate string, ignoring failures if shrinking */
+	tmp = realloc ( buf, ( new_len + 1 /* NUL */ ) );
+	if ( tmp ) {
+		buf = tmp;
+		*(string->buf) = buf;
+	} else if ( new_len > old_len ) {
+		return -ENOMEM;
+	}
+
 	/* Fill in edit history */
 	string->mod_start = string->cursor;
 	string->mod_end = ( ( new_len > old_len ) ? new_len : old_len );
 
 	/* Move data following the cursor */
-	memmove ( ( string->buf + string->cursor + insert_len ),
-		  ( string->buf + string->cursor + delete_len ),
-		  ( max_delete_len + 1 - delete_len ) );
+	memmove ( ( buf + string->cursor + insert_len ),
+		  ( buf + string->cursor + delete_len ),
+		  ( max_delete_len - delete_len ) );
 
 	/* Copy inserted text to cursor position */
-	memcpy ( ( string->buf + string->cursor ), insert_text, insert_len );
+	memcpy ( ( buf + string->cursor ), insert_text, insert_len );
 	string->cursor += insert_len;
+
+	/* Terminate string (if not NULL) */
+	if ( buf ) {
+		buf[new_len] = '\0';
+	} else {
+		assert ( old_len == 0 );
+		assert ( insert_len == 0 );
+		assert ( new_len == 0 );
+	}
+
+	return 0;
 }
 
 /**
@@ -89,11 +112,13 @@ static void insert_delete ( struct edit_string *string, size_t delete_len,
  *
  * @v string		Editable string
  * @v character		Character to insert
+ * @ret rc		Return status code
  */
-static void insert_character ( struct edit_string *string,
+static int insert_character ( struct edit_string *string,
 			      unsigned int character ) {
 	char insert_text[2] = { character, '\0' };
-	insert_delete ( string, 0, insert_text );
+
+	return insert_delete ( string, 0, insert_text );
 }
 
 /**
@@ -102,7 +127,10 @@ static void insert_character ( struct edit_string *string,
  * @v string		Editable string
  */
 static void delete_character ( struct edit_string *string ) {
-	insert_delete ( string, 1, NULL );
+	int rc;
+
+	rc = insert_delete ( string, 1, NULL );
+	assert ( ( rc == 0 ) || ( *(string->buf) == NULL ) );
 }
 
 /**
@@ -111,6 +139,7 @@ static void delete_character ( struct edit_string *string ) {
  * @v string		Editable string
  */
 static void backspace ( struct edit_string *string ) {
+
 	if ( string->cursor > 0 ) {
 		string->cursor--;
 		delete_character ( string );
@@ -123,14 +152,16 @@ static void backspace ( struct edit_string *string ) {
  * @v string		Editable string
  */
 static void previous_word ( struct edit_string *string ) {
-	while ( string->cursor &&
-		isspace ( string->buf[ string->cursor - 1 ] ) ) {
-		string->cursor--;
+	const char *buf = *(string->buf);
+	size_t cursor = string->cursor;
+
+	while ( cursor && isspace ( buf[ cursor - 1 ] ) ) {
+		cursor--;
 	}
-	while ( string->cursor &&
-		( ! isspace ( string->buf[ string->cursor - 1 ] ) ) ) {
-		string->cursor--;
+	while ( cursor && ( ! isspace ( buf[ cursor - 1 ] ) ) ) {
+		cursor--;
 	}
+	string->cursor = cursor;
 }
 
 /**
@@ -140,8 +171,11 @@ static void previous_word ( struct edit_string *string ) {
  */
 static void kill_word ( struct edit_string *string ) {
 	size_t old_cursor = string->cursor;
+	int rc;
+
 	previous_word ( string );
-	insert_delete ( string, ( old_cursor - string->cursor ), NULL );
+	rc = insert_delete ( string, ( old_cursor - string->cursor ), NULL );
+	assert ( ( rc == 0 ) || ( *(string->buf) == NULL ) );
 }
 
 /**
@@ -151,8 +185,11 @@ static void kill_word ( struct edit_string *string ) {
  */
 static void kill_sol ( struct edit_string *string ) {
 	size_t old_cursor = string->cursor;
+	int rc;
+
 	string->cursor = 0;
-	insert_delete ( string, old_cursor, NULL );
+	rc = insert_delete ( string, old_cursor, NULL );
+	assert ( ( rc == 0 ) || ( *(string->buf) == NULL ) );
 }
 
 /**
@@ -161,18 +198,36 @@ static void kill_sol ( struct edit_string *string ) {
  * @v string		Editable string
  */
 static void kill_eol ( struct edit_string *string ) {
-	insert_delete ( string, ~( ( size_t ) 0 ), NULL );
+	int rc;
+
+	rc = insert_delete ( string, ~( ( size_t ) 0 ), NULL );
+	assert ( ( rc == 0 ) || ( *(string->buf) == NULL ) );
 }
 
 /**
  * Replace editable string
  *
  * @v string		Editable string
- * @v replacement	Replacement string
+ * @v replacement	Replacement string, or NULL to empty the string
+ * @ret rc		Return status code
+ *
+ * Replace the entire content of the editable string and update the
+ * edit history to allow the caller to bring the display into sync
+ * with the string content.
+ *
+ * This function does not itself update the display in any way.
+ *
+ * Upon success, the string buffer is guaranteed to be non-NULL (even
+ * if the replacement string is NULL or empty).
+ *
+ * Errors may safely be ignored if it is deemed that subsequently
+ * failing to update the display will provide sufficient feedback to
+ * the user.
  */
-void replace_string ( struct edit_string *string, const char *replacement ) {
+int replace_string ( struct edit_string *string, const char *replacement ) {
+
 	string->cursor = 0;
-	insert_delete ( string, ~( ( size_t ) 0 ), replacement );
+	return insert_delete ( string, ~( ( size_t ) 0 ), replacement );
 }
 
 /**
@@ -180,21 +235,26 @@ void replace_string ( struct edit_string *string, const char *replacement ) {
  *
  * @v string		Editable string
  * @v key		Key pressed by user
- * @ret key		Key returned to application, or zero
+ * @ret key		Key returned to application, zero, or negative error
  *
  * Handles keypresses and updates the content of the editable string.
  * Basic line editing facilities (delete/insert/cursor) are supported.
  * If edit_string() understands and uses the keypress it will return
  * zero, otherwise it will return the original key.
  *
- * This function does not update the display in any way.
- *
  * The string's edit history will be updated to allow the caller to
  * efficiently bring the display into sync with the string content.
+ *
+ * This function does not itself update the display in any way.
+ *
+ * Errors may safely be ignored if it is deemed that subsequently
+ * failing to update the display will provide sufficient feedback to
+ * the user.
  */
 int edit_string ( struct edit_string *string, int key ) {
+	const char *buf = *(string->buf);
+	size_t len = ( buf ? strlen ( buf ) : 0 );
 	int retval = 0;
-	size_t len = strlen ( string->buf );
 
 	/* Prepare edit history */
 	string->last_cursor = string->cursor;
@@ -204,7 +264,7 @@ int edit_string ( struct edit_string *string, int key ) {
 	/* Interpret key */
 	if ( ( key >= 0x20 ) && ( key <= 0x7e ) ) {
 		/* Printable character; insert at current position */
-		insert_character ( string, key );
+		retval = insert_character ( string, key );
 	} else switch ( key ) {
 	case KEY_BACKSPACE:
 		/* Backspace */
diff --git a/src/hci/mucurses/widgets/editbox.c b/src/hci/mucurses/widgets/editbox.c
index 210de4481..fddd2742e 100644
--- a/src/hci/mucurses/widgets/editbox.c
+++ b/src/hci/mucurses/widgets/editbox.c
@@ -39,20 +39,19 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Initialise text box widget
  *
  * @v box		Editable text box widget
- * @v buf		Text buffer
- * @v len		Size of text buffer
+ * @v buf		Dynamically allocated string buffer
  * @v win		Containing window
  * @v row		Row
  * @v col		Starting column
  * @v width		Width
  * @v flags		Flags
  */
-void init_editbox ( struct edit_box *box, char *buf, size_t len,
+void init_editbox ( struct edit_box *box, char **buf,
 		    WINDOW *win, unsigned int row, unsigned int col,
 		    unsigned int width, unsigned int flags ) {
 	memset ( box, 0, sizeof ( *box ) );
-	init_editstring ( &box->string, buf, len );
-	box->string.cursor = strlen ( buf );
+	init_editstring ( &box->string, buf );
+	box->string.cursor = ( *buf ? strlen ( *buf ) : 0 );
 	box->win = ( win ? win : stdscr );
 	box->row = row;
 	box->col = col;
@@ -67,6 +66,7 @@ void init_editbox ( struct edit_box *box, char *buf, size_t len,
  *
  */
 void draw_editbox ( struct edit_box *box ) {
+	const char *content = *(box->string.buf);
 	size_t width = box->width;
 	char buf[ width + 1 ];
 	signed int cursor_offset, underflow, overflow, first;
@@ -90,13 +90,13 @@ void draw_editbox ( struct edit_box *box ) {
 	/* Construct underscore-padded string portion */
 	memset ( buf, '_', width );
 	buf[width] = '\0';
-	len = ( strlen ( box->string.buf ) - first );
+	len = ( content ? ( strlen ( content ) - first ) : 0 );
 	if ( len > width )
 		len = width;
 	if ( box->flags & EDITBOX_STARS ) {
 		memset ( buf, '*', len );
 	} else {
-		memcpy ( buf, ( box->string.buf + first ), len );
+		memcpy ( buf, ( content + first ), len );
 	}
 
 	/* Print box content and move cursor */
diff --git a/src/hci/readline.c b/src/hci/readline.c
index ecc72d43f..5b46413e9 100644
--- a/src/hci/readline.c
+++ b/src/hci/readline.c
@@ -38,8 +38,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
-#define READLINE_MAX 1024
-
 /**
  * Synchronise console with edited string
  *
@@ -49,7 +47,8 @@ static void sync_console ( struct edit_string *string ) {
 	unsigned int mod_start = string->mod_start;
 	unsigned int mod_end = string->mod_end;
 	unsigned int cursor = string->last_cursor;
-	size_t len = strlen ( string->buf );
+	const char *buf = *(string->buf);
+	size_t len = strlen ( buf );
 
 	/* Expand region back to old cursor position if applicable */
 	if ( mod_start > string->last_cursor )
@@ -67,7 +66,7 @@ static void sync_console ( struct edit_string *string ) {
 
 	/* Print modified region */
 	while ( cursor < mod_end ) {
-		putchar ( ( cursor >= len ) ? ' ' : string->buf[cursor] );
+		putchar ( ( cursor >= len ) ? ' ' : buf[cursor] );
 		cursor++;
 	}
 
@@ -259,15 +258,11 @@ int readline_history ( const char *prompt, const char *prefill,
 		       struct readline_history *history, unsigned long timeout,
 		       char **line ) {
 	struct edit_string string;
-	char *buf;
 	int key;
 	int move_by;
 	const char *new_string;
 	int rc;
 
-	/* Avoid returning uninitialised data on error */
-	*line = NULL;
-
 	/* Display prompt, if applicable */
 	if ( prompt )
 		printf ( "%s", prompt );
@@ -275,20 +270,15 @@ int readline_history ( const char *prompt, const char *prefill,
 	/* Ensure cursor is visible */
 	printf ( "\033[?25h" );
 
-	/* Allocate buffer and initialise editable string */
-	buf = zalloc ( READLINE_MAX );
-	if ( ! buf ) {
-		rc = -ENOMEM;
-		goto done;
-	}
+	/* Initialise editable string */
+	*line = NULL;
 	memset ( &string, 0, sizeof ( string ) );
-	init_editstring ( &string, buf, READLINE_MAX );
+	init_editstring ( &string, line );
 
-	/* Prefill string, if applicable */
-	if ( prefill ) {
-		replace_string ( &string, prefill );
-		sync_console ( &string );
-	}
+	/* Prefill string */
+	if ( ( rc = replace_string ( &string, prefill ) ) != 0 )
+		goto error;
+	sync_console ( &string );
 
 	while ( 1 ) {
 
@@ -296,7 +286,7 @@ int readline_history ( const char *prompt, const char *prefill,
 		key = getkey ( timeout );
 		if ( key < 0 ) {
 			rc = -ETIMEDOUT;
-			goto done;
+			goto error;
 		}
 		timeout = 0;
 
@@ -307,17 +297,11 @@ int readline_history ( const char *prompt, const char *prefill,
 		switch ( key ) {
 		case CR:
 		case LF:
-			/* Shrink string (ignoring failures) */
-			*line = realloc ( buf,
-					  ( strlen ( buf ) + 1 /* NUL */ ) );
-			if ( ! *line )
-				*line = buf;
-			buf = NULL;
 			rc = 0;
 			goto done;
 		case CTRL_C:
 			rc = -ECANCELED;
-			goto done;
+			goto error;
 		case KEY_UP:
 			move_by = 1;
 			break;
@@ -325,13 +309,13 @@ int readline_history ( const char *prompt, const char *prefill,
 			move_by = -1;
 			break;
 		default:
-			/* Do nothing */
+			/* Do nothing for unrecognised keys or edit errors */
 			break;
 		}
 
 		/* Handle history movement, if applicable */
 		if ( move_by && history ) {
-			new_string = history_move ( history, move_by, buf );
+			new_string = history_move ( history, move_by, *line );
 			if ( new_string ) {
 				replace_string ( &string, new_string );
 				sync_console ( &string );
@@ -339,9 +323,11 @@ int readline_history ( const char *prompt, const char *prefill,
 		}
 	}
 
+ error:
+	free ( *line );
+	*line = NULL;
  done:
 	putchar ( '\n' );
-	free ( buf );
 	if ( history ) {
 		if ( *line && (*line)[0] )
 			history_append ( history, *line );
diff --git a/src/hci/tui/login_ui.c b/src/hci/tui/login_ui.c
index 3c55325d5..bd24991cd 100644
--- a/src/hci/tui/login_ui.c
+++ b/src/hci/tui/login_ui.c
@@ -30,6 +30,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 
 #include <string.h>
+#include <stdlib.h>
 #include <errno.h>
 #include <curses.h>
 #include <ipxe/console.h>
@@ -49,8 +50,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define EDITBOX_WIDTH		20U
 
 int login_ui ( void ) {
-	char username[64];
-	char password[64];
+	char *username;
+	char *password;
 	struct edit_box username_box;
 	struct edit_box password_box;
 	struct edit_box *current_box = &username_box;
@@ -58,19 +59,16 @@ int login_ui ( void ) {
 	int rc = -EINPROGRESS;
 
 	/* Fetch current setting values */
-	fetch_string_setting ( NULL, &username_setting, username,
-			       sizeof ( username ) );
-	fetch_string_setting ( NULL, &password_setting, password,
-			       sizeof ( password ) );
+	fetchf_setting_copy ( NULL, &username_setting, NULL, NULL, &username );
+	fetchf_setting_copy ( NULL, &password_setting, NULL, NULL, &password );
 
 	/* Initialise UI */
 	initscr();
 	start_color();
-	init_editbox ( &username_box, username, sizeof ( username ), NULL,
-		       USERNAME_ROW, EDITBOX_COL, EDITBOX_WIDTH, 0 );
-	init_editbox ( &password_box, password, sizeof ( password ), NULL,
-		       PASSWORD_ROW, EDITBOX_COL, EDITBOX_WIDTH,
-		       EDITBOX_STARS );
+	init_editbox ( &username_box, &username, NULL, USERNAME_ROW,
+		       EDITBOX_COL, EDITBOX_WIDTH, 0 );
+	init_editbox ( &password_box, &password, NULL, PASSWORD_ROW,
+		       EDITBOX_COL, EDITBOX_WIDTH, EDITBOX_STARS );
 
 	/* Draw initial UI */
 	color_set ( CPAIR_NORMAL, NULL );
@@ -122,16 +120,15 @@ int login_ui ( void ) {
 	erase();
 	endwin();
 
-	if ( rc != 0 )
-		return rc;
+	/* Store settings on successful completion */
+	if ( rc == 0 )
+		rc = storef_setting ( NULL, &username_setting, username );
+	if ( rc == 0 )
+		rc = storef_setting ( NULL, &password_setting, password );
 
-	/* Store settings */
-	if ( ( rc = store_setting ( NULL, &username_setting, username,
-				    strlen ( username ) ) ) != 0 )
-		return rc;
-	if ( ( rc = store_setting ( NULL, &password_setting, password,
-				    strlen ( password ) ) ) != 0 )
-		return rc;
+	/* Free setting values */
+	free ( username );
+	free ( password );
 
-	return 0;
+	return rc;
 }
diff --git a/src/hci/tui/settings_ui.c b/src/hci/tui/settings_ui.c
index be421cc0a..95cbd7758 100644
--- a/src/hci/tui/settings_ui.c
+++ b/src/hci/tui/settings_ui.c
@@ -25,6 +25,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdarg.h>
+#include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
 #include <curses.h>
@@ -58,12 +59,15 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	char start[0];							\
 	char pad1[1];							\
 	union {								\
-		char settings[ cols - 1 - 1 - 1 - 1 ];			\
+		struct {						\
+			char name[ cols - 1 - 1 - 1 - 1 - 1 ];		\
+			char pad2[1];					\
+		} __attribute__ (( packed )) settings;			\
 		struct {						\
 			char name[15];					\
 			char pad2[1];					\
 			char value[ cols - 1 - 15 - 1 - 1 - 1 - 1 ];	\
-		} setting;						\
+		} __attribute__ (( packed )) setting;			\
 	} u;								\
 	char pad3[1];							\
 	char nul;							\
@@ -92,8 +96,8 @@ struct settings_ui_row {
 	struct edit_box editbox;
 	/** Editing in progress flag */
 	int editing;
-	/** Buffer for setting's value */
-	char value[256]; /* enough size for a DHCP string */
+	/** Dynamically allocated buffer for setting's value */
+	char *buf;
 };
 
 /** A settings user interface */
@@ -121,24 +125,22 @@ static unsigned int select_setting_row ( struct settings_ui *ui,
 	struct setting *previous = NULL;
 	unsigned int count = 0;
 
+	/* Free any previous setting value */
+	free ( ui->row.buf );
+	ui->row.buf = NULL;
+
 	/* Initialise structure */
 	memset ( &ui->row, 0, sizeof ( ui->row ) );
 	ui->row.row = ( SETTINGS_LIST_ROW + index - ui->scroll.first );
 
 	/* Include parent settings block, if applicable */
-	if ( ui->settings->parent && ( count++ == index ) ) {
+	if ( ui->settings->parent && ( count++ == index ) )
 		ui->row.settings = ui->settings->parent;
-		snprintf ( ui->row.value, sizeof ( ui->row.value ),
-			   "../" );
-	}
 
 	/* Include any child settings blocks, if applicable */
 	list_for_each_entry ( settings, &ui->settings->children, siblings ) {
-		if ( count++ == index ) {
+		if ( count++ == index )
 			ui->row.settings = settings;
-			snprintf ( ui->row.value, sizeof ( ui->row.value ),
-				   "%s/", settings->name );
-		}
 	}
 
 	/* Include any applicable settings */
@@ -155,15 +157,14 @@ static unsigned int select_setting_row ( struct settings_ui *ui,
 
 		/* Read current setting value and origin */
 		if ( count++ == index ) {
-			fetchf_setting ( ui->settings, setting, &ui->row.origin,
-					 &ui->row.setting, ui->row.value,
-					 sizeof ( ui->row.value ) );
+			fetchf_setting_copy ( ui->settings, setting,
+					      &ui->row.origin,
+					      &ui->row.setting, &ui->row.buf );
 		}
 	}
 
 	/* Initialise edit box */
-	init_editbox ( &ui->row.editbox, ui->row.value,
-		       sizeof ( ui->row.value ), NULL, ui->row.row,
+	init_editbox ( &ui->row.editbox, &ui->row.buf, NULL, ui->row.row,
 		       ( SETTINGS_LIST_COL +
 			 offsetof ( typeof ( *text ), u.setting.value ) ),
 		       sizeof ( text->u.setting.value ), 0 );
@@ -197,7 +198,7 @@ static size_t string_copy ( char *dest, const char *src, size_t len ) {
 static void draw_setting_row ( struct settings_ui *ui ) {
 	SETTING_ROW_TEXT ( COLS ) text;
 	unsigned int curs_offset;
-	char *value;
+	const char *value;
 
 	/* Fill row with spaces */
 	memset ( &text, ' ', sizeof ( text ) );
@@ -207,10 +208,12 @@ static void draw_setting_row ( struct settings_ui *ui ) {
 	if ( ui->row.settings ) {
 
 		/* Construct space-padded name */
-		curs_offset = ( offsetof ( typeof ( text ), u.settings ) +
-				string_copy ( text.u.settings,
-					      ui->row.value,
-					      sizeof ( text.u.settings ) ) );
+		value = ( ( ui->row.settings == ui->settings->parent ) ?
+			  ".." : ui->row.settings->name );
+		curs_offset = string_copy ( text.u.settings.name, value,
+					    sizeof ( text.u.settings.name ) );
+		text.u.settings.name[curs_offset] = '/';
+		curs_offset += offsetof ( typeof ( text ), u.settings );
 
 	} else {
 
@@ -221,12 +224,12 @@ static void draw_setting_row ( struct settings_ui *ui ) {
 			      sizeof ( text.u.setting.name ) );
 
 		/* Construct space-padded value */
-		value = ui->row.value;
-		if ( ! *value )
+		value = ui->row.buf;
+		if ( ! ( value && value[0] ) )
 			value = "<not specified>";
-		curs_offset = ( offsetof ( typeof ( text ), u.setting.value ) +
-				string_copy ( text.u.setting.value, value,
-					      sizeof ( text.u.setting.value )));
+		curs_offset = string_copy ( text.u.setting.value, value,
+					    sizeof ( text.u.setting.value ) );
+		curs_offset += offsetof ( typeof ( text ), u.setting.value );
 	}
 
 	/* Print row */
@@ -257,7 +260,7 @@ static int edit_setting ( struct settings_ui *ui, int key ) {
  */
 static int save_setting ( struct settings_ui *ui ) {
 	assert ( ui->row.setting.name != NULL );
-	return storef_setting ( ui->settings, &ui->row.setting, ui->row.value );
+	return storef_setting ( ui->settings, &ui->row.setting, ui->row.buf );
 }
 
 /**
@@ -527,6 +530,7 @@ static int main_loop ( struct settings *settings ) {
 			redraw = 1;
 			break;
 		case CTRL_X:
+			select_setting_row ( &ui, -1U );
 			return 0;
 		case CR:
 		case LF:
diff --git a/src/include/ipxe/editbox.h b/src/include/ipxe/editbox.h
index 2c70e0b6b..c14bbdc5b 100644
--- a/src/include/ipxe/editbox.h
+++ b/src/include/ipxe/editbox.h
@@ -36,7 +36,7 @@ enum edit_box_flags {
 	EDITBOX_STARS = 0x0001,
 };
 
-extern void init_editbox ( struct edit_box *box, char *buf, size_t len,
+extern void init_editbox ( struct edit_box *box, char **buf,
 			   WINDOW *win, unsigned int row, unsigned int col,
 			   unsigned int width, unsigned int flags )
 			   __attribute__ (( nonnull (1, 2) ));
diff --git a/src/include/ipxe/editstring.h b/src/include/ipxe/editstring.h
index a00a8adaa..7ad8fb304 100644
--- a/src/include/ipxe/editstring.h
+++ b/src/include/ipxe/editstring.h
@@ -11,10 +11,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** An editable string */
 struct edit_string {
-	/** Buffer for string */
-	char *buf;
-	/** Size of buffer (including terminating NUL) */
-	size_t len;
+	/** Dynamically allocated string buffer */
+	char **buf;
 	/** Cursor position */
 	unsigned int cursor;
 
@@ -32,17 +30,28 @@ struct edit_string {
  * Initialise editable string
  *
  * @v string		Editable string
- * @v buf		Buffer for string
- * @v len		Length of buffer
+ * @v buf		Dynamically allocated string buffer
+ *
+ * The @c buf parameter must be the address of a caller-provided
+ * pointer to a NUL-terminated string allocated using malloc() (or
+ * equivalent, such as strdup()).  Any edits made to the string will
+ * realloc() the string buffer as needed.
+ *
+ * The caller may choose leave the initial string buffer pointer as @c
+ * NULL, in which case it will be allocated upon the first attempt to
+ * insert a character into the buffer.  If the caller does this, then
+ * it must be prepared to find the pointer still @c NULL after
+ * editing, since the user may never attempt to insert any characters.
  */
-static inline void init_editstring ( struct edit_string *string, char *buf,
-				     size_t len ) {
+static inline __nonnull void init_editstring ( struct edit_string *string,
+					       char **buf ) {
+
 	string->buf = buf;
-	string->len = len;
 }
 
-extern void replace_string ( struct edit_string *string,
-			     const char *replacement ) __nonnull;
-extern int edit_string ( struct edit_string *string, int key ) __nonnull;
+extern __attribute__ (( nonnull ( 1 ) )) int
+replace_string ( struct edit_string *string, const char *replacement );
+
+extern __nonnull int edit_string ( struct edit_string *string, int key );
 
 #endif /* _IPXE_EDITSTRING_H */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 0ab37230a..96981dd06 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -416,6 +416,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_efi_settings	      ( ERRFILE_OTHER | 0x005e0000 )
 #define ERRFILE_x25519		      ( ERRFILE_OTHER | 0x005f0000 )
 #define ERRFILE_des		      ( ERRFILE_OTHER | 0x00600000 )
+#define ERRFILE_editstring	      ( ERRFILE_OTHER | 0x00610000 )
 
 /** @} */
 

From cb95b5b378b1a61d10770965d82dba535b6be242 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 16 Apr 2024 14:19:01 +0100
Subject: [PATCH 135/237] [efi] Veto the Dhcp6Dxe driver on all platforms

The reference implementation of Dhcp6Dxe in EDK2 has a fatal flaw: the
code in EfiDhcp6Stop() will poll the network in a tight loop until
either a response is received or a timer tick (at TPL_CALLBACK)
occurs.  When EfiDhcp6Stop() is called at TPL_CALLBACK or higher, this
will result in an endless loop and an apparently frozen system.

Since this is the reference implementation of Dhcp6Dxe, it is likely
that almost all platforms have the same problem.

Fix by vetoing the broken driver.  If the upstream driver is ever
fixed and a new version number issued, then we could plausibly test
against the version number exposed via the driver binding protocol.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_veto.c | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/src/interface/efi/efi_veto.c b/src/interface/efi/efi_veto.c
index a3b60d65f..37aa9a379 100644
--- a/src/interface/efi/efi_veto.c
+++ b/src/interface/efi/efi_veto.c
@@ -494,6 +494,32 @@ efi_veto_vmware_uefipxebc ( EFI_DRIVER_BINDING_PROTOCOL *binding __unused,
 	return 1;
 }
 
+/**
+ * Veto Dhcp6Dxe driver
+ *
+ * @v binding		Driver binding protocol
+ * @v loaded		Loaded image protocol
+ * @v wtf		Component name protocol, if present
+ * @v manufacturer	Manufacturer name, if present
+ * @v name		Driver name, if present
+ * @ret vetoed		Driver is to be vetoed
+ */
+static int efi_veto_dhcp6 ( EFI_DRIVER_BINDING_PROTOCOL *binding __unused,
+			    EFI_LOADED_IMAGE_PROTOCOL *loaded __unused,
+			    EFI_COMPONENT_NAME_PROTOCOL *wtf __unused,
+			    const char *manufacturer __unused,
+			    const CHAR16 *name ) {
+	static const CHAR16 dhcp6[] = L"DHCP6 Protocol Driver";
+
+	/* Check driver name */
+	if ( ! name )
+		return 0;
+	if ( memcmp ( name, dhcp6, sizeof ( dhcp6 ) ) != 0 )
+		return 0;
+
+	return 1;
+}
+
 /** Driver vetoes */
 static struct efi_veto_candidate efi_vetoes[] = {
 	{
@@ -508,6 +534,10 @@ static struct efi_veto_candidate efi_vetoes[] = {
 		.name = "VMware UefiPxeBc",
 		.veto = efi_veto_vmware_uefipxebc,
 	},
+	{
+		.name = "Dhcp6",
+		.veto = efi_veto_dhcp6,
+	},
 };
 
 /**

From b01781a2b8fb29669d9a610bfb323fb8a6960612 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 17 Apr 2024 15:45:36 +0100
Subject: [PATCH 136/237] [hci] Fix semantics of replace_string() to match code
 comments

The comments for replace_string() state that a successful return
status guarantees that the dynamically allocated string pointer is no
longer NULL (even if it was initially NULL and the replacement string
is NULL or empty).  This is relied upon by readline() to guarantee
that it will always return a non-NULL string if successful.

The code behaviour does not currently match this comment: an empty
replacement string may result in a successful return status even if
the (single-byte) allocation fails.

Fix up the code behaviour to match the comments, and to additionally
ensure that the edit history is filled in even in the event of an
allocation failure.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/editstring.c | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/src/hci/editstring.c b/src/hci/editstring.c
index b83916c6e..be9ca06a5 100644
--- a/src/hci/editstring.c
+++ b/src/hci/editstring.c
@@ -59,10 +59,14 @@ static __nonnull void kill_eol ( struct edit_string *string );
  */
 static int insert_delete ( struct edit_string *string, size_t delete_len,
 			   const char *insert_text ) {
-	size_t old_len, max_delete_len, insert_len, new_len;
+	size_t old_len, max_delete_len, move_len, insert_len, new_len;
 	char *buf;
 	char *tmp;
 
+	/* Prepare edit history */
+	string->mod_start = string->cursor;
+	string->mod_end = string->cursor;
+
 	/* Calculate lengths */
 	buf = *(string->buf);
 	old_len = ( buf ? strlen ( buf ) : 0 );
@@ -70,39 +74,36 @@ static int insert_delete ( struct edit_string *string, size_t delete_len,
 	max_delete_len = ( old_len - string->cursor );
 	if ( delete_len > max_delete_len )
 		delete_len = max_delete_len;
+	move_len = ( max_delete_len - delete_len );
 	insert_len = ( insert_text ? strlen ( insert_text ) : 0 );
 	new_len = ( old_len - delete_len + insert_len );
 
+	/* Delete existing text */
+	memmove ( ( buf + string->cursor ),
+		  ( buf + string->cursor + delete_len ), move_len );
+
 	/* Reallocate string, ignoring failures if shrinking */
 	tmp = realloc ( buf, ( new_len + 1 /* NUL */ ) );
 	if ( tmp ) {
 		buf = tmp;
 		*(string->buf) = buf;
-	} else if ( new_len > old_len ) {
+	} else if ( ( new_len > old_len ) || ( ! buf ) ) {
 		return -ENOMEM;
 	}
 
-	/* Fill in edit history */
-	string->mod_start = string->cursor;
-	string->mod_end = ( ( new_len > old_len ) ? new_len : old_len );
-
-	/* Move data following the cursor */
+	/* Create space for inserted text */
 	memmove ( ( buf + string->cursor + insert_len ),
-		  ( buf + string->cursor + delete_len ),
-		  ( max_delete_len - delete_len ) );
+		  ( buf + string->cursor ), move_len );
 
 	/* Copy inserted text to cursor position */
 	memcpy ( ( buf + string->cursor ), insert_text, insert_len );
 	string->cursor += insert_len;
 
-	/* Terminate string (if not NULL) */
-	if ( buf ) {
-		buf[new_len] = '\0';
-	} else {
-		assert ( old_len == 0 );
-		assert ( insert_len == 0 );
-		assert ( new_len == 0 );
-	}
+	/* Terminate string */
+	buf[new_len] = '\0';
+
+	/* Update edit history */
+	string->mod_end = ( ( new_len > old_len ) ? new_len : old_len );
 
 	return 0;
 }

From d7e58c5a812988c341ec4ad19f79faf067388d58 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 17 Apr 2024 15:49:37 +0100
Subject: [PATCH 137/237] [test] Add test cases for editable strings

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/tests/editstring_test.c | 198 ++++++++++++++++++++++++++++++++++++
 src/tests/tests.c           |   1 +
 2 files changed, 199 insertions(+)
 create mode 100644 src/tests/editstring_test.c

diff --git a/src/tests/editstring_test.c b/src/tests/editstring_test.c
new file mode 100644
index 000000000..72da33a77
--- /dev/null
+++ b/src/tests/editstring_test.c
@@ -0,0 +1,198 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Editable string tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <ipxe/keys.h>
+#include <ipxe/editstring.h>
+#include <ipxe/test.h>
+
+/** An editable string test */
+struct editstring_test {
+	/** Initial string, or NULL */
+	const char *start;
+	/** Key sequence */
+	const int *keys;
+	/** Length of key sequence */
+	unsigned int count;
+	/** Expected result */
+	const char *expected;
+};
+
+/** Define an inline key sequence */
+#define KEYS(...) { __VA_ARGS__ }
+
+/** Define an editable string test */
+#define EDITSTRING_TEST( name, START, EXPECTED, KEYS )			\
+	static const int name ## _keys[] = KEYS;			\
+	static struct editstring_test name = {				\
+		.start = START,						\
+		.keys = name ## _keys,					\
+		.count = ( sizeof ( name ## _keys ) /			\
+			   sizeof ( name ## _keys[0] ) ),		\
+		.expected = EXPECTED,					\
+	};
+
+/* Simple typing */
+EDITSTRING_TEST ( simple, "", "hello world!",
+		  KEYS ( 'h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l',
+			 'd', '!' ) );
+
+/* Simple typing from a NULL starting value */
+EDITSTRING_TEST ( simple_null, NULL, "hi there",
+		  KEYS ( 'h', 'i', ' ', 't', 'h', 'e', 'r', 'e' ) );
+
+/* Insertion */
+EDITSTRING_TEST ( insert, "in middle", "in the middle",
+		  KEYS ( KEY_LEFT, KEY_LEFT, KEY_LEFT, KEY_LEFT, KEY_LEFT,
+			 KEY_LEFT, 't', 'h', 'e', ' ' ) );
+
+/* Backspace at end */
+EDITSTRING_TEST ( backspace_end, "byebye", "bye",
+		  KEYS ( KEY_BACKSPACE, KEY_BACKSPACE, KEY_BACKSPACE ) );
+
+/* Backspace of whole string */
+EDITSTRING_TEST ( backspace_all, "abc", "",
+		  KEYS ( KEY_BACKSPACE, KEY_BACKSPACE, KEY_BACKSPACE ) );
+
+/* Backspace of empty string */
+EDITSTRING_TEST ( backspace_empty, NULL, "", KEYS ( KEY_BACKSPACE ) );
+
+/* Backspace beyond start of string */
+EDITSTRING_TEST ( backspace_beyond, "too far", "",
+		  KEYS ( KEY_BACKSPACE, KEY_BACKSPACE, KEY_BACKSPACE,
+			 KEY_BACKSPACE, KEY_BACKSPACE, KEY_BACKSPACE,
+			 KEY_BACKSPACE, KEY_BACKSPACE, KEY_BACKSPACE ) );
+
+/* Deletion of character at cursor via DEL */
+EDITSTRING_TEST ( delete_dc, "go away", "goaway",
+		  KEYS ( KEY_HOME, KEY_RIGHT, KEY_RIGHT, KEY_DC ) );
+
+/* Deletion of character at cursor via Ctrl-D */
+EDITSTRING_TEST ( delete_ctrl_d, "not here", "nohere",
+		  KEYS ( KEY_LEFT, KEY_LEFT, KEY_LEFT, KEY_LEFT, KEY_LEFT,
+			 KEY_LEFT, CTRL_D, CTRL_D ) );
+
+/* Deletion of word at end of string */
+EDITSTRING_TEST ( word_end, "remove these two words", "remove these ",
+		  KEYS ( CTRL_W, CTRL_W ) );
+
+/* Deletion of word at start of string */
+EDITSTRING_TEST ( word_start, "no word", "word",
+		  KEYS ( CTRL_A, KEY_RIGHT, KEY_RIGHT, KEY_RIGHT, CTRL_W ) );
+
+/* Deletion of word mid-string */
+EDITSTRING_TEST ( word_mid, "delete this word", "delete word",
+		  KEYS ( KEY_LEFT, KEY_LEFT, KEY_LEFT, KEY_LEFT, CTRL_W ) );
+
+/* Deletion to start of line */
+EDITSTRING_TEST ( sol, "everything must go", "go",
+		  KEYS ( KEY_LEFT, KEY_LEFT, CTRL_U ) );
+
+/* Delete to end of line */
+EDITSTRING_TEST ( eol, "all is lost", "all",
+		  KEYS ( KEY_HOME, KEY_RIGHT, KEY_RIGHT, KEY_RIGHT, CTRL_K ) );
+
+/**
+ * Report an editable string test result
+ *
+ * @v test		Editable string test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void editstring_okx ( struct editstring_test *test, const char *file,
+			     unsigned int line ) {
+	struct edit_string string;
+	unsigned int i;
+	char *actual;
+	int key;
+
+	/* Initialise editable string */
+	memset ( &string, 0, sizeof ( string ) );
+	actual = NULL;
+	init_editstring ( &string, &actual );
+
+	/* Set initial string content */
+	okx ( replace_string ( &string, test->start ) == 0, file, line );
+	okx ( actual != NULL, file, line );
+	okx ( string.cursor == ( test->start ? strlen ( test->start ) : 0 ),
+	      file, line );
+	DBGC ( test, "Initial string: \"%s\"\n", actual );
+
+	/* Inject keypresses */
+	for ( i = 0 ; i < test->count ; i++ ) {
+		key = test->keys[i];
+		okx ( edit_string ( &string, key ) == 0, file, line );
+		okx ( actual != NULL, file, line );
+		okx ( string.cursor <= strlen ( actual ), file, line );
+		DBGC ( test, "After key %#02x (%c): \"%s\"\n",
+		       key, ( isprint ( key ) ? key : '.' ), actual );
+	}
+
+	/* Verify result string */
+	okx ( strcmp ( actual, test->expected ) == 0, file, line );
+
+	/* Free result string */
+	free ( actual );
+}
+#define editstring_ok( test ) editstring_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Perform editable string self-tests
+ *
+ */
+static void editstring_test_exec ( void ) {
+
+	editstring_ok ( &simple );
+	editstring_ok ( &simple_null );
+	editstring_ok ( &insert );
+	editstring_ok ( &backspace_end );
+	editstring_ok ( &backspace_all );
+	editstring_ok ( &backspace_empty );
+	editstring_ok ( &backspace_beyond );
+	editstring_ok ( &delete_dc );
+	editstring_ok ( &delete_ctrl_d );
+	editstring_ok ( &word_end );
+	editstring_ok ( &word_start );
+	editstring_ok ( &word_mid );
+	editstring_ok ( &sol );
+	editstring_ok ( &eol );
+}
+
+/** Editable string self-test */
+struct self_test editstring_test __self_test = {
+	.name = "editstring",
+	.exec = editstring_test_exec,
+};
diff --git a/src/tests/tests.c b/src/tests/tests.c
index cb296049e..0e9b3e806 100644
--- a/src/tests/tests.c
+++ b/src/tests/tests.c
@@ -85,3 +85,4 @@ REQUIRE_OBJECT ( x25519_test );
 REQUIRE_OBJECT ( des_test );
 REQUIRE_OBJECT ( mschapv2_test );
 REQUIRE_OBJECT ( uuid_test );
+REQUIRE_OBJECT ( editstring_test );

From dc118c53696af6a0b1a8ee78fc9a4d28a217fb21 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 15 May 2024 14:10:33 +0100
Subject: [PATCH 138/237] [hci] Provide a general concept of a text widget set

Create a generic abstraction of a text widget, refactor the existing
editable text box widget to use this abstraction, add an
implementation of a non-editable text label widget, and generalise the
login user interface to use this generic widget abstraction.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/mucurses/widgets/editbox.c |  65 ++++++-------
 src/hci/mucurses/widgets/label.c   |  77 +++++++++++++++
 src/hci/tui/login_ui.c             | 106 +++++++-------------
 src/hci/tui/settings_ui.c          |  14 +--
 src/hci/tui/widget_ui.c            | 143 +++++++++++++++++++++++++++
 src/include/ipxe/editbox.h         |  50 ++++------
 src/include/ipxe/errfile.h         |   1 +
 src/include/ipxe/label.h           |  42 ++++++++
 src/include/ipxe/widget.h          | 151 +++++++++++++++++++++++++++++
 9 files changed, 509 insertions(+), 140 deletions(-)
 create mode 100644 src/hci/mucurses/widgets/label.c
 create mode 100644 src/hci/tui/widget_ui.c
 create mode 100644 src/include/ipxe/label.h
 create mode 100644 src/include/ipxe/widget.h

diff --git a/src/hci/mucurses/widgets/editbox.c b/src/hci/mucurses/widgets/editbox.c
index fddd2742e..79e7cdd16 100644
--- a/src/hci/mucurses/widgets/editbox.c
+++ b/src/hci/mucurses/widgets/editbox.c
@@ -25,6 +25,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <assert.h>
+#include <ipxe/ansicol.h>
 #include <ipxe/editbox.h>
 
 /** @file
@@ -35,39 +36,16 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define EDITBOX_MIN_CHARS 3
 
-/**
- * Initialise text box widget
- *
- * @v box		Editable text box widget
- * @v buf		Dynamically allocated string buffer
- * @v win		Containing window
- * @v row		Row
- * @v col		Starting column
- * @v width		Width
- * @v flags		Flags
- */
-void init_editbox ( struct edit_box *box, char **buf,
-		    WINDOW *win, unsigned int row, unsigned int col,
-		    unsigned int width, unsigned int flags ) {
-	memset ( box, 0, sizeof ( *box ) );
-	init_editstring ( &box->string, buf );
-	box->string.cursor = ( *buf ? strlen ( *buf ) : 0 );
-	box->win = ( win ? win : stdscr );
-	box->row = row;
-	box->col = col;
-	box->width = width;
-	box->flags = flags;
-}
-
 /**
  * Draw text box widget
  *
- * @v box		Editable text box widget
- *
+ * @v widgets		Text widget set
+ * @v widget		Text widget
  */
-void draw_editbox ( struct edit_box *box ) {
+static void draw_editbox ( struct widgets *widgets, struct widget *widget ) {
+	struct edit_box *box = container_of ( widget, struct edit_box, widget );
 	const char *content = *(box->string.buf);
-	size_t width = box->width;
+	size_t width = widget->width;
 	char buf[ width + 1 ];
 	signed int cursor_offset, underflow, overflow, first;
 	size_t len;
@@ -93,15 +71,36 @@ void draw_editbox ( struct edit_box *box ) {
 	len = ( content ? ( strlen ( content ) - first ) : 0 );
 	if ( len > width )
 		len = width;
-	if ( box->flags & EDITBOX_STARS ) {
+	if ( widget->flags & WIDGET_SECRET ) {
 		memset ( buf, '*', len );
 	} else {
 		memcpy ( buf, ( content + first ), len );
 	}
 
 	/* Print box content and move cursor */
-	if ( ! box->win )
-		box->win = stdscr;
-	mvwprintw ( box->win, box->row, box->col, "%s", buf );
-	wmove ( box->win, box->row, ( box->col + cursor_offset ) );
+	color_set ( CPAIR_EDIT, NULL );
+	mvwprintw ( widgets->win, widget->row, widget->col, "%s", buf );
+	wmove ( widgets->win, widget->row, ( widget->col + cursor_offset ) );
+	color_set ( CPAIR_NORMAL, NULL );
 }
+
+/**
+ * Edit text box widget
+ *
+ * @v widgets		Text widget set
+ * @v widget		Text widget
+ * @v key		Key pressed by user
+ * @ret key		Key returned to application, or zero
+ */
+static int edit_editbox ( struct widgets *widgets __unused,
+			  struct widget *widget, int key ) {
+	struct edit_box *box = container_of ( widget, struct edit_box, widget );
+
+	return edit_string ( &box->string, key );
+}
+
+/** Text box widget operations */
+struct widget_operations editbox_operations = {
+	.draw = draw_editbox,
+	.edit = edit_editbox,
+};
diff --git a/src/hci/mucurses/widgets/label.c b/src/hci/mucurses/widgets/label.c
new file mode 100644
index 000000000..29645a6ad
--- /dev/null
+++ b/src/hci/mucurses/widgets/label.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <assert.h>
+#include <ipxe/label.h>
+
+/** @file
+ *
+ * Text label widget
+ *
+ */
+
+/**
+ * Draw text label widget
+ *
+ * @v widgets		Text widget set
+ * @v widget		Text widget
+ */
+static void draw_label ( struct widgets *widgets, struct widget *widget ) {
+	struct label *label = container_of ( widget, struct label, widget );
+	unsigned int width = widget->width;
+	unsigned int col = widget->col;
+	const char *text = label->text;
+
+	/* Centre label if width is non-zero */
+	if ( width )
+		col += ( ( width - strlen ( text ) ) / 2 );
+
+	/* Print label content */
+	attron ( A_BOLD );
+	mvwprintw ( widgets->win, widget->row, col, "%s", text );
+	attroff ( A_BOLD );
+}
+
+/**
+ * Edit text label widget
+ *
+ * @v widgets		Text widget set
+ * @v widget		Text widget
+ * @v key		Key pressed by user
+ * @ret key		Key returned to application, or zero
+ */
+static int edit_label ( struct widgets *widgets __unused,
+			struct widget *widget __unused, int key ) {
+
+	/* Cannot be edited */
+	return key;
+}
+
+/** Text label widget operations */
+struct widget_operations label_operations = {
+	.draw = draw_label,
+	.edit = edit_label,
+};
diff --git a/src/hci/tui/login_ui.c b/src/hci/tui/login_ui.c
index bd24991cd..b76afb92e 100644
--- a/src/hci/tui/login_ui.c
+++ b/src/hci/tui/login_ui.c
@@ -35,6 +35,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <curses.h>
 #include <ipxe/console.h>
 #include <ipxe/settings.h>
+#include <ipxe/label.h>
 #include <ipxe/editbox.h>
 #include <ipxe/keys.h>
 #include <ipxe/ansicol.h>
@@ -45,90 +46,55 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define USERNAME_ROW		( ( LINES / 2U ) - 2U )
 #define PASSWORD_LABEL_ROW	( ( LINES / 2U ) + 2U )
 #define PASSWORD_ROW		( ( LINES / 2U ) + 4U )
-#define LABEL_COL		( ( COLS / 2U ) - 4U )
-#define EDITBOX_COL		( ( COLS / 2U ) - 10U )
-#define EDITBOX_WIDTH		20U
+#define WIDGET_COL		( ( COLS / 2U ) - 10U )
+#define WIDGET_WIDTH		20U
 
 int login_ui ( void ) {
 	char *username;
 	char *password;
-	struct edit_box username_box;
-	struct edit_box password_box;
-	struct edit_box *current_box = &username_box;
-	int key;
-	int rc = -EINPROGRESS;
+	struct {
+		struct widgets widgets;
+		struct label username_label;
+		struct label password_label;
+		struct edit_box username_box;
+		struct edit_box password_box;
+	} widgets;
+	int rc;
 
 	/* Fetch current setting values */
 	fetchf_setting_copy ( NULL, &username_setting, NULL, NULL, &username );
 	fetchf_setting_copy ( NULL, &password_setting, NULL, NULL, &password );
 
-	/* Initialise UI */
-	initscr();
-	start_color();
-	init_editbox ( &username_box, &username, NULL, USERNAME_ROW,
-		       EDITBOX_COL, EDITBOX_WIDTH, 0 );
-	init_editbox ( &password_box, &password, NULL, PASSWORD_ROW,
-		       EDITBOX_COL, EDITBOX_WIDTH, EDITBOX_STARS );
+	/* Construct user interface */
+	memset ( &widgets, 0, sizeof ( widgets ) );
+	init_widgets ( &widgets.widgets, NULL );
+	init_label ( &widgets.username_label, USERNAME_LABEL_ROW, WIDGET_COL,
+		     WIDGET_WIDTH, "Username" );
+	init_label ( &widgets.password_label, PASSWORD_LABEL_ROW, WIDGET_COL,
+		     WIDGET_WIDTH, "Password" );
+	init_editbox ( &widgets.username_box, USERNAME_ROW, WIDGET_COL,
+		       WIDGET_WIDTH, 0, &username );
+	init_editbox ( &widgets.password_box, PASSWORD_ROW, WIDGET_COL,
+		       WIDGET_WIDTH, WIDGET_SECRET, &password );
+	add_widget ( &widgets.widgets, &widgets.username_label.widget );
+	add_widget ( &widgets.widgets, &widgets.password_label.widget );
+	add_widget ( &widgets.widgets, &widgets.username_box.widget );
+	add_widget ( &widgets.widgets, &widgets.password_box.widget );
 
-	/* Draw initial UI */
-	color_set ( CPAIR_NORMAL, NULL );
-	erase();
-	attron ( A_BOLD );
-	mvprintw ( USERNAME_LABEL_ROW, LABEL_COL, "Username:" );
-	mvprintw ( PASSWORD_LABEL_ROW, LABEL_COL, "Password:" );
-	attroff ( A_BOLD );
-	color_set ( CPAIR_EDIT, NULL );
-	draw_editbox ( &username_box );
-	draw_editbox ( &password_box );
-
-	/* Main loop */
-	while ( rc == -EINPROGRESS ) {
-
-		draw_editbox ( current_box );
-
-		key = getkey ( 0 );
-		switch ( key ) {
-		case KEY_DOWN:
-			current_box = &password_box;
-			break;
-		case KEY_UP:
-			current_box = &username_box;
-			break;
-		case TAB:
-			current_box = ( ( current_box == &username_box ) ?
-					&password_box : &username_box );
-			break;
-		case KEY_ENTER:
-			if ( current_box == &username_box ) {
-				current_box = &password_box;
-			} else {
-				rc = 0;
-			}
-			break;
-		case CTRL_C:
-		case ESC:
-			rc = -ECANCELED;
-			break;
-		default:
-			edit_editbox ( current_box, key );
-			break;
-		}
-	}
-
-	/* Terminate UI */
-	color_set ( CPAIR_NORMAL, NULL );
-	erase();
-	endwin();
+	/* Present user interface */
+	if ( ( rc = widget_ui ( &widgets.widgets ) ) != 0 )
+		goto err_ui;
 
 	/* Store settings on successful completion */
-	if ( rc == 0 )
-		rc = storef_setting ( NULL, &username_setting, username );
-	if ( rc == 0 )
-		rc = storef_setting ( NULL, &password_setting, password );
+	if ( ( rc = storef_setting ( NULL, &username_setting, username ) ) !=0)
+		goto err_store_username;
+	if ( ( rc = storef_setting ( NULL, &password_setting, password ) ) !=0)
+		goto err_store_password;
 
-	/* Free setting values */
+ err_store_username:
+ err_store_password:
+ err_ui:
 	free ( username );
 	free ( password );
-
 	return rc;
 }
diff --git a/src/hci/tui/settings_ui.c b/src/hci/tui/settings_ui.c
index 95cbd7758..53bf24d1a 100644
--- a/src/hci/tui/settings_ui.c
+++ b/src/hci/tui/settings_ui.c
@@ -108,6 +108,8 @@ struct settings_ui {
 	struct jump_scroller scroll;
 	/** Current row */
 	struct settings_ui_row row;
+	/** Widget set used for editing setting */
+	struct widgets widgets;
 };
 
 /**
@@ -164,10 +166,11 @@ static unsigned int select_setting_row ( struct settings_ui *ui,
 	}
 
 	/* Initialise edit box */
-	init_editbox ( &ui->row.editbox, &ui->row.buf, NULL, ui->row.row,
+	memset ( &ui->row.editbox, 0, sizeof ( ui->row.editbox ) );
+	init_editbox ( &ui->row.editbox, ui->row.row,
 		       ( SETTINGS_LIST_COL +
 			 offsetof ( typeof ( *text ), u.setting.value ) ),
-		       sizeof ( text->u.setting.value ), 0 );
+		       sizeof ( text->u.setting.value ), 0, &ui->row.buf );
 
 	return count;
 }
@@ -250,7 +253,7 @@ static void draw_setting_row ( struct settings_ui *ui ) {
 static int edit_setting ( struct settings_ui *ui, int key ) {
 	assert ( ui->row.setting.name != NULL );
 	ui->row.editing = 1;
-	return edit_editbox ( &ui->row.editbox, key );
+	return edit_widget ( &ui->widgets, &ui->row.editbox.widget, key );
 }
 
 /**
@@ -454,6 +457,7 @@ static int main_loop ( struct settings *settings ) {
 	/* Print initial screen content */
 	color_set ( CPAIR_NORMAL, NULL );
 	memset ( &ui, 0, sizeof ( ui ) );
+	init_widgets ( &ui.widgets, NULL );
 	select_settings ( &ui, settings );
 
 	while ( 1 ) {
@@ -477,9 +481,7 @@ static int main_loop ( struct settings *settings ) {
 			assert ( ui.row.setting.name != NULL );
 
 			/* Redraw edit box */
-			color_set ( CPAIR_EDIT, NULL );
-			draw_editbox ( &ui.row.editbox );
-			color_set ( CPAIR_NORMAL, NULL );
+			draw_widget ( &ui.widgets, &ui.row.editbox.widget );
 
 			/* Process keypress */
 			key = edit_setting ( &ui, getkey ( 0 ) );
diff --git a/src/hci/tui/widget_ui.c b/src/hci/tui/widget_ui.c
new file mode 100644
index 000000000..5079eefdd
--- /dev/null
+++ b/src/hci/tui/widget_ui.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Text widget UI
+ *
+ */
+
+#include <errno.h>
+#include <curses.h>
+#include <ipxe/ansicol.h>
+#include <ipxe/widget.h>
+
+/**
+ * Find editable widget in widget set
+ *
+ * @v widgets		Text widget set
+ * @v index		Editable widget index
+ * @ret widget		Editable widget, or NULL
+ */
+static struct widget * find_widget ( struct widgets *widgets,
+				     unsigned int index ) {
+	struct widget *widget;
+
+	list_for_each_entry ( widget, &widgets->list, list ) {
+		if ( ! ( widget->flags & WIDGET_EDITABLE ) )
+			continue;
+		if ( index-- == 0 )
+			return widget;
+	}
+	return NULL;
+}
+
+/**
+ * Text widget user interface main loop
+ *
+ * @v widgets		Text widget set
+ * @ret rc		Return status code
+ */
+static int widget_ui_loop ( struct widgets *widgets ) {
+	struct widget *widget;
+	unsigned int current;
+	unsigned int count;
+	int key;
+
+	/* Draw all widgets */
+	list_for_each_entry ( widget, &widgets->list, list )
+		draw_widget ( widgets, widget );
+
+	/* Count editable widgets */
+	count = 0;
+	while ( find_widget ( widgets, count ) != NULL )
+		count++;
+
+	/* Main loop */
+	current = 0;
+	while ( 1 ) {
+
+		/* Identify current widget */
+		widget = find_widget ( widgets, current );
+		if ( ! widget )
+			return -ENOENT;
+
+		/* Redraw current widget */
+		draw_widget ( widgets, widget );
+
+		/* Process keypress */
+		key = edit_widget ( widgets, widget, getkey ( 0 ) );
+		switch ( key ) {
+		case KEY_UP:
+			if ( current > 0 )
+				current--;
+			break;
+		case KEY_DOWN:
+			if ( ++current == count )
+				current--;
+			break;
+		case TAB:
+			if ( ++current == count )
+				current = 0;
+			break;
+		case KEY_ENTER:
+			current++;
+			if ( current >= count )
+				return 0;
+			break;
+		case CTRL_C:
+		case ESC:
+			return -ECANCELED;
+		default:
+			/* Do nothing for unrecognised keys or edit errors */
+			break;
+		}
+	}
+}
+
+/**
+ * Present text widget user interface
+ *
+ * @v widgets		Text widget set
+ * @ret rc		Return status code
+ */
+int widget_ui ( struct widgets *widgets ) {
+	int rc;
+
+	/* Initialise UI */
+	initscr();
+	start_color();
+	color_set ( CPAIR_NORMAL, NULL );
+	erase();
+
+	/* Run main loop */
+	rc = widget_ui_loop ( widgets );
+
+	/* Terminate UI */
+	color_set ( CPAIR_NORMAL, NULL );
+	endwin();
+
+	return rc;
+}
diff --git a/src/include/ipxe/editbox.h b/src/include/ipxe/editbox.h
index c14bbdc5b..1f62485fe 100644
--- a/src/include/ipxe/editbox.h
+++ b/src/include/ipxe/editbox.h
@@ -11,51 +11,39 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <curses.h>
 #include <ipxe/editstring.h>
+#include <ipxe/widget.h>
 
 /** An editable text box widget */
 struct edit_box {
+	/** Text widget */
+	struct widget widget;
 	/** Editable string */
 	struct edit_string string;
-	/** Containing window */
-	WINDOW *win;
-	/** Row */
-	unsigned int row;
-	/** Starting column */
-	unsigned int col;
-	/** Width */
-	unsigned int width;
 	/** First displayed character */
 	unsigned int first;
-	/** Flags */
-	unsigned int flags;
 };
 
-/** Editable text box widget flags */
-enum edit_box_flags {
-	/** Show stars instead of contents (for password widgets) */
-	EDITBOX_STARS = 0x0001,
-};
-
-extern void init_editbox ( struct edit_box *box, char **buf,
-			   WINDOW *win, unsigned int row, unsigned int col,
-			   unsigned int width, unsigned int flags )
-			   __attribute__ (( nonnull (1, 2) ));
-extern void draw_editbox ( struct edit_box *box ) __nonnull;
-static inline int edit_editbox ( struct edit_box *box, int key ) __nonnull;
+extern struct widget_operations editbox_operations;
 
 /**
- * Edit text box widget
+ * Initialise text box widget
  *
  * @v box		Editable text box widget
- * @v key		Key pressed by user
- * @ret key		Key returned to application, or zero
- *
- * You must call draw_editbox() to update the display after calling
- * edit_editbox().
- *
+ * @v row		Row
+ * @v col		Starting column
+ * @v width		Width
+ * @v flags		Flags
+ * @v buf		Dynamically allocated string buffer
  */
-static inline int edit_editbox ( struct edit_box *box, int key ) {
-	return edit_string ( &box->string, key );
+static inline __attribute__ (( always_inline )) void
+init_editbox ( struct edit_box *box, unsigned int row, unsigned int col,
+	       unsigned int width, unsigned int flags, char **buf ) {
+
+	init_widget ( &box->widget, &editbox_operations, row, col,
+		      width, ( flags | WIDGET_EDITABLE ) );
+	init_editstring ( &box->string, buf );
+	if ( *buf )
+		box->string.cursor = strlen ( *buf );
 }
 
 #endif /* _IPXE_EDITBOX_H */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 96981dd06..a9ce6569b 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -417,6 +417,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_x25519		      ( ERRFILE_OTHER | 0x005f0000 )
 #define ERRFILE_des		      ( ERRFILE_OTHER | 0x00600000 )
 #define ERRFILE_editstring	      ( ERRFILE_OTHER | 0x00610000 )
+#define ERRFILE_widget_ui	      ( ERRFILE_OTHER | 0x00620000 )
 
 /** @} */
 
diff --git a/src/include/ipxe/label.h b/src/include/ipxe/label.h
new file mode 100644
index 000000000..48e36cb2f
--- /dev/null
+++ b/src/include/ipxe/label.h
@@ -0,0 +1,42 @@
+#ifndef _IPXE_LABEL_H
+#define _IPXE_LABEL_H
+
+/** @file
+ *
+ * Text label widget
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <curses.h>
+#include <ipxe/widget.h>
+
+/** A text label widget */
+struct label {
+	/** Text widget */
+	struct widget widget;
+	/** Label text */
+	const char *text;
+};
+
+extern struct widget_operations label_operations;
+
+/**
+ * Initialise text label widget
+ *
+ * @v label		Text label widget
+ * @v row		Row
+ * @v col		Starting column
+ * @v width		Width
+ * @v text		Label text
+ */
+static inline __attribute__ (( always_inline )) void
+init_label ( struct label *label, unsigned int row, unsigned int col,
+	     unsigned int width, const char *text ) {
+
+	init_widget ( &label->widget, &label_operations, row, col, width, 0 );
+	label->text = text;
+}
+
+#endif /* _IPXE_LABEL_H */
diff --git a/src/include/ipxe/widget.h b/src/include/ipxe/widget.h
new file mode 100644
index 000000000..fe93b9f5d
--- /dev/null
+++ b/src/include/ipxe/widget.h
@@ -0,0 +1,151 @@
+#ifndef _IPXE_WIDGET_H
+#define _IPXE_WIDGET_H
+
+/** @file
+ *
+ * Text widgets
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <curses.h>
+#include <ipxe/list.h>
+
+/** A text widget set */
+struct widgets {
+	/** List of widgets (in tab order) */
+	struct list_head list;
+	/** Containing window */
+	WINDOW *win;
+};
+
+/** A text widget */
+struct widget {
+	/** List of widgets (in tab order) */
+	struct list_head list;
+	/** Widget operations */
+	struct widget_operations *op;
+
+	/** Row */
+	unsigned int row;
+	/** Starting column */
+	unsigned int col;
+	/** Width */
+	unsigned int width;
+	/** Flags */
+	unsigned int flags;
+};
+
+/** Text widget flags */
+enum widget_flags {
+	/** Widget may have input focus */
+	WIDGET_EDITABLE = 0x0001,
+	/** Widget contains a secret */
+	WIDGET_SECRET = 0x0002,
+};
+
+/** Text widget operations */
+struct widget_operations {
+	/**
+	 * Draw widget
+	 *
+	 * @v widgets		Text widget set
+	 * @v widget		Text widget
+	 */
+	void ( * draw ) ( struct widgets *widgets, struct widget *widget );
+	/**
+	 * Edit widget
+	 *
+	 * @v widgets		Text widget set
+	 * @v widget		Text widget
+	 * @v key		Key pressed by user
+	 * @ret key		Key returned to application, or zero
+	 *
+	 * This will not update the display: you must call the draw()
+	 * method to ensure that any changes to an editable widget are
+	 * displayed to the user.
+	 */
+	int ( * edit ) ( struct widgets *widgets, struct widget *widget,
+			 int key );
+};
+
+/**
+ * Initialise text widget set
+ *
+ * @v widgets		Text widget set
+ * @v win		Containing window
+ */
+static inline __attribute__ (( always_inline )) void
+init_widgets ( struct widgets *widgets, WINDOW *win ) {
+
+	INIT_LIST_HEAD ( &widgets->list );
+	widgets->win = ( win ? win : stdscr );
+}
+
+/**
+ * Initialise text widget
+ *
+ * @v widget		Text widget
+ * @v op		Text widget operations
+ * @v row		Row
+ * @v col		Starting column
+ * @v width		Width
+ */
+static inline __attribute__ (( always_inline )) void
+init_widget ( struct widget *widget, struct widget_operations *op,
+	      unsigned int row, unsigned int col, unsigned int width,
+	      unsigned int flags ) {
+
+	widget->op = op;
+	widget->row = row;
+	widget->col = col;
+	widget->width = width;
+	widget->flags = flags;
+}
+
+/**
+ * Append text widget
+ *
+ * @v widgets		Text widget set
+ * @v widget		Text widget
+ */
+static inline __attribute__ (( always_inline )) void
+add_widget ( struct widgets *widgets, struct widget *widget ) {
+
+	list_add_tail ( &widget->list, &widgets->list );
+}
+
+/**
+ * Draw text widget
+ *
+ * @v widgets		Text widget set
+ * @v widget		Text widget
+ */
+static inline __attribute__ (( always_inline )) void
+draw_widget ( struct widgets *widgets, struct widget *widget ) {
+
+	widget->op->draw ( widgets, widget );
+}
+
+/**
+ * Edit text widget
+ *
+ * @v widgets		Text widget set
+ * @v widget		Text widget
+ * @v key		Key pressed by user
+ * @ret key		Key returned to application, or zero
+ *
+ * This will not update the display: you must call draw_widget() to
+ * ensure that any changes to an editable widget are displayed to the
+ * user.
+ */
+static inline __attribute__ (( always_inline )) int
+edit_widget ( struct widgets *widgets, struct widget *widget, int key ) {
+
+	return widget->op->edit ( widgets, widget, key );
+}
+
+extern int widget_ui ( struct widgets *widgets );
+
+#endif /* _IPXE_WIDGET_H */

From e965f179e1654103eca33feed7a9cc4c51d91be6 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 23 May 2024 13:18:16 +0100
Subject: [PATCH 139/237] [libc] Add stpcpy()

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/string.c       | 17 +++++++++++++++--
 src/include/string.h    |  1 +
 src/tests/string_test.c | 18 ++++++++++++++++++
 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/src/core/string.c b/src/core/string.c
index 9a1b9b72a..364c4cf0e 100644
--- a/src/core/string.c
+++ b/src/core/string.c
@@ -321,9 +321,9 @@ char * strstr ( const char *haystack, const char *needle ) {
  *
  * @v dest		Destination string
  * @v src		Source string
- * @ret dest		Destination string
+ * @ret dnul		Terminating NUL of destination string
  */
-char * strcpy ( char *dest, const char *src ) {
+char * stpcpy ( char *dest, const char *src ) {
 	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
 	uint8_t *dest_bytes = ( ( uint8_t * ) dest );
 
@@ -333,6 +333,19 @@ char * strcpy ( char *dest, const char *src ) {
 		if ( ! *dest_bytes )
 			break;
 	}
+	return ( ( char * ) dest_bytes );
+}
+
+/**
+ * Copy string
+ *
+ * @v dest		Destination string
+ * @v src		Source string
+ * @ret dest		Destination string
+ */
+char * strcpy ( char *dest, const char *src ) {
+
+	stpcpy ( dest, src );
 	return dest;
 }
 
diff --git a/src/include/string.h b/src/include/string.h
index 5f5aecb92..4ee9c7344 100644
--- a/src/include/string.h
+++ b/src/include/string.h
@@ -43,6 +43,7 @@ extern char * __pure strchr ( const char *src, int character ) __nonnull;
 extern char * __pure strrchr ( const char *src, int character ) __nonnull;
 extern char * __pure strstr ( const char *haystack,
 			      const char *needle ) __nonnull;
+extern char * stpcpy ( char *dest, const char *src ) __nonnull;
 extern char * strcpy ( char *dest, const char *src ) __nonnull;
 extern char * strncpy ( char *dest, const char *src, size_t max ) __nonnull;
 extern char * strcat ( char *dest, const char *src ) __nonnull;
diff --git a/src/tests/string_test.c b/src/tests/string_test.c
index 3afb8deb2..c0436c3ad 100644
--- a/src/tests/string_test.c
+++ b/src/tests/string_test.c
@@ -204,6 +204,24 @@ static void string_test_exec ( void ) {
 		free ( dup );
 	}
 
+	/* Test stpcpy() */
+	{
+		const char longer[12] = "duplicateme";
+		const char shorter[6] = "hello";
+		char dest[12];
+		char *dnul;
+
+		dnul = stpcpy ( dest, longer );
+		ok ( *dnul == '\0' );
+		ok ( dnul == &dest[11] );
+		ok ( memcmp ( dest, longer, 12 ) == 0 );
+		dnul = stpcpy ( dest, shorter );
+		ok ( *dnul == '\0' );
+		ok ( dnul == &dest[5] );
+		ok ( memcmp ( dest, shorter, 6 ) == 0 );
+		ok ( memcmp ( ( dest + 6 ), ( longer + 6 ), 6 ) == 0 );
+	}
+
 	/* Test strcpy() */
 	{
 		const char longer[7] = "copyme";

From bb4a10696fb6541fdcfc2c057cf698de37fa5eba Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 14 Jun 2024 11:47:55 +0100
Subject: [PATCH 140/237] [hci] Draw all widgets on the standard screen

The curses concept of a window has been supported but never actively
used in iPXE since the mucurses library was first implemented in 2006.

Simplify the code by removing the ability to place a widget set in a
specified window, and instead use the standard screen for all drawing
operations.

This simplification allows the widget set parameter to be omitted for
the draw_widget() and edit_widget() operations, since the only reason
for its inclusion was to provide access to the specified window.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/mucurses/widgets/editbox.c | 11 ++++-------
 src/hci/mucurses/widgets/label.c   |  9 +++------
 src/hci/tui/login_ui.c             |  2 +-
 src/hci/tui/settings_ui.c          |  6 +++---
 src/hci/tui/widget_ui.c            |  6 +++---
 src/include/ipxe/widget.h          | 23 +++++++----------------
 6 files changed, 21 insertions(+), 36 deletions(-)

diff --git a/src/hci/mucurses/widgets/editbox.c b/src/hci/mucurses/widgets/editbox.c
index 79e7cdd16..c024688ab 100644
--- a/src/hci/mucurses/widgets/editbox.c
+++ b/src/hci/mucurses/widgets/editbox.c
@@ -39,10 +39,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /**
  * Draw text box widget
  *
- * @v widgets		Text widget set
  * @v widget		Text widget
  */
-static void draw_editbox ( struct widgets *widgets, struct widget *widget ) {
+static void draw_editbox ( struct widget *widget ) {
 	struct edit_box *box = container_of ( widget, struct edit_box, widget );
 	const char *content = *(box->string.buf);
 	size_t width = widget->width;
@@ -79,21 +78,19 @@ static void draw_editbox ( struct widgets *widgets, struct widget *widget ) {
 
 	/* Print box content and move cursor */
 	color_set ( CPAIR_EDIT, NULL );
-	mvwprintw ( widgets->win, widget->row, widget->col, "%s", buf );
-	wmove ( widgets->win, widget->row, ( widget->col + cursor_offset ) );
+	mvprintw ( widget->row, widget->col, "%s", buf );
+	move ( widget->row, ( widget->col + cursor_offset ) );
 	color_set ( CPAIR_NORMAL, NULL );
 }
 
 /**
  * Edit text box widget
  *
- * @v widgets		Text widget set
  * @v widget		Text widget
  * @v key		Key pressed by user
  * @ret key		Key returned to application, or zero
  */
-static int edit_editbox ( struct widgets *widgets __unused,
-			  struct widget *widget, int key ) {
+static int edit_editbox ( struct widget *widget, int key ) {
 	struct edit_box *box = container_of ( widget, struct edit_box, widget );
 
 	return edit_string ( &box->string, key );
diff --git a/src/hci/mucurses/widgets/label.c b/src/hci/mucurses/widgets/label.c
index 29645a6ad..29057f089 100644
--- a/src/hci/mucurses/widgets/label.c
+++ b/src/hci/mucurses/widgets/label.c
@@ -36,10 +36,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /**
  * Draw text label widget
  *
- * @v widgets		Text widget set
  * @v widget		Text widget
  */
-static void draw_label ( struct widgets *widgets, struct widget *widget ) {
+static void draw_label ( struct widget *widget ) {
 	struct label *label = container_of ( widget, struct label, widget );
 	unsigned int width = widget->width;
 	unsigned int col = widget->col;
@@ -51,20 +50,18 @@ static void draw_label ( struct widgets *widgets, struct widget *widget ) {
 
 	/* Print label content */
 	attron ( A_BOLD );
-	mvwprintw ( widgets->win, widget->row, col, "%s", text );
+	mvprintw ( widget->row, col, "%s", text );
 	attroff ( A_BOLD );
 }
 
 /**
  * Edit text label widget
  *
- * @v widgets		Text widget set
  * @v widget		Text widget
  * @v key		Key pressed by user
  * @ret key		Key returned to application, or zero
  */
-static int edit_label ( struct widgets *widgets __unused,
-			struct widget *widget __unused, int key ) {
+static int edit_label ( struct widget *widget __unused, int key ) {
 
 	/* Cannot be edited */
 	return key;
diff --git a/src/hci/tui/login_ui.c b/src/hci/tui/login_ui.c
index b76afb92e..9c5148865 100644
--- a/src/hci/tui/login_ui.c
+++ b/src/hci/tui/login_ui.c
@@ -67,7 +67,7 @@ int login_ui ( void ) {
 
 	/* Construct user interface */
 	memset ( &widgets, 0, sizeof ( widgets ) );
-	init_widgets ( &widgets.widgets, NULL );
+	init_widgets ( &widgets.widgets );
 	init_label ( &widgets.username_label, USERNAME_LABEL_ROW, WIDGET_COL,
 		     WIDGET_WIDTH, "Username" );
 	init_label ( &widgets.password_label, PASSWORD_LABEL_ROW, WIDGET_COL,
diff --git a/src/hci/tui/settings_ui.c b/src/hci/tui/settings_ui.c
index 53bf24d1a..045d97853 100644
--- a/src/hci/tui/settings_ui.c
+++ b/src/hci/tui/settings_ui.c
@@ -253,7 +253,7 @@ static void draw_setting_row ( struct settings_ui *ui ) {
 static int edit_setting ( struct settings_ui *ui, int key ) {
 	assert ( ui->row.setting.name != NULL );
 	ui->row.editing = 1;
-	return edit_widget ( &ui->widgets, &ui->row.editbox.widget, key );
+	return edit_widget ( &ui->row.editbox.widget, key );
 }
 
 /**
@@ -457,7 +457,7 @@ static int main_loop ( struct settings *settings ) {
 	/* Print initial screen content */
 	color_set ( CPAIR_NORMAL, NULL );
 	memset ( &ui, 0, sizeof ( ui ) );
-	init_widgets ( &ui.widgets, NULL );
+	init_widgets ( &ui.widgets );
 	select_settings ( &ui, settings );
 
 	while ( 1 ) {
@@ -481,7 +481,7 @@ static int main_loop ( struct settings *settings ) {
 			assert ( ui.row.setting.name != NULL );
 
 			/* Redraw edit box */
-			draw_widget ( &ui.widgets, &ui.row.editbox.widget );
+			draw_widget ( &ui.row.editbox.widget );
 
 			/* Process keypress */
 			key = edit_setting ( &ui, getkey ( 0 ) );
diff --git a/src/hci/tui/widget_ui.c b/src/hci/tui/widget_ui.c
index 5079eefdd..961545cf4 100644
--- a/src/hci/tui/widget_ui.c
+++ b/src/hci/tui/widget_ui.c
@@ -68,7 +68,7 @@ static int widget_ui_loop ( struct widgets *widgets ) {
 
 	/* Draw all widgets */
 	list_for_each_entry ( widget, &widgets->list, list )
-		draw_widget ( widgets, widget );
+		draw_widget ( widget );
 
 	/* Count editable widgets */
 	count = 0;
@@ -85,10 +85,10 @@ static int widget_ui_loop ( struct widgets *widgets ) {
 			return -ENOENT;
 
 		/* Redraw current widget */
-		draw_widget ( widgets, widget );
+		draw_widget ( widget );
 
 		/* Process keypress */
-		key = edit_widget ( widgets, widget, getkey ( 0 ) );
+		key = edit_widget ( widget, getkey ( 0 ) );
 		switch ( key ) {
 		case KEY_UP:
 			if ( current > 0 )
diff --git a/src/include/ipxe/widget.h b/src/include/ipxe/widget.h
index fe93b9f5d..0d8af1030 100644
--- a/src/include/ipxe/widget.h
+++ b/src/include/ipxe/widget.h
@@ -16,8 +16,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 struct widgets {
 	/** List of widgets (in tab order) */
 	struct list_head list;
-	/** Containing window */
-	WINDOW *win;
 };
 
 /** A text widget */
@@ -50,14 +48,12 @@ struct widget_operations {
 	/**
 	 * Draw widget
 	 *
-	 * @v widgets		Text widget set
 	 * @v widget		Text widget
 	 */
-	void ( * draw ) ( struct widgets *widgets, struct widget *widget );
+	void ( * draw ) ( struct widget *widget );
 	/**
 	 * Edit widget
 	 *
-	 * @v widgets		Text widget set
 	 * @v widget		Text widget
 	 * @v key		Key pressed by user
 	 * @ret key		Key returned to application, or zero
@@ -66,21 +62,18 @@ struct widget_operations {
 	 * method to ensure that any changes to an editable widget are
 	 * displayed to the user.
 	 */
-	int ( * edit ) ( struct widgets *widgets, struct widget *widget,
-			 int key );
+	int ( * edit ) ( struct widget *widget, int key );
 };
 
 /**
  * Initialise text widget set
  *
  * @v widgets		Text widget set
- * @v win		Containing window
  */
 static inline __attribute__ (( always_inline )) void
-init_widgets ( struct widgets *widgets, WINDOW *win ) {
+init_widgets ( struct widgets *widgets ) {
 
 	INIT_LIST_HEAD ( &widgets->list );
-	widgets->win = ( win ? win : stdscr );
 }
 
 /**
@@ -119,19 +112,17 @@ add_widget ( struct widgets *widgets, struct widget *widget ) {
 /**
  * Draw text widget
  *
- * @v widgets		Text widget set
  * @v widget		Text widget
  */
 static inline __attribute__ (( always_inline )) void
-draw_widget ( struct widgets *widgets, struct widget *widget ) {
+draw_widget ( struct widget *widget ) {
 
-	widget->op->draw ( widgets, widget );
+	widget->op->draw ( widget );
 }
 
 /**
  * Edit text widget
  *
- * @v widgets		Text widget set
  * @v widget		Text widget
  * @v key		Key pressed by user
  * @ret key		Key returned to application, or zero
@@ -141,9 +132,9 @@ draw_widget ( struct widgets *widgets, struct widget *widget ) {
  * user.
  */
 static inline __attribute__ (( always_inline )) int
-edit_widget ( struct widgets *widgets, struct widget *widget, int key ) {
+edit_widget ( struct widget *widget, int key ) {
 
-	return widget->op->edit ( widgets, widget, key );
+	return widget->op->edit ( widget, key );
 }
 
 extern int widget_ui ( struct widgets *widgets );

From bf98eae5daeb0b9281562e70fdf3768a629adde8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 14 Jun 2024 11:51:57 +0100
Subject: [PATCH 141/237] [hci] Split out msg() and alert() from settings UI
 code

The msg() and alert() functions currently defined in settings_ui.c
provide a general-purpose facility for printing messages centred on
the screen.

Split this out to a separate file to allow for reuse by the form
presentation code.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/tui/message.c      | 109 +++++++++++++++++++++++++++++++++++++
 src/hci/tui/settings_ui.c  |  78 ++------------------------
 src/include/ipxe/message.h |  16 ++++++
 3 files changed, 131 insertions(+), 72 deletions(-)
 create mode 100644 src/hci/tui/message.c
 create mode 100644 src/include/ipxe/message.h

diff --git a/src/hci/tui/message.c b/src/hci/tui/message.c
new file mode 100644
index 000000000..e3331d655
--- /dev/null
+++ b/src/hci/tui/message.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Message printing
+ *
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <ipxe/ansicol.h>
+#include <ipxe/message.h>
+
+/**
+ * Print message centred on specified row
+ *
+ * @v row		Row
+ * @v fmt		printf() format string
+ * @v args		printf() argument list
+ */
+static void vmsg ( unsigned int row, const char *fmt, va_list args ) {
+	char buf[COLS];
+	size_t len;
+
+	len = vsnprintf ( buf, sizeof ( buf ), fmt, args );
+	mvprintw ( row, ( ( COLS - len ) / 2 ), "%s", buf );
+}
+
+/**
+ * Print message centred on specified row
+ *
+ * @v row		Row
+ * @v fmt		printf() format string
+ * @v ..		printf() arguments
+ */
+void msg ( unsigned int row, const char *fmt, ... ) {
+	va_list args;
+
+	va_start ( args, fmt );
+	vmsg ( row, fmt, args );
+	va_end ( args );
+}
+
+/**
+ * Clear message on specified row
+ *
+ * @v row		Row
+ */
+void clearmsg ( unsigned int row ) {
+	move ( row, 0 );
+	clrtoeol();
+}
+
+/**
+ * Show alert message
+ *
+ * @v row		Row
+ * @v fmt		printf() format string
+ * @v args		printf() argument list
+ */
+static void valert ( unsigned int row, const char *fmt, va_list args ) {
+
+	clearmsg ( row );
+	color_set ( CPAIR_ALERT, NULL );
+	vmsg ( row, fmt, args );
+	sleep ( 2 );
+	color_set ( CPAIR_NORMAL, NULL );
+	clearmsg ( row );
+}
+
+/**
+ * Show alert message
+ *
+ * @v row		Row
+ * @v fmt		printf() format string
+ * @v ...		printf() arguments
+ */
+void alert ( unsigned int row, const char *fmt, ... ) {
+	va_list args;
+
+	va_start ( args, fmt );
+	valert ( row, fmt, args );
+	va_end ( args );
+}
diff --git a/src/hci/tui/settings_ui.c b/src/hci/tui/settings_ui.c
index 045d97853..10c942321 100644
--- a/src/hci/tui/settings_ui.c
+++ b/src/hci/tui/settings_ui.c
@@ -35,6 +35,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/keys.h>
 #include <ipxe/ansicol.h>
 #include <ipxe/jumpscroll.h>
+#include <ipxe/message.h>
 #include <ipxe/settings_ui.h>
 #include <config/branding.h>
 
@@ -266,75 +267,6 @@ static int save_setting ( struct settings_ui *ui ) {
 	return storef_setting ( ui->settings, &ui->row.setting, ui->row.buf );
 }
 
-/**
- * Print message centred on specified row
- *
- * @v row		Row
- * @v fmt		printf() format string
- * @v args		printf() argument list
- */
-static void vmsg ( unsigned int row, const char *fmt, va_list args ) {
-	char buf[COLS];
-	size_t len;
-
-	len = vsnprintf ( buf, sizeof ( buf ), fmt, args );
-	mvprintw ( row, ( ( COLS - len ) / 2 ), "%s", buf );
-}
-
-/**
- * Print message centred on specified row
- *
- * @v row		Row
- * @v fmt		printf() format string
- * @v ..		printf() arguments
- */
-static void msg ( unsigned int row, const char *fmt, ... ) {
-	va_list args;
-
-	va_start ( args, fmt );
-	vmsg ( row, fmt, args );
-	va_end ( args );
-}
-
-/**
- * Clear message on specified row
- *
- * @v row		Row
- */
-static void clearmsg ( unsigned int row ) {
-	move ( row, 0 );
-	clrtoeol();
-}
-
-/**
- * Print alert message
- *
- * @v fmt		printf() format string
- * @v args		printf() argument list
- */
-static void valert ( const char *fmt, va_list args ) {
-	clearmsg ( ALERT_ROW );
-	color_set ( CPAIR_ALERT, NULL );
-	vmsg ( ALERT_ROW, fmt, args );
-	sleep ( 2 );
-	color_set ( CPAIR_NORMAL, NULL );
-	clearmsg ( ALERT_ROW );
-}
-
-/**
- * Print alert message
- *
- * @v fmt		printf() format string
- * @v ...		printf() arguments
- */
-static void alert ( const char *fmt, ... ) {
-	va_list args;
-
-	va_start ( args, fmt );
-	valert ( fmt, args );
-	va_end ( args );
-}
-
 /**
  * Draw title row
  *
@@ -488,8 +420,10 @@ static int main_loop ( struct settings *settings ) {
 			switch ( key ) {
 			case CR:
 			case LF:
-				if ( ( rc = save_setting ( &ui ) ) != 0 )
-					alert ( " %s ", strerror ( rc ) );
+				if ( ( rc = save_setting ( &ui ) ) != 0 ) {
+					alert ( ALERT_ROW, " %s ",
+						strerror ( rc ) );
+				}
 				/* Fall through */
 			case CTRL_C:
 				select_setting_row ( &ui, ui.scroll.current );
@@ -526,7 +460,7 @@ static int main_loop ( struct settings *settings ) {
 				break;
 			if ( ( rc = delete_setting ( ui.settings,
 						     &ui.row.setting ) ) != 0 ){
-				alert ( " %s ", strerror ( rc ) );
+				alert ( ALERT_ROW, " %s ", strerror ( rc ) );
 			}
 			select_setting_row ( &ui, ui.scroll.current );
 			redraw = 1;
diff --git a/src/include/ipxe/message.h b/src/include/ipxe/message.h
new file mode 100644
index 000000000..e2e783740
--- /dev/null
+++ b/src/include/ipxe/message.h
@@ -0,0 +1,16 @@
+#ifndef _IPXE_MESSAGE_H
+#define _IPXE_MESSAGE_H
+
+/** @file
+ *
+ * Message printing
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+extern void msg ( unsigned int row, const char *fmt, ... );
+extern void clearmsg ( unsigned int row );
+extern void alert ( unsigned int row, const char *fmt, ... );
+
+#endif /* _IPXE_MESSAGE_H */

From 76e0933d781474acf52000cba0afaebe32361667 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 18 Jun 2024 15:17:03 -0700
Subject: [PATCH 142/237] [hci] Rename "item" command's first parameter from
 "label" to "name"

Switch terminology for the "item" command from "item <label> <text>"
to "item <name> <text>", in preparation for repurposing the "item"
command to cover interactive forms as well as menus.

Since this renaming affects only a positional parameter, it does not
break compatibility with any existing scripts.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/menu.c             | 22 +++++++++++-----------
 src/hci/commands/menu_cmd.c | 12 ++++++------
 src/hci/tui/menu_ui.c       | 26 +++++++++++++-------------
 src/include/ipxe/menu.h     |  6 +++---
 4 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/src/core/menu.c b/src/core/menu.c
index ab5b0c7f5..b9d0d5b77 100644
--- a/src/core/menu.c
+++ b/src/core/menu.c
@@ -94,20 +94,20 @@ struct menu * create_menu ( const char *name, const char *title ) {
  * Add menu item
  *
  * @v menu		Menu
- * @v label		Label, or NULL
+ * @v name		Name, or NULL
  * @v text		Text, or NULL
  * @v shortcut		Shortcut key
  * @v is_default	Item is the default item
  * @ret item		Menu item, or NULL on failure
  */
-struct menu_item * add_menu_item ( struct menu *menu, const char *label,
+struct menu_item * add_menu_item ( struct menu *menu, const char *name,
 				   const char *text, int shortcut,
 				   int is_default ) {
-	size_t label_len;
+	size_t name_len;
 	size_t text_len;
 	size_t len;
 	struct menu_item *item;
-	char *label_copy;
+	char *name_copy;
 	char *text_copy;
 
 	/* Use empty text if none given */
@@ -115,19 +115,19 @@ struct menu_item * add_menu_item ( struct menu *menu, const char *label,
 		text = "";
 
 	/* Allocate item */
-	label_len = ( label ? ( strlen ( label ) + 1 /* NUL */ ) : 0 );
+	name_len = ( name ? ( strlen ( name ) + 1 /* NUL */ ) : 0 );
 	text_len = ( strlen ( text ) + 1 /* NUL */ );
-	len = ( sizeof ( *item ) + label_len + text_len );
+	len = ( sizeof ( *item ) + name_len + text_len );
 	item = zalloc ( len );
 	if ( ! item )
 		return NULL;
-	label_copy = ( ( void * ) ( item + 1 ) );
-	text_copy = ( label_copy + label_len );
+	name_copy = ( ( void * ) ( item + 1 ) );
+	text_copy = ( name_copy + name_len );
 
 	/* Initialise item */
-	if ( label ) {
-		strcpy ( label_copy, label );
-		item->label = label_copy;
+	if ( name ) {
+		strcpy ( name_copy, name );
+		item->name = name_copy;
 	}
 	strcpy ( text_copy, text );
 	item->text = text_copy;
diff --git a/src/hci/commands/menu_cmd.c b/src/hci/commands/menu_cmd.c
index 76bce8695..8b6d2a9f1 100644
--- a/src/hci/commands/menu_cmd.c
+++ b/src/hci/commands/menu_cmd.c
@@ -135,7 +135,7 @@ static struct option_descriptor item_opts[] = {
 /** "item" command descriptor */
 static struct command_descriptor item_cmd =
 	COMMAND_DESC ( struct item_options, item_opts, 0, MAX_ARGUMENTS,
-		       "[<label> [<text>]]" );
+		       "[<name> [<text>]]" );
 
 /**
  * The "item" command
@@ -148,7 +148,7 @@ static int item_exec ( int argc, char **argv ) {
 	struct item_options opts;
 	struct menu *menu;
 	struct menu_item *item;
-	char *label = NULL;
+	char *name = NULL;
 	char *text = NULL;
 	int rc;
 
@@ -156,9 +156,9 @@ static int item_exec ( int argc, char **argv ) {
 	if ( ( rc = parse_options ( argc, argv, &item_cmd, &opts ) ) != 0 )
 		goto err_parse_options;
 
-	/* Parse label, if present */
+	/* Parse name, if present */
 	if ( ! opts.is_gap )
-		label = argv[optind++]; /* May be NULL */
+		name = argv[optind++]; /* May be NULL */
 
 	/* Parse text, if present */
 	if ( optind < argc ) {
@@ -174,7 +174,7 @@ static int item_exec ( int argc, char **argv ) {
 		goto err_parse_menu;
 
 	/* Add menu item */
-	item = add_menu_item ( menu, label, ( text ? text : "" ),
+	item = add_menu_item ( menu, name, ( text ? text : "" ),
 			       opts.key, opts.is_default );
 	if ( ! item ) {
 		rc = -ENOMEM;
@@ -257,7 +257,7 @@ static int choose_exec ( int argc, char **argv ) {
 
 	/* Store setting */
 	if ( ( rc = storef_setting ( setting.settings, &setting.setting,
-				     item->label ) ) != 0 ) {
+				     item->name ) ) != 0 ) {
 		printf ( "Could not store \"%s\": %s\n",
 			 setting.setting.name, strerror ( rc ) );
 		goto err_store;
diff --git a/src/hci/tui/menu_ui.c b/src/hci/tui/menu_ui.c
index f9dd9d100..cac61a7da 100644
--- a/src/hci/tui/menu_ui.c
+++ b/src/hci/tui/menu_ui.c
@@ -99,7 +99,7 @@ static void draw_menu_item ( struct menu_ui *ui, unsigned int index ) {
 	if ( item ) {
 
 		/* Draw separators in a different colour */
-		if ( ! item->label )
+		if ( ! item->name )
 			color_set ( CPAIR_SEPARATOR, NULL );
 
 		/* Highlight if this is the selected item */
@@ -225,7 +225,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 						continue;
 					}
 					ui->scroll.current = i;
-					if ( item->label ) {
+					if ( item->name ) {
 						chosen = 1;
 					} else {
 						move = +1;
@@ -239,7 +239,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 		while ( move ) {
 			move = jump_scroll_move ( &ui->scroll, move );
 			item = menu_item ( ui->menu, ui->scroll.current );
-			if ( item->label )
+			if ( item->name )
 				break;
 		}
 
@@ -254,7 +254,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 		/* Record selection */
 		item = menu_item ( ui->menu, ui->scroll.current );
 		assert ( item != NULL );
-		assert ( item->label != NULL );
+		assert ( item->name != NULL );
 		*selected = item;
 
 	} while ( ( rc == 0 ) && ! chosen );
@@ -275,7 +275,7 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 	struct menu_item *item;
 	struct menu_ui ui;
 	char buf[ MENU_COLS + 1 /* NUL */ ];
-	int labelled_count = 0;
+	int named_count = 0;
 	int rc;
 
 	/* Initialise UI */
@@ -284,12 +284,12 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 	ui.scroll.rows = MENU_ROWS;
 	ui.timeout = timeout;
 	list_for_each_entry ( item, &menu->items, list ) {
-		if ( item->label ) {
-			if ( ! labelled_count )
+		if ( item->name ) {
+			if ( ! named_count )
 				ui.scroll.current = ui.scroll.count;
-			labelled_count++;
+			named_count++;
 			if ( select ) {
-				if ( strcmp ( select, item->label ) == 0 )
+				if ( strcmp ( select, item->name ) == 0 )
 					ui.scroll.current = ui.scroll.count;
 			} else {
 				if ( item->is_default )
@@ -298,10 +298,10 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 		}
 		ui.scroll.count++;
 	}
-	if ( ! labelled_count ) {
-		/* Menus with no labelled items cannot be selected
-		 * from, and will seriously confuse the navigation
-		 * logic.  Refuse to display any such menus.
+	if ( ! named_count ) {
+		/* Menus with no named items cannot be selected from,
+		 * and will seriously confuse the navigation logic.
+		 * Refuse to display any such menus.
 		 */
 		return -ENOENT;
 	}
diff --git a/src/include/ipxe/menu.h b/src/include/ipxe/menu.h
index 3cc99be48..d11a92559 100644
--- a/src/include/ipxe/menu.h
+++ b/src/include/ipxe/menu.h
@@ -27,8 +27,8 @@ struct menu {
 struct menu_item {
 	/** List of menu items */
 	struct list_head list;
-	/** Label */
-	const char *label;
+	/** Name */
+	const char *name;
 	/** Text */
 	const char *text;
 	/** Shortcut key */
@@ -38,7 +38,7 @@ struct menu_item {
 };
 
 extern struct menu * create_menu ( const char *name, const char *title );
-extern struct menu_item * add_menu_item ( struct menu *menu, const char *label,
+extern struct menu_item * add_menu_item ( struct menu *menu, const char *name,
 					  const char *text, int shortcut,
 					  int is_default );
 extern void destroy_menu ( struct menu *menu );

From 122777f789a68512593e4aa6da3ace3d0d8664ec Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 20 Jun 2024 12:21:57 -0700
Subject: [PATCH 143/237] [hci] Allow tab key to be used to cycle through UI
 elements

Add support for wraparound scrolling and allow the tab key to be used
to move forward through a list of elements, wrapping back around to
the beginning of the list on overflow.

This is mildly useful for a menu, and likely to be a strong user
expectation for an interactive form.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/jumpscroll.c          | 77 ++++++++++++++++++++++-------------
 src/hci/tui/menu_ui.c         |  6 +--
 src/hci/tui/settings_ui.c     |  2 +-
 src/include/ipxe/jumpscroll.h | 36 +++++++++++++++-
 4 files changed, 87 insertions(+), 34 deletions(-)

diff --git a/src/hci/jumpscroll.c b/src/hci/jumpscroll.c
index dd6bcac2b..641f781a0 100644
--- a/src/hci/jumpscroll.c
+++ b/src/hci/jumpscroll.c
@@ -39,7 +39,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v key		Key pressed by user
  * @ret move		Scroller movement, or zero
  */
-int jump_scroll_key ( struct jump_scroller *scroll, int key ) {
+unsigned int jump_scroll_key ( struct jump_scroller *scroll, int key ) {
+	unsigned int flags = 0;
+	int16_t delta;
 
 	/* Sanity checks */
 	assert ( scroll->rows != 0 );
@@ -52,20 +54,32 @@ int jump_scroll_key ( struct jump_scroller *scroll, int key ) {
 	/* Handle key, if applicable */
 	switch ( key ) {
 	case KEY_UP:
-		return -1;
+		delta = -1;
+		break;
+	case TAB:
+		flags = SCROLL_WRAP;
+		/* fall through */
 	case KEY_DOWN:
-		return +1;
+		delta = +1;
+		break;
 	case KEY_PPAGE:
-		return ( scroll->first - scroll->current - 1 );
+		delta = ( scroll->first - scroll->current - 1 );
+		break;
 	case KEY_NPAGE:
-		return ( scroll->first - scroll->current + scroll->rows );
+		delta = ( scroll->first - scroll->current + scroll->rows );
+		break;
 	case KEY_HOME:
-		return -( scroll->count );
+		delta = -( scroll->count );
+		break;
 	case KEY_END:
-		return +( scroll->count );
+		delta = +( scroll->count );
+		break;
 	default:
-		return 0;
+		delta = 0;
+		break;
 	}
+
+	return ( SCROLL ( delta ) | flags );
 }
 
 /**
@@ -75,7 +89,9 @@ int jump_scroll_key ( struct jump_scroller *scroll, int key ) {
  * @v move		Scroller movement
  * @ret move		Continuing scroller movement (if applicable)
  */
-int jump_scroll_move ( struct jump_scroller *scroll, int move ) {
+unsigned int jump_scroll_move ( struct jump_scroller *scroll,
+				unsigned int move ) {
+	int16_t delta = SCROLL_DELTA ( move );
 	int current = scroll->current;
 	int last = ( scroll->count - 1 );
 
@@ -84,30 +100,35 @@ int jump_scroll_move ( struct jump_scroller *scroll, int move ) {
 	assert ( scroll->count != 0 );
 
 	/* Move to the new current item */
-	current += move;
+	current += delta;
+
+	/* Default to continuing movement in the same direction */
+	delta = ( ( delta >= 0 ) ? +1 : -1 );
 
 	/* Check for start/end of list */
-	if ( current < 0 ) {
-		/* We have attempted to move before the start of the
-		 * list.  Move to the start of the list and continue
-		 * moving forwards (if applicable).
-		 */
-		scroll->current = 0;
-		return +1;
-	} else if ( current > last ) {
-		/* We have attempted to move after the end of the
-		 * list.  Move to the end of the list and continue
-		 * moving backwards (if applicable).
-		 */
-		scroll->current = last;
-		return -1;
-	} else {
-		/* Update the current item and continue moving in the
-		 * same direction (if applicable).
+	if ( ( current >= 0 ) && ( current <= last ) ) {
+		/* We are still within the list.  Update the current
+		 * item and continue moving in the same direction (if
+		 * applicable).
 		 */
 		scroll->current = current;
-		return ( ( move > 0 ) ? +1 : -1 );
+	} else {
+		/* We have attempted to move outside the list.  If we
+		 * are wrapping around, then continue in the same
+		 * direction (if applicable), otherwise reverse.
+		 */
+		if ( ! ( move & SCROLL_WRAP ) )
+			delta = -delta;
+
+		/* Move to start or end of list as appropriate */
+		if ( delta >= 0 ) {
+			scroll->current = 0;
+		} else {
+			scroll->current = last;
+		}
 	}
+
+	return ( SCROLL ( delta ) | ( move & SCROLL_FLAGS ) );
 }
 
 /**
diff --git a/src/hci/tui/menu_ui.c b/src/hci/tui/menu_ui.c
index cac61a7da..067e2d8ca 100644
--- a/src/hci/tui/menu_ui.c
+++ b/src/hci/tui/menu_ui.c
@@ -175,9 +175,9 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 	struct menu_item *item;
 	unsigned long timeout;
 	unsigned int previous;
+	unsigned int move;
 	int key;
 	int i;
-	int move;
 	int chosen = 0;
 	int rc = 0;
 
@@ -192,7 +192,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 		ui->timeout -= timeout;
 
 		/* Get key */
-		move = 0;
+		move = SCROLL_NONE;
 		key = getkey ( timeout );
 		if ( key < 0 ) {
 			/* Choose default if we finally time out */
@@ -228,7 +228,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 					if ( item->name ) {
 						chosen = 1;
 					} else {
-						move = +1;
+						move = SCROLL_DOWN;
 					}
 				}
 				break;
diff --git a/src/hci/tui/settings_ui.c b/src/hci/tui/settings_ui.c
index 10c942321..bc08750a0 100644
--- a/src/hci/tui/settings_ui.c
+++ b/src/hci/tui/settings_ui.c
@@ -381,8 +381,8 @@ static void select_settings ( struct settings_ui *ui,
 static int main_loop ( struct settings *settings ) {
 	struct settings_ui ui;
 	unsigned int previous;
+	unsigned int move;
 	int redraw = 1;
-	int move;
 	int key;
 	int rc;
 
diff --git a/src/include/ipxe/jumpscroll.h b/src/include/ipxe/jumpscroll.h
index 7a5b111c1..470f08e71 100644
--- a/src/include/ipxe/jumpscroll.h
+++ b/src/include/ipxe/jumpscroll.h
@@ -9,6 +9,8 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stdint.h>
+
 /** A jump scroller */
 struct jump_scroller {
 	/** Maximum number of visible rows */
@@ -21,6 +23,35 @@ struct jump_scroller {
 	unsigned int first;
 };
 
+/**
+ * Construct scroll movement
+ *
+ * @v delta		Change in scroller position
+ * @ret move		Scroll movement
+ */
+#define SCROLL( delta ) ( ( unsigned int ) ( uint16_t ) ( int16_t ) (delta) )
+
+/**
+ * Extract change in scroller position
+ *
+ * @v move		Scroll movement
+ * @ret delta		Change in scroller position
+ */
+#define SCROLL_DELTA( scroll ) ( ( int16_t ) ( (scroll) & 0x0000ffffUL ) )
+
+/** Scroll movement flags */
+#define SCROLL_FLAGS	0xffff0000UL
+#define SCROLL_WRAP	0x80000000UL	/**< Wrap around scrolling */
+
+/** Do not scroll */
+#define SCROLL_NONE SCROLL ( 0 )
+
+/** Scroll up by one line */
+#define SCROLL_UP SCROLL ( -1 )
+
+/** Scroll down by one line */
+#define SCROLL_DOWN SCROLL ( +1 )
+
 /**
  * Check if jump scroller is currently on first page
  *
@@ -43,8 +74,9 @@ static inline int jump_scroll_is_last ( struct jump_scroller *scroll ) {
 	return ( ( scroll->first + scroll->rows ) >= scroll->count );
 }
 
-extern int jump_scroll_key ( struct jump_scroller *scroll, int key );
-extern int jump_scroll_move ( struct jump_scroller *scroll, int move );
+extern unsigned int jump_scroll_key ( struct jump_scroller *scroll, int key );
+extern unsigned int jump_scroll_move ( struct jump_scroller *scroll,
+				       unsigned int move );
 extern int jump_scroll ( struct jump_scroller *scroll );
 
 #endif /* _IPXE_JUMPSCROLL_H */

From 5719cde838b6e86a02831373dae81642653b872f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 20 Jun 2024 14:24:53 -0700
Subject: [PATCH 144/237] [dynui] Generalise the concept of a menu to a dynamic
 user interface

We currently have an abstract model of a dynamic menu as a list of
items, each of which has a name, a description, and assorted metadata
such as a shortcut key.  The "menu" and "item" commands construct
representations in this abstract model, and the "choose" command then
presents the items as a single-choice menu, with the selected item's
name used as the output value.

This same abstraction may be used to model a dynamic form as a list of
editable items, each of which has a corresponding setting name, an
optional description label, and assorted metadata such as a shortcut
key.  By defining a "form" command as an alias for the "menu" command,
we could construct and present forms using commands such as:

  #!ipxe
  form                     Login to ${url}
  item          username   Username or email address
  item --secret password   Password
  present

or

  #!ipxe
  form                Configure IPv4 networking for ${netX/ifname}
  item netX/ip        IPv4 address
  item netX/netmask   Subnet mask
  item netX/gateway   Gateway address
  item netX/dns       DNS server address
  present

Reusing the same abstract model for both menus and forms allows us to
minimise the increase in code size, since the implementation of the
"form" and "item" commands is essentially zero-cost.

Rename everything within the abstract data model from "menu" to
"dynamic user interface" to reflect this generalisation.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config.c                          |   2 +-
 src/core/{menu.c => dynui.c}                 | 110 +++++++++----------
 src/core/parseopt.c                          |  18 +--
 src/hci/commands/{menu_cmd.c => dynui_cmd.c} | 109 +++++++++---------
 src/hci/tui/menu_ui.c                        |  43 ++++----
 src/include/ipxe/dynui.h                     |  50 +++++++++
 src/include/ipxe/errfile.h                   |   2 +-
 src/include/ipxe/menu.h                      |  49 ---------
 src/include/ipxe/parseopt.h                  |   4 +-
 9 files changed, 195 insertions(+), 192 deletions(-)
 rename src/core/{menu.c => dynui.c} (54%)
 rename src/hci/commands/{menu_cmd.c => dynui_cmd.c} (73%)
 create mode 100644 src/include/ipxe/dynui.h
 delete mode 100644 src/include/ipxe/menu.h

diff --git a/src/config/config.c b/src/config/config.c
index 209336c25..666712304 100644
--- a/src/config/config.c
+++ b/src/config/config.c
@@ -225,7 +225,7 @@ REQUIRE_OBJECT ( dhcp_cmd );
 REQUIRE_OBJECT ( sanboot_cmd );
 #endif
 #ifdef MENU_CMD
-REQUIRE_OBJECT ( menu_cmd );
+REQUIRE_OBJECT ( dynui_cmd );
 #endif
 #ifdef LOGIN_CMD
 REQUIRE_OBJECT ( login_cmd );
diff --git a/src/core/menu.c b/src/core/dynui.c
similarity index 54%
rename from src/core/menu.c
rename to src/core/dynui.c
index b9d0d5b77..6e053b905 100644
--- a/src/core/menu.c
+++ b/src/core/dynui.c
@@ -25,7 +25,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
- * Menu selection
+ * Dynamic user interfaces
  *
  */
 
@@ -33,80 +33,80 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <string.h>
 #include <assert.h>
 #include <ipxe/list.h>
-#include <ipxe/menu.h>
+#include <ipxe/dynui.h>
 
-/** List of all menus */
-static LIST_HEAD ( menus );
+/** List of all dynamic user interfaces */
+static LIST_HEAD ( dynamic_uis );
 
 /**
- * Create menu
+ * Create dynamic user interface
  *
- * @v name		Menu name, or NULL
- * @v title		Menu title, or NULL
- * @ret menu		Menu, or NULL on failure
+ * @v name		User interface name, or NULL
+ * @v title		User interface title, or NULL
+ * @ret dynui		Dynamic user interface, or NULL on failure
  */
-struct menu * create_menu ( const char *name, const char *title ) {
+struct dynamic_ui * create_dynui ( const char *name, const char *title ) {
+	struct dynamic_ui *dynui;
 	size_t name_len;
 	size_t title_len;
 	size_t len;
-	struct menu *menu;
 	char *name_copy;
 	char *title_copy;
 
-	/* Destroy any existing menu of this name */
-	menu = find_menu ( name );
-	if ( menu )
-		destroy_menu ( menu );
+	/* Destroy any existing user interface of this name */
+	dynui = find_dynui ( name );
+	if ( dynui )
+		destroy_dynui ( dynui );
 
 	/* Use empty title if none given */
 	if ( ! title )
 		title = "";
 
-	/* Allocate menu */
+	/* Allocate user interface */
 	name_len = ( name ? ( strlen ( name ) + 1 /* NUL */ ) : 0 );
 	title_len = ( strlen ( title ) + 1 /* NUL */ );
-	len = ( sizeof ( *menu ) + name_len + title_len );
-	menu = zalloc ( len );
-	if ( ! menu )
+	len = ( sizeof ( *dynui ) + name_len + title_len );
+	dynui = zalloc ( len );
+	if ( ! dynui )
 		return NULL;
-	name_copy = ( ( void * ) ( menu + 1 ) );
+	name_copy = ( ( void * ) ( dynui + 1 ) );
 	title_copy = ( name_copy + name_len );
 
-	/* Initialise menu */
+	/* Initialise user interface */
 	if ( name ) {
 		strcpy ( name_copy, name );
-		menu->name = name_copy;
+		dynui->name = name_copy;
 	}
 	strcpy ( title_copy, title );
-	menu->title = title_copy;
-	INIT_LIST_HEAD ( &menu->items );
+	dynui->title = title_copy;
+	INIT_LIST_HEAD ( &dynui->items );
 
-	/* Add to list of menus */
-	list_add_tail ( &menu->list, &menus );
+	/* Add to list of user interfaces */
+	list_add_tail ( &dynui->list, &dynamic_uis );
 
-	DBGC ( menu, "MENU %s created with title \"%s\"\n",
-	       menu->name, menu->title );
+	DBGC ( dynui, "DYNUI %s created with title \"%s\"\n",
+	       dynui->name, dynui->title );
 
-	return menu;
+	return dynui;
 }
 
 /**
- * Add menu item
+ * Add dynamic user interface item
  *
- * @v menu		Menu
+ * @v dynui		Dynamic user interface
  * @v name		Name, or NULL
  * @v text		Text, or NULL
  * @v shortcut		Shortcut key
  * @v is_default	Item is the default item
- * @ret item		Menu item, or NULL on failure
+ * @ret item		User interface item, or NULL on failure
  */
-struct menu_item * add_menu_item ( struct menu *menu, const char *name,
-				   const char *text, int shortcut,
-				   int is_default ) {
+struct dynamic_item * add_dynui_item ( struct dynamic_ui *dynui,
+				       const char *name, const char *text,
+				       int shortcut, int is_default ) {
+	struct dynamic_item *item;
 	size_t name_len;
 	size_t text_len;
 	size_t len;
-	struct menu_item *item;
 	char *name_copy;
 	char *text_copy;
 
@@ -135,46 +135,46 @@ struct menu_item * add_menu_item ( struct menu *menu, const char *name,
 	item->is_default = is_default;
 
 	/* Add to list of items */
-	list_add_tail ( &item->list, &menu->items );
+	list_add_tail ( &item->list, &dynui->items );
 
 	return item;
 }
 
 /**
- * Destroy menu
+ * Destroy dynamic user interface
  *
- * @v menu		Menu
+ * @v dynui		Dynamic user interface
  */
-void destroy_menu ( struct menu *menu ) {
-	struct menu_item *item;
-	struct menu_item *tmp;
+void destroy_dynui ( struct dynamic_ui *dynui ) {
+	struct dynamic_item *item;
+	struct dynamic_item *tmp;
 
-	/* Remove from list of menus */
-	list_del ( &menu->list );
+	/* Remove from list of user interfaces */
+	list_del ( &dynui->list );
 
 	/* Free items */
-	list_for_each_entry_safe ( item, tmp, &menu->items, list ) {
+	list_for_each_entry_safe ( item, tmp, &dynui->items, list ) {
 		list_del ( &item->list );
 		free ( item );
 	}
 
-	/* Free menu */
-	free ( menu );
+	/* Free user interface */
+	free ( dynui );
 }
 
 /**
- * Find menu
+ * Find dynamic user interface
  *
- * @v name		Menu name, or NULL
- * @ret menu		Menu, or NULL if not found
+ * @v name		User interface name, or NULL
+ * @ret dynui		Dynamic user interface, or NULL if not found
  */
-struct menu * find_menu ( const char *name ) {
-	struct menu *menu;
+struct dynamic_ui * find_dynui ( const char *name ) {
+	struct dynamic_ui *dynui;
 
-	list_for_each_entry ( menu, &menus, list ) {
-		if ( ( menu->name == name ) ||
-		     ( strcmp ( menu->name, name ) == 0 ) ) {
-			return menu;
+	list_for_each_entry ( dynui, &dynamic_uis, list ) {
+		if ( ( dynui->name == name ) ||
+		     ( strcmp ( dynui->name, name ) == 0 ) ) {
+			return dynui;
 		}
 	}
 
diff --git a/src/core/parseopt.c b/src/core/parseopt.c
index 8410e6e90..b657c3fce 100644
--- a/src/core/parseopt.c
+++ b/src/core/parseopt.c
@@ -33,7 +33,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <getopt.h>
 #include <ipxe/uuid.h>
 #include <ipxe/netdevice.h>
-#include <ipxe/menu.h>
+#include <ipxe/dynui.h>
 #include <ipxe/settings.h>
 #include <ipxe/params.h>
 #include <ipxe/timer.h>
@@ -194,21 +194,21 @@ int parse_netdev_configurator ( char *text,
 }
 
 /**
- * Parse menu name
+ * Parse dynamic user interface name
  *
  * @v text		Text
- * @ret menu		Menu
+ * @ret dynui		Dynamic user interface
  * @ret rc		Return status code
  */
-int parse_menu ( char *text, struct menu **menu ) {
+int parse_dynui ( char *text, struct dynamic_ui **dynui ) {
 
-	/* Find menu */
-	*menu = find_menu ( text );
-	if ( ! *menu ) {
+	/* Find user interface */
+	*dynui = find_dynui ( text );
+	if ( ! *dynui ) {
 		if ( text ) {
-			printf ( "\"%s\": no such menu\n", text );
+			printf ( "\"%s\": no such user interface\n", text );
 		} else {
-			printf ( "No default menu\n" );
+			printf ( "No default user interface\n" );
 		}
 		return -ENOENT;
 	}
diff --git a/src/hci/commands/menu_cmd.c b/src/hci/commands/dynui_cmd.c
similarity index 73%
rename from src/hci/commands/menu_cmd.c
rename to src/hci/commands/dynui_cmd.c
index 8b6d2a9f1..dbaa669ef 100644
--- a/src/hci/commands/menu_cmd.c
+++ b/src/hci/commands/dynui_cmd.c
@@ -25,7 +25,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
- * Menu commands
+ * Dynamic user interface commands
  *
  */
 
@@ -34,7 +34,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <string.h>
 #include <errno.h>
 #include <getopt.h>
-#include <ipxe/menu.h>
+#include <ipxe/dynui.h>
 #include <ipxe/command.h>
 #include <ipxe/parseopt.h>
 #include <ipxe/settings.h>
@@ -42,42 +42,42 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 FEATURE ( FEATURE_MISC, "Menu", DHCP_EB_FEATURE_MENU, 1 );
 
-/** "menu" options */
-struct menu_options {
+/** "dynui" options */
+struct dynui_options {
 	/** Name */
 	char *name;
 	/** Delete */
 	int delete;
 };
 
-/** "menu" option list */
-static struct option_descriptor menu_opts[] = {
+/** "dynui" option list */
+static struct option_descriptor dynui_opts[] = {
 	OPTION_DESC ( "name", 'n', required_argument,
-		      struct menu_options, name, parse_string ),
+		      struct dynui_options, name, parse_string ),
 	OPTION_DESC ( "delete", 'd', no_argument,
-		      struct menu_options, delete, parse_flag ),
+		      struct dynui_options, delete, parse_flag ),
 };
 
-/** "menu" command descriptor */
-static struct command_descriptor menu_cmd =
-	COMMAND_DESC ( struct menu_options, menu_opts, 0, MAX_ARGUMENTS,
+/** "dynui" command descriptor */
+static struct command_descriptor dynui_cmd =
+	COMMAND_DESC ( struct dynui_options, dynui_opts, 0, MAX_ARGUMENTS,
 		       "[<title>]" );
 
 /**
- * The "menu" command
+ * The "dynui" command
  *
  * @v argc		Argument count
  * @v argv		Argument list
  * @ret rc		Return status code
  */
-static int menu_exec ( int argc, char **argv ) {
-	struct menu_options opts;
-	struct menu *menu;
+static int dynui_exec ( int argc, char **argv ) {
+	struct dynui_options opts;
+	struct dynamic_ui *dynui;
 	char *title;
 	int rc;
 
 	/* Parse options */
-	if ( ( rc = parse_options ( argc, argv, &menu_cmd, &opts ) ) != 0 )
+	if ( ( rc = parse_options ( argc, argv, &dynui_cmd, &opts ) ) != 0 )
 		goto err_parse_options;
 
 	/* Parse title */
@@ -87,21 +87,21 @@ static int menu_exec ( int argc, char **argv ) {
 		goto err_parse_title;
 	}
 
-	/* Create menu */
-	menu = create_menu ( opts.name, title );
-	if ( ! menu ) {
+	/* Create dynamic user interface */
+	dynui = create_dynui ( opts.name, title );
+	if ( ! dynui ) {
 		rc = -ENOMEM;
-		goto err_create_menu;
+		goto err_create_dynui;
 	}
 
-	/* Destroy menu, if applicable */
+	/* Destroy dynamic user interface, if applicable */
 	if ( opts.delete )
-		destroy_menu ( menu );
+		destroy_dynui ( dynui );
 
 	/* Success */
 	rc = 0;
 
- err_create_menu:
+ err_create_dynui:
 	free ( title );
  err_parse_title:
  err_parse_options:
@@ -110,8 +110,8 @@ static int menu_exec ( int argc, char **argv ) {
 
 /** "item" options */
 struct item_options {
-	/** Menu name */
-	char *menu;
+	/** Dynamic user interface name */
+	char *dynui;
 	/** Shortcut key */
 	unsigned int key;
 	/** Use as default */
@@ -123,7 +123,7 @@ struct item_options {
 /** "item" option list */
 static struct option_descriptor item_opts[] = {
 	OPTION_DESC ( "menu", 'm', required_argument,
-		      struct item_options, menu, parse_string ),
+		      struct item_options, dynui, parse_string ),
 	OPTION_DESC ( "key", 'k', required_argument,
 		      struct item_options, key, parse_key ),
 	OPTION_DESC ( "default", 'd', no_argument,
@@ -146,8 +146,8 @@ static struct command_descriptor item_cmd =
  */
 static int item_exec ( int argc, char **argv ) {
 	struct item_options opts;
-	struct menu *menu;
-	struct menu_item *item;
+	struct dynamic_ui *dynui;
+	struct dynamic_item *item;
 	char *name = NULL;
 	char *text = NULL;
 	int rc;
@@ -169,23 +169,23 @@ static int item_exec ( int argc, char **argv ) {
 		}
 	}
 
-	/* Identify menu */
-	if ( ( rc = parse_menu ( opts.menu, &menu ) ) != 0 )
-		goto err_parse_menu;
+	/* Identify dynamic user interface */
+	if ( ( rc = parse_dynui ( opts.dynui, &dynui ) ) != 0 )
+		goto err_parse_dynui;
 
-	/* Add menu item */
-	item = add_menu_item ( menu, name, ( text ? text : "" ),
-			       opts.key, opts.is_default );
+	/* Add dynamic user interface item */
+	item = add_dynui_item ( dynui, name, ( text ? text : "" ),
+				opts.key, opts.is_default );
 	if ( ! item ) {
 		rc = -ENOMEM;
-		goto err_add_menu_item;
+		goto err_add_dynui_item;
 	}
 
 	/* Success */
 	rc = 0;
 
- err_add_menu_item:
- err_parse_menu:
+ err_add_dynui_item:
+ err_parse_dynui:
 	free ( text );
  err_parse_text:
  err_parse_options:
@@ -194,20 +194,20 @@ static int item_exec ( int argc, char **argv ) {
 
 /** "choose" options */
 struct choose_options {
-	/** Menu name */
-	char *menu;
+	/** Dynamic user interface name */
+	char *dynui;
 	/** Timeout */
 	unsigned long timeout;
 	/** Default selection */
 	char *select;
-	/** Keep menu */
+	/** Keep dynamic user interface */
 	int keep;
 };
 
 /** "choose" option list */
 static struct option_descriptor choose_opts[] = {
 	OPTION_DESC ( "menu", 'm', required_argument,
-		      struct choose_options, menu, parse_string ),
+		      struct choose_options, dynui, parse_string ),
 	OPTION_DESC ( "default", 'd', required_argument,
 		      struct choose_options, select, parse_string ),
 	OPTION_DESC ( "timeout", 't', required_argument,
@@ -230,8 +230,8 @@ static struct command_descriptor choose_cmd =
 static int choose_exec ( int argc, char **argv ) {
 	struct choose_options opts;
 	struct named_setting setting;
-	struct menu *menu;
-	struct menu_item *item;
+	struct dynamic_ui *dynui;
+	struct dynamic_item *item;
 	int rc;
 
 	/* Parse options */
@@ -243,12 +243,13 @@ static int choose_exec ( int argc, char **argv ) {
 						 &setting ) ) != 0 )
 		goto err_parse_setting;
 
-	/* Identify menu */
-	if ( ( rc = parse_menu ( opts.menu, &menu ) ) != 0 )
-		goto err_parse_menu;
+	/* Identify dynamic user interface */
+	if ( ( rc = parse_dynui ( opts.dynui, &dynui ) ) != 0 )
+		goto err_parse_dynui;
 
-	/* Show menu */
-	if ( ( rc = show_menu ( menu, opts.timeout, opts.select, &item ) ) != 0)
+	/* Show as menu */
+	if ( ( rc = show_menu ( dynui, opts.timeout, opts.select,
+				&item ) ) != 0 )
 		goto err_show_menu;
 
 	/* Apply default type if necessary */
@@ -268,20 +269,20 @@ static int choose_exec ( int argc, char **argv ) {
 
  err_store:
  err_show_menu:
-	/* Destroy menu, if applicable */
+	/* Destroy dynamic user interface, if applicable */
 	if ( ! opts.keep )
-		destroy_menu ( menu );
- err_parse_menu:
+		destroy_dynui ( dynui );
+ err_parse_dynui:
  err_parse_setting:
  err_parse_options:
 	return rc;
 }
 
-/** Menu commands */
-struct command menu_commands[] __command = {
+/** Dynamic user interface commands */
+struct command dynui_commands[] __command = {
 	{
 		.name = "menu",
-		.exec = menu_exec,
+		.exec = dynui_exec,
 	},
 	{
 		.name = "item",
diff --git a/src/hci/tui/menu_ui.c b/src/hci/tui/menu_ui.c
index 067e2d8ca..ab4e602a7 100644
--- a/src/hci/tui/menu_ui.c
+++ b/src/hci/tui/menu_ui.c
@@ -37,7 +37,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/console.h>
 #include <ipxe/ansicol.h>
 #include <ipxe/jumpscroll.h>
-#include <ipxe/menu.h>
+#include <ipxe/dynui.h>
 
 /* Screen layout */
 #define TITLE_ROW	1U
@@ -49,8 +49,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A menu user interface */
 struct menu_ui {
-	/** Menu */
-	struct menu *menu;
+	/** Dynamic user interface */
+	struct dynamic_ui *dynui;
 	/** Jump scroller */
 	struct jump_scroller scroll;
 	/** Timeout (0=indefinite) */
@@ -60,14 +60,15 @@ struct menu_ui {
 /**
  * Return a numbered menu item
  *
- * @v menu		Menu
+ * @v dynui		Dynamic user interface
  * @v index		Index
  * @ret item		Menu item, or NULL
  */
-static struct menu_item * menu_item ( struct menu *menu, unsigned int index ) {
-	struct menu_item *item;
+static struct dynamic_item * menu_item ( struct dynamic_ui *dynui,
+					 unsigned int index ) {
+	struct dynamic_item *item;
 
-	list_for_each_entry ( item, &menu->items, list ) {
+	list_for_each_entry ( item, &dynui->items, list ) {
 		if ( index-- == 0 )
 			return item;
 	}
@@ -82,7 +83,7 @@ static struct menu_item * menu_item ( struct menu *menu, unsigned int index ) {
  * @v index		Index
  */
 static void draw_menu_item ( struct menu_ui *ui, unsigned int index ) {
-	struct menu_item *item;
+	struct dynamic_item *item;
 	unsigned int row_offset;
 	char buf[ MENU_COLS + 1 /* NUL */ ];
 	char timeout_buf[6]; /* "(xxx)" + NUL */
@@ -95,7 +96,7 @@ static void draw_menu_item ( struct menu_ui *ui, unsigned int index ) {
 	move ( ( MENU_ROW + row_offset ), MENU_COL );
 
 	/* Get menu item */
-	item = menu_item ( ui->menu, index );
+	item = menu_item ( ui->dynui, index );
 	if ( item ) {
 
 		/* Draw separators in a different colour */
@@ -171,8 +172,8 @@ static void draw_menu_items ( struct menu_ui *ui ) {
  * @ret selected	Selected item
  * @ret rc		Return status code
  */
-static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
-	struct menu_item *item;
+static int menu_loop ( struct menu_ui *ui, struct dynamic_item **selected ) {
+	struct dynamic_item *item;
 	unsigned long timeout;
 	unsigned int previous;
 	unsigned int move;
@@ -217,7 +218,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 				break;
 			default:
 				i = 0;
-				list_for_each_entry ( item, &ui->menu->items,
+				list_for_each_entry ( item, &ui->dynui->items,
 						      list ) {
 					if ( ! ( item->shortcut &&
 						 ( item->shortcut == key ) ) ) {
@@ -238,7 +239,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 		/* Move selection, if applicable */
 		while ( move ) {
 			move = jump_scroll_move ( &ui->scroll, move );
-			item = menu_item ( ui->menu, ui->scroll.current );
+			item = menu_item ( ui->dynui, ui->scroll.current );
 			if ( item->name )
 				break;
 		}
@@ -252,7 +253,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 		}
 
 		/* Record selection */
-		item = menu_item ( ui->menu, ui->scroll.current );
+		item = menu_item ( ui->dynui, ui->scroll.current );
 		assert ( item != NULL );
 		assert ( item->name != NULL );
 		*selected = item;
@@ -265,14 +266,14 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 /**
  * Show menu
  *
- * @v menu		Menu
+ * @v dynui		Dynamic user interface
  * @v timeout		Timeout period, in ticks (0=indefinite)
  * @ret selected	Selected item
  * @ret rc		Return status code
  */
-int show_menu ( struct menu *menu, unsigned long timeout,
-		const char *select, struct menu_item **selected ) {
-	struct menu_item *item;
+int show_menu ( struct dynamic_ui *dynui, unsigned long timeout,
+		const char *select, struct dynamic_item **selected ) {
+	struct dynamic_item *item;
 	struct menu_ui ui;
 	char buf[ MENU_COLS + 1 /* NUL */ ];
 	int named_count = 0;
@@ -280,10 +281,10 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 
 	/* Initialise UI */
 	memset ( &ui, 0, sizeof ( ui ) );
-	ui.menu = menu;
+	ui.dynui = dynui;
 	ui.scroll.rows = MENU_ROWS;
 	ui.timeout = timeout;
-	list_for_each_entry ( item, &menu->items, list ) {
+	list_for_each_entry ( item, &dynui->items, list ) {
 		if ( item->name ) {
 			if ( ! named_count )
 				ui.scroll.current = ui.scroll.count;
@@ -315,7 +316,7 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 
 	/* Draw initial content */
 	attron ( A_BOLD );
-	snprintf ( buf, sizeof ( buf ), "%s", ui.menu->title );
+	snprintf ( buf, sizeof ( buf ), "%s", ui.dynui->title );
 	mvprintw ( TITLE_ROW, ( ( COLS - strlen ( buf ) ) / 2 ), "%s", buf );
 	attroff ( A_BOLD );
 	jump_scroll ( &ui.scroll );
diff --git a/src/include/ipxe/dynui.h b/src/include/ipxe/dynui.h
new file mode 100644
index 000000000..25124a3af
--- /dev/null
+++ b/src/include/ipxe/dynui.h
@@ -0,0 +1,50 @@
+#ifndef _IPXE_DYNUI_H
+#define _IPXE_DYNUI_H
+
+/** @file
+ *
+ * Dynamic user interfaces
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/list.h>
+
+/** A dynamic user interface */
+struct dynamic_ui {
+	/** List of dynamic user interfaces */
+	struct list_head list;
+	/** Name */
+	const char *name;
+	/** Title */
+	const char *title;
+	/** Dynamic user interface items */
+	struct list_head items;
+};
+
+/** A dynamic user interface item */
+struct dynamic_item {
+	/** List of dynamic user interface items */
+	struct list_head list;
+	/** Name */
+	const char *name;
+	/** Text */
+	const char *text;
+	/** Shortcut key */
+	int shortcut;
+	/** Is default item */
+	int is_default;
+};
+
+extern struct dynamic_ui * create_dynui ( const char *name, const char *title );
+extern struct dynamic_item * add_dynui_item ( struct dynamic_ui *dynui,
+					      const char *name,
+					      const char *text, int shortcut,
+					      int is_default );
+extern void destroy_dynui ( struct dynamic_ui *dynui );
+extern struct dynamic_ui * find_dynui ( const char *name );
+extern int show_menu ( struct dynamic_ui *dynui, unsigned long timeout,
+		       const char *select, struct dynamic_item **selected );
+
+#endif /* _IPXE_DYNUI_H */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index a9ce6569b..d75661ba7 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -364,7 +364,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_cms		      ( ERRFILE_OTHER | 0x002a0000 )
 #define ERRFILE_imgtrust	      ( ERRFILE_OTHER | 0x002b0000 )
 #define ERRFILE_menu_ui		      ( ERRFILE_OTHER | 0x002c0000 )
-#define ERRFILE_menu_cmd	      ( ERRFILE_OTHER | 0x002d0000 )
+#define ERRFILE_dynui_cmd	      ( ERRFILE_OTHER | 0x002d0000 )
 #define ERRFILE_validator	      ( ERRFILE_OTHER | 0x002e0000 )
 #define ERRFILE_ocsp		      ( ERRFILE_OTHER | 0x002f0000 )
 #define ERRFILE_nslookup	      ( ERRFILE_OTHER | 0x00300000 )
diff --git a/src/include/ipxe/menu.h b/src/include/ipxe/menu.h
deleted file mode 100644
index d11a92559..000000000
--- a/src/include/ipxe/menu.h
+++ /dev/null
@@ -1,49 +0,0 @@
-#ifndef _IPXE_MENU_H
-#define _IPXE_MENU_H
-
-/** @file
- *
- * Menu selection
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <ipxe/list.h>
-
-/** A menu */
-struct menu {
-	/** List of menus */
-	struct list_head list;
-	/** Name */
-	const char *name;
-	/** Title */
-	const char *title;
-	/** Menu items */
-	struct list_head items;
-};
-
-/** A menu item */
-struct menu_item {
-	/** List of menu items */
-	struct list_head list;
-	/** Name */
-	const char *name;
-	/** Text */
-	const char *text;
-	/** Shortcut key */
-	int shortcut;
-	/** Is default item */
-	int is_default;
-};
-
-extern struct menu * create_menu ( const char *name, const char *title );
-extern struct menu_item * add_menu_item ( struct menu *menu, const char *name,
-					  const char *text, int shortcut,
-					  int is_default );
-extern void destroy_menu ( struct menu *menu );
-extern struct menu * find_menu ( const char *name );
-extern int show_menu ( struct menu *menu, unsigned long timeout,
-		       const char *select, struct menu_item **selected );
-
-#endif /* _IPXE_MENU_H */
diff --git a/src/include/ipxe/parseopt.h b/src/include/ipxe/parseopt.h
index 61010f22a..5c449cd42 100644
--- a/src/include/ipxe/parseopt.h
+++ b/src/include/ipxe/parseopt.h
@@ -16,7 +16,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct net_device;
 struct net_device_configurator;
-struct menu;
+struct dynamic_ui;
 struct parameters;
 
 /** A command-line option descriptor */
@@ -142,7 +142,7 @@ extern int parse_netdev ( char *text, struct net_device **netdev );
 extern int
 parse_netdev_configurator ( char *text,
 			    struct net_device_configurator **configurator );
-extern int parse_menu ( char *text, struct menu **menu );
+extern int parse_dynui ( char *text, struct dynamic_ui **dynui );
 extern int parse_flag ( char *text __unused, int *flag );
 extern int parse_key ( char *text, unsigned int *key );
 extern int parse_settings ( char *text, struct settings **settings );

From c8e50bb0fd3d9d1629e5c78f92bbf7bc9af84f2c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 20 Jun 2024 14:16:18 -0700
Subject: [PATCH 145/237] [dynui] Generalise mechanisms for looking up user
 interface items

Generalise the ability to look up a dynamic user interface item by
index or by shortcut key, to allow for reuse of this code for
interactive forms.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/dynui.c         | 38 ++++++++++++++++++++++++++++++++++++++
 src/hci/tui/menu_ui.c    | 38 ++++++--------------------------------
 src/include/ipxe/dynui.h |  8 ++++++++
 3 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/src/core/dynui.c b/src/core/dynui.c
index 6e053b905..33218f598 100644
--- a/src/core/dynui.c
+++ b/src/core/dynui.c
@@ -131,6 +131,7 @@ struct dynamic_item * add_dynui_item ( struct dynamic_ui *dynui,
 	}
 	strcpy ( text_copy, text );
 	item->text = text_copy;
+	item->index = dynui->count++;
 	item->shortcut = shortcut;
 	item->is_default = is_default;
 
@@ -180,3 +181,40 @@ struct dynamic_ui * find_dynui ( const char *name ) {
 
 	return NULL;
 }
+
+/**
+ * Find dynamic user interface item by index
+ *
+ * @v dynui		Dynamic user interface
+ * @v index		Index
+ * @ret item		User interface item, or NULL if not found
+ */
+struct dynamic_item * dynui_item ( struct dynamic_ui *dynui,
+				   unsigned int index ) {
+	struct dynamic_item *item;
+
+	list_for_each_entry ( item, &dynui->items, list ) {
+		if ( index-- == 0 )
+			return item;
+	}
+
+	return NULL;
+}
+
+/**
+ * Find dynamic user interface item by shortcut key
+ *
+ * @v dynui		Dynamic user interface
+ * @v key		Shortcut key
+ * @ret item		User interface item, or NULL if not found
+ */
+struct dynamic_item * dynui_shortcut ( struct dynamic_ui *dynui, int key ) {
+	struct dynamic_item *item;
+
+	list_for_each_entry ( item, &dynui->items, list ) {
+		if ( key && ( key == item->shortcut ) )
+			return item;
+	}
+
+	return NULL;
+}
diff --git a/src/hci/tui/menu_ui.c b/src/hci/tui/menu_ui.c
index ab4e602a7..00cdab8df 100644
--- a/src/hci/tui/menu_ui.c
+++ b/src/hci/tui/menu_ui.c
@@ -57,25 +57,6 @@ struct menu_ui {
 	unsigned long timeout;
 };
 
-/**
- * Return a numbered menu item
- *
- * @v dynui		Dynamic user interface
- * @v index		Index
- * @ret item		Menu item, or NULL
- */
-static struct dynamic_item * menu_item ( struct dynamic_ui *dynui,
-					 unsigned int index ) {
-	struct dynamic_item *item;
-
-	list_for_each_entry ( item, &dynui->items, list ) {
-		if ( index-- == 0 )
-			return item;
-	}
-
-	return NULL;
-}
-
 /**
  * Draw a numbered menu item
  *
@@ -96,7 +77,7 @@ static void draw_menu_item ( struct menu_ui *ui, unsigned int index ) {
 	move ( ( MENU_ROW + row_offset ), MENU_COL );
 
 	/* Get menu item */
-	item = menu_item ( ui->dynui, index );
+	item = dynui_item ( ui->dynui, index );
 	if ( item ) {
 
 		/* Draw separators in a different colour */
@@ -178,7 +159,6 @@ static int menu_loop ( struct menu_ui *ui, struct dynamic_item **selected ) {
 	unsigned int previous;
 	unsigned int move;
 	int key;
-	int i;
 	int chosen = 0;
 	int rc = 0;
 
@@ -217,15 +197,9 @@ static int menu_loop ( struct menu_ui *ui, struct dynamic_item **selected ) {
 				chosen = 1;
 				break;
 			default:
-				i = 0;
-				list_for_each_entry ( item, &ui->dynui->items,
-						      list ) {
-					if ( ! ( item->shortcut &&
-						 ( item->shortcut == key ) ) ) {
-						i++;
-						continue;
-					}
-					ui->scroll.current = i;
+				item = dynui_shortcut ( ui->dynui, key );
+				if ( item ) {
+					ui->scroll.current = item->index;
 					if ( item->name ) {
 						chosen = 1;
 					} else {
@@ -239,7 +213,7 @@ static int menu_loop ( struct menu_ui *ui, struct dynamic_item **selected ) {
 		/* Move selection, if applicable */
 		while ( move ) {
 			move = jump_scroll_move ( &ui->scroll, move );
-			item = menu_item ( ui->dynui, ui->scroll.current );
+			item = dynui_item ( ui->dynui, ui->scroll.current );
 			if ( item->name )
 				break;
 		}
@@ -253,7 +227,7 @@ static int menu_loop ( struct menu_ui *ui, struct dynamic_item **selected ) {
 		}
 
 		/* Record selection */
-		item = menu_item ( ui->dynui, ui->scroll.current );
+		item = dynui_item ( ui->dynui, ui->scroll.current );
 		assert ( item != NULL );
 		assert ( item->name != NULL );
 		*selected = item;
diff --git a/src/include/ipxe/dynui.h b/src/include/ipxe/dynui.h
index 25124a3af..f38d44825 100644
--- a/src/include/ipxe/dynui.h
+++ b/src/include/ipxe/dynui.h
@@ -21,6 +21,8 @@ struct dynamic_ui {
 	const char *title;
 	/** Dynamic user interface items */
 	struct list_head items;
+	/** Number of user interface items */
+	unsigned int count;
 };
 
 /** A dynamic user interface item */
@@ -31,6 +33,8 @@ struct dynamic_item {
 	const char *name;
 	/** Text */
 	const char *text;
+	/** Index */
+	unsigned int index;
 	/** Shortcut key */
 	int shortcut;
 	/** Is default item */
@@ -44,6 +48,10 @@ extern struct dynamic_item * add_dynui_item ( struct dynamic_ui *dynui,
 					      int is_default );
 extern void destroy_dynui ( struct dynamic_ui *dynui );
 extern struct dynamic_ui * find_dynui ( const char *name );
+extern struct dynamic_item * dynui_item ( struct dynamic_ui *dynui,
+					  unsigned int index );
+extern struct dynamic_item * dynui_shortcut ( struct dynamic_ui *dynui,
+					      int key );
 extern int show_menu ( struct dynamic_ui *dynui, unsigned long timeout,
 		       const char *select, struct dynamic_item **selected );
 

From 039019039e89aedf4c5a0d81c351638a1e036335 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 20 Jun 2024 16:20:05 -0700
Subject: [PATCH 146/237] [dynui] Allow for multiple flags on a user interface
 item

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/dynui.c             |  6 +++---
 src/hci/commands/dynui_cmd.c |  7 +++++--
 src/hci/tui/menu_ui.c        |  2 +-
 src/include/ipxe/dynui.h     | 12 ++++++++----
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/src/core/dynui.c b/src/core/dynui.c
index 33218f598..3d139c02a 100644
--- a/src/core/dynui.c
+++ b/src/core/dynui.c
@@ -96,13 +96,13 @@ struct dynamic_ui * create_dynui ( const char *name, const char *title ) {
  * @v dynui		Dynamic user interface
  * @v name		Name, or NULL
  * @v text		Text, or NULL
+ * @v flags		Flags
  * @v shortcut		Shortcut key
- * @v is_default	Item is the default item
  * @ret item		User interface item, or NULL on failure
  */
 struct dynamic_item * add_dynui_item ( struct dynamic_ui *dynui,
 				       const char *name, const char *text,
-				       int shortcut, int is_default ) {
+				       unsigned int flags, int shortcut ) {
 	struct dynamic_item *item;
 	size_t name_len;
 	size_t text_len;
@@ -132,8 +132,8 @@ struct dynamic_item * add_dynui_item ( struct dynamic_ui *dynui,
 	strcpy ( text_copy, text );
 	item->text = text_copy;
 	item->index = dynui->count++;
+	item->flags = flags;
 	item->shortcut = shortcut;
-	item->is_default = is_default;
 
 	/* Add to list of items */
 	list_add_tail ( &item->list, &dynui->items );
diff --git a/src/hci/commands/dynui_cmd.c b/src/hci/commands/dynui_cmd.c
index dbaa669ef..557c5db5d 100644
--- a/src/hci/commands/dynui_cmd.c
+++ b/src/hci/commands/dynui_cmd.c
@@ -148,6 +148,7 @@ static int item_exec ( int argc, char **argv ) {
 	struct item_options opts;
 	struct dynamic_ui *dynui;
 	struct dynamic_item *item;
+	unsigned int flags = 0;
 	char *name = NULL;
 	char *text = NULL;
 	int rc;
@@ -174,8 +175,10 @@ static int item_exec ( int argc, char **argv ) {
 		goto err_parse_dynui;
 
 	/* Add dynamic user interface item */
-	item = add_dynui_item ( dynui, name, ( text ? text : "" ),
-				opts.key, opts.is_default );
+	if ( opts.is_default )
+		flags |= DYNUI_DEFAULT;
+	item = add_dynui_item ( dynui, name, ( text ? text : "" ), flags,
+				opts.key );
 	if ( ! item ) {
 		rc = -ENOMEM;
 		goto err_add_dynui_item;
diff --git a/src/hci/tui/menu_ui.c b/src/hci/tui/menu_ui.c
index 00cdab8df..b7b52ee62 100644
--- a/src/hci/tui/menu_ui.c
+++ b/src/hci/tui/menu_ui.c
@@ -267,7 +267,7 @@ int show_menu ( struct dynamic_ui *dynui, unsigned long timeout,
 				if ( strcmp ( select, item->name ) == 0 )
 					ui.scroll.current = ui.scroll.count;
 			} else {
-				if ( item->is_default )
+				if ( item->flags & DYNUI_DEFAULT )
 					ui.scroll.current = ui.scroll.count;
 			}
 		}
diff --git a/src/include/ipxe/dynui.h b/src/include/ipxe/dynui.h
index f38d44825..5ba007032 100644
--- a/src/include/ipxe/dynui.h
+++ b/src/include/ipxe/dynui.h
@@ -35,17 +35,21 @@ struct dynamic_item {
 	const char *text;
 	/** Index */
 	unsigned int index;
+	/** Flags */
+	unsigned int flags;
 	/** Shortcut key */
 	int shortcut;
-	/** Is default item */
-	int is_default;
 };
 
+/** Dynamic user interface item is default selection */
+#define DYNUI_DEFAULT 0x0001
+
 extern struct dynamic_ui * create_dynui ( const char *name, const char *title );
 extern struct dynamic_item * add_dynui_item ( struct dynamic_ui *dynui,
 					      const char *name,
-					      const char *text, int shortcut,
-					      int is_default );
+					      const char *text,
+					      unsigned int flags,
+					      int shortcut );
 extern void destroy_dynui ( struct dynamic_ui *dynui );
 extern struct dynamic_ui * find_dynui ( const char *name );
 extern struct dynamic_item * dynui_item ( struct dynamic_ui *dynui,

From 1c3c5e2b22ca31bbf77c39aef51671d0b6e95767 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 20 Jun 2024 16:21:28 -0700
Subject: [PATCH 147/237] [dynui] Add concept of a secret user interface item

For interactive forms, the concept of a secret value becomes
meaningful (e.g. for password fields).

Add a flag to indicate that an item represents a secret value, and
allow this flag to be set via the "--secret" option of the "item"
command.

This flag has no meaning for menu items, but is silently accepted
anyway to keep the code size minimal.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/commands/dynui_cmd.c | 6 ++++++
 src/include/ipxe/dynui.h     | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/src/hci/commands/dynui_cmd.c b/src/hci/commands/dynui_cmd.c
index 557c5db5d..5aed3d034 100644
--- a/src/hci/commands/dynui_cmd.c
+++ b/src/hci/commands/dynui_cmd.c
@@ -116,6 +116,8 @@ struct item_options {
 	unsigned int key;
 	/** Use as default */
 	int is_default;
+	/** Value is a secret */
+	int is_secret;
 	/** Use as a separator */
 	int is_gap;
 };
@@ -128,6 +130,8 @@ static struct option_descriptor item_opts[] = {
 		      struct item_options, key, parse_key ),
 	OPTION_DESC ( "default", 'd', no_argument,
 		      struct item_options, is_default, parse_flag ),
+	OPTION_DESC ( "secret", 's', no_argument,
+		      struct item_options, is_secret, parse_flag ),
 	OPTION_DESC ( "gap", 'g', no_argument,
 		      struct item_options, is_gap, parse_flag ),
 };
@@ -177,6 +181,8 @@ static int item_exec ( int argc, char **argv ) {
 	/* Add dynamic user interface item */
 	if ( opts.is_default )
 		flags |= DYNUI_DEFAULT;
+	if ( opts.is_secret )
+		flags |= DYNUI_SECRET;
 	item = add_dynui_item ( dynui, name, ( text ? text : "" ), flags,
 				opts.key );
 	if ( ! item ) {
diff --git a/src/include/ipxe/dynui.h b/src/include/ipxe/dynui.h
index 5ba007032..5029e5850 100644
--- a/src/include/ipxe/dynui.h
+++ b/src/include/ipxe/dynui.h
@@ -44,6 +44,9 @@ struct dynamic_item {
 /** Dynamic user interface item is default selection */
 #define DYNUI_DEFAULT 0x0001
 
+/** Dynamic user interface item represents a secret */
+#define DYNUI_SECRET 0x0002
+
 extern struct dynamic_ui * create_dynui ( const char *name, const char *title );
 extern struct dynamic_item * add_dynui_item ( struct dynamic_ui *dynui,
 					      const char *name,

From f417f0b6a56956137d75c77f344d798f6b30a27c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 31 May 2024 10:10:53 +0100
Subject: [PATCH 148/237] [form] Add support for dynamically created
 interactive forms

Add support for presenting a dynamic user interface as an interactive
form, alongside the existing support for presenting a dynamic user
interface as a menu.

An interactive form may be used to allow a user to input (or edit)
values for multiple settings on a single screen, as a user-friendly
alternative to prompting for setting values via the "read" command.

In the present implementation, all input fields must fit on a single
screen (with no scrolling), and the only supported widget type is an
editable text box.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config.c          |   3 +
 src/config/general.h         |   1 +
 src/hci/commands/dynui_cmd.c |  66 +++++
 src/hci/tui/form_ui.c        | 544 +++++++++++++++++++++++++++++++++++
 src/include/ipxe/dynui.h     |   1 +
 src/include/ipxe/errfile.h   |   1 +
 6 files changed, 616 insertions(+)
 create mode 100644 src/hci/tui/form_ui.c

diff --git a/src/config/config.c b/src/config/config.c
index 666712304..6bcd3c1ca 100644
--- a/src/config/config.c
+++ b/src/config/config.c
@@ -227,6 +227,9 @@ REQUIRE_OBJECT ( sanboot_cmd );
 #ifdef MENU_CMD
 REQUIRE_OBJECT ( dynui_cmd );
 #endif
+#ifdef FORM_CMD
+REQUIRE_OBJECT ( dynui_cmd );
+#endif
 #ifdef LOGIN_CMD
 REQUIRE_OBJECT ( login_cmd );
 #endif
diff --git a/src/config/general.h b/src/config/general.h
index e883e072f..f936e874c 100644
--- a/src/config/general.h
+++ b/src/config/general.h
@@ -145,6 +145,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define DHCP_CMD		/* DHCP management commands */
 #define SANBOOT_CMD		/* SAN boot commands */
 #define MENU_CMD		/* Menu commands */
+#define FORM_CMD		/* Form commands */
 #define LOGIN_CMD		/* Login command */
 #define SYNC_CMD		/* Sync command */
 #define SHELL_CMD		/* Shell command */
diff --git a/src/hci/commands/dynui_cmd.c b/src/hci/commands/dynui_cmd.c
index 5aed3d034..d4446dc7c 100644
--- a/src/hci/commands/dynui_cmd.c
+++ b/src/hci/commands/dynui_cmd.c
@@ -126,6 +126,8 @@ struct item_options {
 static struct option_descriptor item_opts[] = {
 	OPTION_DESC ( "menu", 'm', required_argument,
 		      struct item_options, dynui, parse_string ),
+	OPTION_DESC ( "form", 'f', required_argument,
+		      struct item_options, dynui, parse_string ),
 	OPTION_DESC ( "key", 'k', required_argument,
 		      struct item_options, key, parse_key ),
 	OPTION_DESC ( "default", 'd', no_argument,
@@ -287,12 +289,72 @@ static int choose_exec ( int argc, char **argv ) {
 	return rc;
 }
 
+/** "present" options */
+struct present_options {
+	/** Dynamic user interface name */
+	char *dynui;
+	/** Keep dynamic user interface */
+	int keep;
+};
+
+/** "present" option list */
+static struct option_descriptor present_opts[] = {
+	OPTION_DESC ( "form", 'f', required_argument,
+		      struct present_options, dynui, parse_string ),
+	OPTION_DESC ( "keep", 'k', no_argument,
+		      struct present_options, keep, parse_flag ),
+};
+
+/** "present" command descriptor */
+static struct command_descriptor present_cmd =
+	COMMAND_DESC ( struct present_options, present_opts, 0, 0, NULL );
+
+/**
+ * The "present" command
+ *
+ * @v argc		Argument count
+ * @v argv		Argument list
+ * @ret rc		Return status code
+ */
+static int present_exec ( int argc, char **argv ) {
+	struct present_options opts;
+	struct dynamic_ui *dynui;
+	int rc;
+
+	/* Parse options */
+	if ( ( rc = parse_options ( argc, argv, &present_cmd, &opts ) ) != 0 )
+		goto err_parse_options;
+
+	/* Identify dynamic user interface */
+	if ( ( rc = parse_dynui ( opts.dynui, &dynui ) ) != 0 )
+		goto err_parse_dynui;
+
+	/* Show as form */
+	if ( ( rc = show_form ( dynui ) ) != 0 )
+		goto err_show_form;
+
+	/* Success */
+	rc = 0;
+
+ err_show_form:
+	/* Destroy dynamic user interface, if applicable */
+	if ( ! opts.keep )
+		destroy_dynui ( dynui );
+ err_parse_dynui:
+ err_parse_options:
+	return rc;
+}
+
 /** Dynamic user interface commands */
 struct command dynui_commands[] __command = {
 	{
 		.name = "menu",
 		.exec = dynui_exec,
 	},
+	{
+		.name = "form",
+		.exec = dynui_exec,
+	},
 	{
 		.name = "item",
 		.exec = item_exec,
@@ -301,4 +363,8 @@ struct command dynui_commands[] __command = {
 		.name = "choose",
 		.exec = choose_exec,
 	},
+	{
+		.name = "present",
+		.exec = present_exec,
+	},
 };
diff --git a/src/hci/tui/form_ui.c b/src/hci/tui/form_ui.c
new file mode 100644
index 000000000..6cc28c369
--- /dev/null
+++ b/src/hci/tui/form_ui.c
@@ -0,0 +1,544 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Text widget forms
+ *
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <ipxe/ansicol.h>
+#include <ipxe/dynui.h>
+#include <ipxe/jumpscroll.h>
+#include <ipxe/settings.h>
+#include <ipxe/editbox.h>
+#include <ipxe/message.h>
+
+/** Form title row */
+#define TITLE_ROW 1U
+
+/** Starting control row */
+#define START_ROW 3U
+
+/** Ending control row */
+#define END_ROW ( LINES - 3U )
+
+/** Instructions row */
+#define INSTRUCTION_ROW ( LINES - 2U )
+
+/** Padding between instructions */
+#define INSTRUCTION_PAD "     "
+
+/** Input field width */
+#define INPUT_WIDTH ( COLS / 2U )
+
+/** Input field column */
+#define INPUT_COL ( ( COLS - INPUT_WIDTH ) / 2U )
+
+/** A form */
+struct form {
+	/** Dynamic user interface */
+	struct dynamic_ui *dynui;
+	/** Jump scroller */
+	struct jump_scroller scroll;
+	/** Array of form controls */
+	struct form_control *controls;
+};
+
+/** A form control */
+struct form_control {
+	/** Dynamic user interface item */
+	struct dynamic_item *item;
+	/** Settings block */
+	struct settings *settings;
+	/** Setting */
+	struct setting setting;
+	/** Label row */
+	unsigned int row;
+	/** Editable text box */
+	struct edit_box editbox;
+	/** Modifiable setting name */
+	char *name;
+	/** Modifiable setting value */
+	char *value;
+	/** Most recent error in saving */
+	int rc;
+};
+
+/**
+ * Allocate form
+ *
+ * @v dynui		Dynamic user interface
+ * @ret form		Form, or NULL on error
+ */
+static struct form * alloc_form ( struct dynamic_ui *dynui ) {
+	struct form *form;
+	struct form_control *control;
+	struct dynamic_item *item;
+	char *name;
+	size_t len;
+
+	/* Calculate total length */
+	len = sizeof ( *form );
+	list_for_each_entry ( item, &dynui->items, list ) {
+		len += sizeof ( *control );
+		if ( item->name )
+			len += ( strlen ( item->name ) + 1 /* NUL */ );
+	}
+
+	/* Allocate and initialise structure */
+	form = zalloc ( len );
+	if ( ! form )
+		return NULL;
+	control = ( ( ( void * ) form ) + sizeof ( *form ) );
+	name = ( ( ( void * ) control ) +
+		 ( dynui->count * sizeof ( *control ) ) );
+	form->dynui = dynui;
+	form->controls = control;
+	list_for_each_entry ( item, &dynui->items, list ) {
+		control->item = item;
+		if ( item->name ) {
+			control->name = name;
+			name = ( stpcpy ( name, item->name ) + 1 /* NUL */ );
+		}
+		control++;
+	}
+	assert ( ( ( void * ) name ) == ( ( ( void * ) form ) + len ) );
+
+	return form;
+}
+
+/**
+ * Free form
+ *
+ * @v form		Form
+ */
+static void free_form ( struct form *form ) {
+	unsigned int i;
+
+	/* Free input value buffers */
+	for ( i = 0 ; i < form->dynui->count ; i++ )
+		free ( form->controls[i].value );
+
+	/* Free form */
+	free ( form );
+}
+
+/**
+ * Assign form rows
+ *
+ * @v form		Form
+ * @ret rc		Return status code
+ */
+static int layout_form ( struct form *form ) {
+	struct form_control *control;
+	struct dynamic_item *item;
+	unsigned int labels = 0;
+	unsigned int inputs = 0;
+	unsigned int pad_control = 0;
+	unsigned int pad_label = 0;
+	unsigned int minimum;
+	unsigned int remaining;
+	unsigned int between;
+	unsigned int row;
+	unsigned int flags;
+	unsigned int i;
+
+	/* Count labels and inputs */
+	for ( i = 0 ; i < form->dynui->count ; i++ ) {
+		control = &form->controls[i];
+		item = control->item;
+		if ( item->text[0] )
+			labels++;
+		if ( item->name ) {
+			if ( ! inputs )
+				form->scroll.current = i;
+			inputs++;
+			if ( item->flags & DYNUI_DEFAULT )
+				form->scroll.current = i;
+			form->scroll.count = ( i + 1 );
+		}
+	}
+	form->scroll.rows = form->scroll.count;
+	DBGC ( form, "FORM %p has %d controls (%d labels, %d inputs)\n",
+	       form, form->dynui->count, labels, inputs );
+
+	/* Refuse to create forms with no inputs */
+	if ( ! inputs )
+		return -EINVAL;
+
+	/* Calculate minimum number of rows */
+	minimum = ( labels + ( inputs * 2 /* edit box and error message */ ) );
+	remaining = ( END_ROW - START_ROW );
+	DBGC ( form, "FORM %p has %d (of %d) usable rows\n",
+	       form, remaining, LINES );
+	if ( minimum > remaining )
+		return -ERANGE;
+	remaining -= minimum;
+
+	/* Insert blank row between controls, if space exists */
+	between = ( form->dynui->count - 1 );
+	if ( between <= remaining ) {
+		pad_control = 1;
+		remaining -= between;
+		DBGC ( form, "FORM %p padding between controls\n", form );
+	}
+
+	/* Insert blank row after label, if space exists */
+	if ( labels <= remaining ) {
+		pad_label = 1;
+		remaining -= labels;
+		DBGC ( form, "FORM %p padding after labels\n", form );
+	}
+
+	/* Centre on screen */
+	DBGC ( form, "FORM %p has %d spare rows\n", form, remaining );
+	row = ( START_ROW + ( remaining / 2 ) );
+
+	/* Position each control */
+	for ( i = 0 ; i < form->dynui->count ; i++ ) {
+		control = &form->controls[i];
+		item = control->item;
+		if ( item->text[0] ) {
+			control->row = row;
+			row++; /* Label text */
+			row += pad_label;
+		}
+		if ( item->name ) {
+			flags = ( ( item->flags & DYNUI_SECRET ) ?
+				  WIDGET_SECRET : 0 );
+			init_editbox ( &control->editbox, row, INPUT_COL,
+				       INPUT_WIDTH, flags, &control->value );
+			row++; /* Edit box */
+			row++; /* Error message (if any) */
+		}
+		row += pad_control;
+	}
+	assert ( row <= END_ROW );
+
+	return 0;
+}
+
+/**
+ * Draw form
+ *
+ * @v form		Form
+ */
+static void draw_form ( struct form *form ) {
+	struct form_control *control;
+	unsigned int i;
+
+	/* Clear screen */
+	color_set ( CPAIR_NORMAL, NULL );
+	erase();
+
+	/* Draw title, if any */
+	attron ( A_BOLD );
+	if ( form->dynui->title )
+		msg ( TITLE_ROW, "%s", form->dynui->title );
+	attroff ( A_BOLD );
+
+	/* Draw controls */
+	for ( i = 0 ; i < form->dynui->count ; i++ ) {
+		control = &form->controls[i];
+
+		/* Draw label, if any */
+		if ( control->row )
+			msg ( control->row, "%s", control->item->text );
+
+		/* Draw input, if any */
+		if ( control->name )
+			draw_widget ( &control->editbox.widget );
+	}
+
+	/* Draw instructions */
+	msg ( INSTRUCTION_ROW, "%s", "Ctrl-X - save changes"
+	      INSTRUCTION_PAD "Ctrl-C - discard changes" );
+}
+
+/**
+ * Draw (or clear) error messages
+ *
+ * @v form		Form
+ */
+static void draw_errors ( struct form *form ) {
+	struct form_control *control;
+	unsigned int row;
+	unsigned int i;
+
+	/* Draw (or clear) errors */
+	for ( i = 0 ; i < form->dynui->count ; i++ ) {
+		control = &form->controls[i];
+
+		/* Skip non-input controls */
+		if ( ! control->name )
+			continue;
+
+		/* Draw or clear error message as appropriate */
+		row = ( control->editbox.widget.row + 1 );
+		if ( control->rc != 0 ) {
+			color_set ( CPAIR_ALERT, NULL );
+			msg ( row, " %s ", strerror ( control->rc ) );
+			color_set ( CPAIR_NORMAL, NULL );
+		} else {
+			clearmsg ( row );
+		}
+	}
+}
+
+/**
+ * Parse setting names
+ *
+ * @v form		Form
+ * @ret rc		Return status code
+ */
+static int parse_names ( struct form *form ) {
+	struct form_control *control;
+	unsigned int i;
+	int rc;
+
+	/* Parse all setting names */
+	for ( i = 0 ; i < form->dynui->count ; i++ ) {
+		control = &form->controls[i];
+
+		/* Skip labels */
+		if ( ! control->name ) {
+			DBGC ( form, "FORM %p item %d is a label\n", form, i );
+			continue;
+		}
+
+		/* Parse setting name */
+		DBGC ( form, "FORM %p item %d is for %s\n",
+		       form, i, control->name );
+		if ( ( rc = parse_setting_name ( control->name,
+						 autovivify_child_settings,
+						 &control->settings,
+						 &control->setting ) ) != 0 )
+			return rc;
+
+		/* Apply default type if necessary */
+		if ( ! control->setting.type )
+			control->setting.type = &setting_type_string;
+	}
+
+	return 0;
+}
+
+/**
+ * Load current input values
+ *
+ * @v form		Form
+ */
+static void load_values ( struct form *form ) {
+	struct form_control *control;
+	unsigned int i;
+
+	/* Fetch all current setting values */
+	for ( i = 0 ; i < form->dynui->count ; i++ ) {
+		control = &form->controls[i];
+		if ( ! control->name )
+			continue;
+		fetchf_setting_copy ( control->settings, &control->setting,
+				      NULL, &control->setting,
+				      &control->value );
+	}
+}
+
+/**
+ * Store current input values
+ *
+ * @v form		Form
+ * @ret rc		Return status code
+ */
+static int save_values ( struct form *form ) {
+	struct form_control *control;
+	unsigned int i;
+	int rc = 0;
+
+	/* Store all current setting values */
+	for ( i = 0 ; i < form->dynui->count ; i++ ) {
+		control = &form->controls[i];
+		if ( ! control->name )
+			continue;
+		control->rc = storef_setting ( control->settings,
+					       &control->setting,
+					       control->value );
+		if ( control->rc != 0 )
+			rc = control->rc;
+	}
+
+	return rc;
+}
+
+/**
+ * Submit form
+ *
+ * @v form		Form
+ * @ret rc		Return status code
+ */
+static int submit_form ( struct form *form ) {
+	int rc;
+
+	/* Attempt to save values */
+	rc = save_values ( form );
+
+	/* Draw (or clear) errors */
+	draw_errors ( form );
+
+	return rc;
+}
+
+/**
+ * Form main loop
+ *
+ * @v form		Form
+ * @ret rc		Return status code
+ */
+static int form_loop ( struct form *form ) {
+	struct jump_scroller *scroll = &form->scroll;
+	struct form_control *control;
+	struct dynamic_item *item;
+	unsigned int move;
+	unsigned int i;
+	int key;
+	int rc;
+
+	/* Main loop */
+	while ( 1 ) {
+
+		/* Draw current input */
+		control = &form->controls[scroll->current];
+		draw_widget ( &control->editbox.widget );
+
+		/* Process keypress */
+		key = edit_widget ( &control->editbox.widget, getkey ( 0 ) );
+
+		/* Handle scroll keys */
+		move = jump_scroll_key ( &form->scroll, key );
+
+		/* Handle special keys */
+		switch ( key ) {
+		case CTRL_C:
+		case ESC:
+			/* Cancel form */
+			return -ECANCELED;
+		case KEY_ENTER:
+			/* Attempt to do the most intuitive thing when
+			 * Enter is pressed.  If we are on the last
+			 * input, then submit the form.  If we are
+			 * editing an input which failed, then
+			 * resubmit the form.  Otherwise, move to the
+			 * next input.
+			 */
+			if ( ( control->rc == 0 ) &&
+			     ( scroll->current < ( scroll->count - 1 ) ) ) {
+				move = SCROLL_DOWN;
+				break;
+			}
+			/* fall through */
+		case CTRL_X:
+			/* Submit form */
+			if ( ( rc = submit_form ( form ) ) == 0 )
+				return 0;
+			/* If current input is not the problem, move
+			 * to the first input that needs fixing.
+			 */
+			if ( control->rc == 0 ) {
+				for ( i = 0 ; i < form->dynui->count ; i++ ) {
+					if ( form->controls[i].rc != 0 ) {
+						scroll->current = i;
+						break;
+					}
+				}
+			}
+			break;
+		default:
+			/* Move to input with matching shortcut key, if any */
+			item = dynui_shortcut ( form->dynui, key );
+			if ( item ) {
+				scroll->current = item->index;
+				if ( ! item->name )
+					move = SCROLL_DOWN;
+			}
+			break;
+		}
+
+		/* Move selection, if applicable */
+		while ( move ) {
+			move = jump_scroll_move ( &form->scroll, move );
+			control = &form->controls[scroll->current];
+			if ( control->name )
+				break;
+		}
+	}
+}
+
+/**
+ * Show form
+ *
+ * @v dynui		Dynamic user interface
+ * @ret rc		Return status code
+ */
+int show_form ( struct dynamic_ui *dynui ) {
+	struct form *form;
+	int rc;
+
+	/* Allocate and initialise structure */
+	form = alloc_form ( dynui );
+	if ( ! form ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Parse setting names and load current values */
+	if ( ( rc = parse_names ( form ) ) != 0 )
+		goto err_parse_names;
+	load_values ( form );
+
+	/* Lay out form on screen */
+	if ( ( rc = layout_form ( form ) ) != 0 )
+		goto err_layout;
+
+	/* Draw initial form */
+	initscr();
+	start_color();
+	draw_form ( form );
+
+	/* Run main loop */
+	if ( ( rc = form_loop ( form ) ) != 0 )
+		goto err_loop;
+
+ err_loop:
+	color_set ( CPAIR_NORMAL, NULL );
+	endwin();
+ err_layout:
+ err_parse_names:
+	free_form ( form );
+ err_alloc:
+	return rc;
+}
diff --git a/src/include/ipxe/dynui.h b/src/include/ipxe/dynui.h
index 5029e5850..67eb8b8f8 100644
--- a/src/include/ipxe/dynui.h
+++ b/src/include/ipxe/dynui.h
@@ -61,5 +61,6 @@ extern struct dynamic_item * dynui_shortcut ( struct dynamic_ui *dynui,
 					      int key );
 extern int show_menu ( struct dynamic_ui *dynui, unsigned long timeout,
 		       const char *select, struct dynamic_item **selected );
+extern int show_form ( struct dynamic_ui *dynui );
 
 #endif /* _IPXE_DYNUI_H */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index d75661ba7..fcb4f0e68 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -418,6 +418,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_des		      ( ERRFILE_OTHER | 0x00600000 )
 #define ERRFILE_editstring	      ( ERRFILE_OTHER | 0x00610000 )
 #define ERRFILE_widget_ui	      ( ERRFILE_OTHER | 0x00620000 )
+#define ERRFILE_form_ui		      ( ERRFILE_OTHER | 0x00630000 )
 
 /** @} */
 

From 162cc51b6db2caa888e2fb834511d7e4af7e0bfe Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 20 Jun 2024 15:42:48 -0700
Subject: [PATCH 149/237] [form] Reimplement the "login" user interface

Rewrite the code implementing the "login" user interface to use a
predefined interactive form.  The command "login" then becomes roughly
equivalent to:

  #!ipxe

  form
  item          username   Username
  item --secret password   Password
  present

with the result that login form customisations (e.g. to add a Windows
domain name) may be implemented within the scripting language.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/tui/login_ui.c | 96 +++++++++++++++---------------------------
 1 file changed, 33 insertions(+), 63 deletions(-)

diff --git a/src/hci/tui/login_ui.c b/src/hci/tui/login_ui.c
index 9c5148865..02552f0d2 100644
--- a/src/hci/tui/login_ui.c
+++ b/src/hci/tui/login_ui.c
@@ -29,72 +29,42 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
-#include <string.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <curses.h>
-#include <ipxe/console.h>
-#include <ipxe/settings.h>
-#include <ipxe/label.h>
-#include <ipxe/editbox.h>
-#include <ipxe/keys.h>
-#include <ipxe/ansicol.h>
+#include <ipxe/dynui.h>
 #include <ipxe/login_ui.h>
 
-/* Screen layout */
-#define USERNAME_LABEL_ROW	( ( LINES / 2U ) - 4U )
-#define USERNAME_ROW		( ( LINES / 2U ) - 2U )
-#define PASSWORD_LABEL_ROW	( ( LINES / 2U ) + 2U )
-#define PASSWORD_ROW		( ( LINES / 2U ) + 4U )
-#define WIDGET_COL		( ( COLS / 2U ) - 10U )
-#define WIDGET_WIDTH		20U
+static struct dynamic_item username;
+static struct dynamic_item password;
+
+static struct dynamic_ui login = {
+	.items = {
+		.prev = &password.list,
+		.next = &username.list,
+	},
+	.count = 2,
+};
+
+static struct dynamic_item username = {
+	.list = {
+		.prev = &login.items,
+		.next = &password.list,
+	},
+	.name = "username",
+	.text = "Username",
+	.index = 0,
+};
+
+static struct dynamic_item password = {
+	.list = {
+		.prev = &username.list,
+		.next = &login.items,
+	},
+	.name = "password",
+	.text = "Password",
+	.index = 1,
+	.flags = DYNUI_SECRET,
+};
 
 int login_ui ( void ) {
-	char *username;
-	char *password;
-	struct {
-		struct widgets widgets;
-		struct label username_label;
-		struct label password_label;
-		struct edit_box username_box;
-		struct edit_box password_box;
-	} widgets;
-	int rc;
 
-	/* Fetch current setting values */
-	fetchf_setting_copy ( NULL, &username_setting, NULL, NULL, &username );
-	fetchf_setting_copy ( NULL, &password_setting, NULL, NULL, &password );
-
-	/* Construct user interface */
-	memset ( &widgets, 0, sizeof ( widgets ) );
-	init_widgets ( &widgets.widgets );
-	init_label ( &widgets.username_label, USERNAME_LABEL_ROW, WIDGET_COL,
-		     WIDGET_WIDTH, "Username" );
-	init_label ( &widgets.password_label, PASSWORD_LABEL_ROW, WIDGET_COL,
-		     WIDGET_WIDTH, "Password" );
-	init_editbox ( &widgets.username_box, USERNAME_ROW, WIDGET_COL,
-		       WIDGET_WIDTH, 0, &username );
-	init_editbox ( &widgets.password_box, PASSWORD_ROW, WIDGET_COL,
-		       WIDGET_WIDTH, WIDGET_SECRET, &password );
-	add_widget ( &widgets.widgets, &widgets.username_label.widget );
-	add_widget ( &widgets.widgets, &widgets.password_label.widget );
-	add_widget ( &widgets.widgets, &widgets.username_box.widget );
-	add_widget ( &widgets.widgets, &widgets.password_box.widget );
-
-	/* Present user interface */
-	if ( ( rc = widget_ui ( &widgets.widgets ) ) != 0 )
-		goto err_ui;
-
-	/* Store settings on successful completion */
-	if ( ( rc = storef_setting ( NULL, &username_setting, username ) ) !=0)
-		goto err_store_username;
-	if ( ( rc = storef_setting ( NULL, &password_setting, password ) ) !=0)
-		goto err_store_password;
-
- err_store_username:
- err_store_password:
- err_ui:
-	free ( username );
-	free ( password );
-	return rc;
+	return show_form ( &login );
 }

From 821bb326f87fbc000376fdc5371e9e53f666267a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 20 Jun 2024 15:48:59 -0700
Subject: [PATCH 150/237] [hci] Remove the generalised widget user interface
 abstraction

Remove the now-unused generalised text widget user interface, along
with the associated concept of a widget set and the implementation of
a read-only label widget.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/hci/mucurses/widgets/label.c |  74 ----------------
 src/hci/tui/settings_ui.c        |   3 -
 src/hci/tui/widget_ui.c          | 143 -------------------------------
 src/include/ipxe/widget.h        |  34 --------
 4 files changed, 254 deletions(-)
 delete mode 100644 src/hci/mucurses/widgets/label.c
 delete mode 100644 src/hci/tui/widget_ui.c

diff --git a/src/hci/mucurses/widgets/label.c b/src/hci/mucurses/widgets/label.c
deleted file mode 100644
index 29057f089..000000000
--- a/src/hci/mucurses/widgets/label.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <string.h>
-#include <assert.h>
-#include <ipxe/label.h>
-
-/** @file
- *
- * Text label widget
- *
- */
-
-/**
- * Draw text label widget
- *
- * @v widget		Text widget
- */
-static void draw_label ( struct widget *widget ) {
-	struct label *label = container_of ( widget, struct label, widget );
-	unsigned int width = widget->width;
-	unsigned int col = widget->col;
-	const char *text = label->text;
-
-	/* Centre label if width is non-zero */
-	if ( width )
-		col += ( ( width - strlen ( text ) ) / 2 );
-
-	/* Print label content */
-	attron ( A_BOLD );
-	mvprintw ( widget->row, col, "%s", text );
-	attroff ( A_BOLD );
-}
-
-/**
- * Edit text label widget
- *
- * @v widget		Text widget
- * @v key		Key pressed by user
- * @ret key		Key returned to application, or zero
- */
-static int edit_label ( struct widget *widget __unused, int key ) {
-
-	/* Cannot be edited */
-	return key;
-}
-
-/** Text label widget operations */
-struct widget_operations label_operations = {
-	.draw = draw_label,
-	.edit = edit_label,
-};
diff --git a/src/hci/tui/settings_ui.c b/src/hci/tui/settings_ui.c
index bc08750a0..57ff9e9a0 100644
--- a/src/hci/tui/settings_ui.c
+++ b/src/hci/tui/settings_ui.c
@@ -109,8 +109,6 @@ struct settings_ui {
 	struct jump_scroller scroll;
 	/** Current row */
 	struct settings_ui_row row;
-	/** Widget set used for editing setting */
-	struct widgets widgets;
 };
 
 /**
@@ -389,7 +387,6 @@ static int main_loop ( struct settings *settings ) {
 	/* Print initial screen content */
 	color_set ( CPAIR_NORMAL, NULL );
 	memset ( &ui, 0, sizeof ( ui ) );
-	init_widgets ( &ui.widgets );
 	select_settings ( &ui, settings );
 
 	while ( 1 ) {
diff --git a/src/hci/tui/widget_ui.c b/src/hci/tui/widget_ui.c
deleted file mode 100644
index 961545cf4..000000000
--- a/src/hci/tui/widget_ui.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-/** @file
- *
- * Text widget UI
- *
- */
-
-#include <errno.h>
-#include <curses.h>
-#include <ipxe/ansicol.h>
-#include <ipxe/widget.h>
-
-/**
- * Find editable widget in widget set
- *
- * @v widgets		Text widget set
- * @v index		Editable widget index
- * @ret widget		Editable widget, or NULL
- */
-static struct widget * find_widget ( struct widgets *widgets,
-				     unsigned int index ) {
-	struct widget *widget;
-
-	list_for_each_entry ( widget, &widgets->list, list ) {
-		if ( ! ( widget->flags & WIDGET_EDITABLE ) )
-			continue;
-		if ( index-- == 0 )
-			return widget;
-	}
-	return NULL;
-}
-
-/**
- * Text widget user interface main loop
- *
- * @v widgets		Text widget set
- * @ret rc		Return status code
- */
-static int widget_ui_loop ( struct widgets *widgets ) {
-	struct widget *widget;
-	unsigned int current;
-	unsigned int count;
-	int key;
-
-	/* Draw all widgets */
-	list_for_each_entry ( widget, &widgets->list, list )
-		draw_widget ( widget );
-
-	/* Count editable widgets */
-	count = 0;
-	while ( find_widget ( widgets, count ) != NULL )
-		count++;
-
-	/* Main loop */
-	current = 0;
-	while ( 1 ) {
-
-		/* Identify current widget */
-		widget = find_widget ( widgets, current );
-		if ( ! widget )
-			return -ENOENT;
-
-		/* Redraw current widget */
-		draw_widget ( widget );
-
-		/* Process keypress */
-		key = edit_widget ( widget, getkey ( 0 ) );
-		switch ( key ) {
-		case KEY_UP:
-			if ( current > 0 )
-				current--;
-			break;
-		case KEY_DOWN:
-			if ( ++current == count )
-				current--;
-			break;
-		case TAB:
-			if ( ++current == count )
-				current = 0;
-			break;
-		case KEY_ENTER:
-			current++;
-			if ( current >= count )
-				return 0;
-			break;
-		case CTRL_C:
-		case ESC:
-			return -ECANCELED;
-		default:
-			/* Do nothing for unrecognised keys or edit errors */
-			break;
-		}
-	}
-}
-
-/**
- * Present text widget user interface
- *
- * @v widgets		Text widget set
- * @ret rc		Return status code
- */
-int widget_ui ( struct widgets *widgets ) {
-	int rc;
-
-	/* Initialise UI */
-	initscr();
-	start_color();
-	color_set ( CPAIR_NORMAL, NULL );
-	erase();
-
-	/* Run main loop */
-	rc = widget_ui_loop ( widgets );
-
-	/* Terminate UI */
-	color_set ( CPAIR_NORMAL, NULL );
-	endwin();
-
-	return rc;
-}
diff --git a/src/include/ipxe/widget.h b/src/include/ipxe/widget.h
index 0d8af1030..945b4672a 100644
--- a/src/include/ipxe/widget.h
+++ b/src/include/ipxe/widget.h
@@ -10,18 +10,9 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <curses.h>
-#include <ipxe/list.h>
-
-/** A text widget set */
-struct widgets {
-	/** List of widgets (in tab order) */
-	struct list_head list;
-};
 
 /** A text widget */
 struct widget {
-	/** List of widgets (in tab order) */
-	struct list_head list;
 	/** Widget operations */
 	struct widget_operations *op;
 
@@ -65,17 +56,6 @@ struct widget_operations {
 	int ( * edit ) ( struct widget *widget, int key );
 };
 
-/**
- * Initialise text widget set
- *
- * @v widgets		Text widget set
- */
-static inline __attribute__ (( always_inline )) void
-init_widgets ( struct widgets *widgets ) {
-
-	INIT_LIST_HEAD ( &widgets->list );
-}
-
 /**
  * Initialise text widget
  *
@@ -97,18 +77,6 @@ init_widget ( struct widget *widget, struct widget_operations *op,
 	widget->flags = flags;
 }
 
-/**
- * Append text widget
- *
- * @v widgets		Text widget set
- * @v widget		Text widget
- */
-static inline __attribute__ (( always_inline )) void
-add_widget ( struct widgets *widgets, struct widget *widget ) {
-
-	list_add_tail ( &widget->list, &widgets->list );
-}
-
 /**
  * Draw text widget
  *
@@ -137,6 +105,4 @@ edit_widget ( struct widget *widget, int key ) {
 	return widget->op->edit ( widget, key );
 }
 
-extern int widget_ui ( struct widgets *widgets );
-
 #endif /* _IPXE_WIDGET_H */

From 77acf6b41f705384593a057c2bea057283bf429b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 26 Jun 2024 04:29:38 -0700
Subject: [PATCH 151/237] [ipv4] Support small subnets with no directed
 broadcast address

In a small subnet (with a /31 or /32 subnet mask), all addresses
within the subnet are valid host addresses: there is no separate
network address or directed broadcast address.

The logic used in iPXE to determine whether or not to use a link-layer
broadcast address will currently fail in these subnets.  In a /31
subnet, the higher of the two host addresses (i.e. the address with
all host bits set) will be treated as a broadcast address.  In a /32
subnet, the single valid host address will be treated as a broadcast
address.

Fix by adding the concept of a host mask, defined such that an address
in the local subnet with all of the mask bits set to zero represents
the network address, and an address in the local subnet with all of
the mask bits set to one represents the directed broadcast address.
For most subnets, this is simply the inverse of the subnet mask.  For
small subnets (/31 or /32) we can obtain the desired behaviour by
setting the host mask to all ones, so that only the local broadcast
address 255.255.255.255 will be treated as a broadcast address.

Originally-fixed-by: Lukas Stockner <lstockner@genesiscloud.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/in.h |  2 ++
 src/include/ipxe/ip.h | 22 ++++++++++++++++++++--
 src/net/ipv4.c        | 18 ++++++++++++++----
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/src/include/ipxe/in.h b/src/include/ipxe/in.h
index 3044d6316..05a8122ef 100644
--- a/src/include/ipxe/in.h
+++ b/src/include/ipxe/in.h
@@ -33,6 +33,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define IN_IS_MULTICAST( addr ) \
 	( ( (addr) & htonl ( 0xf0000000 ) ) == htonl ( 0xe0000000 ) )
 
+#define IN_IS_SMALL( mask ) ( (mask) & htonl ( 0x00000003 ) )
+
 /**
  * IP address structure
  */
diff --git a/src/include/ipxe/ip.h b/src/include/ipxe/ip.h
index 285be6dcd..b1b5cb2e7 100644
--- a/src/include/ipxe/ip.h
+++ b/src/include/ipxe/ip.h
@@ -64,9 +64,27 @@ struct ipv4_miniroute {
 
 	/** IPv4 address */
 	struct in_addr address;
-	/** Subnet mask */
+	/** Subnet mask
+	 *
+	 * An address with all of these bits in common with our IPv4
+	 * address is in the local subnet.
+	 */
 	struct in_addr netmask;
-	/** Gateway address */
+	/** Host mask
+	 *
+	 * An address in the local subnet with all of these bits set
+	 * to zero represents the network address, and an address in
+	 * the local subnet with all of these bits set to one
+	 * represents the directed broadcast address.  All other
+	 * addresses in the local subnet are valid host addresses.
+	 *
+	 * For most subnets, this is the inverse of the subnet mask.
+	 * In a small subnet (/31 or /32) there is no network address
+	 * or directed broadcast address, and all addresses in the
+	 * subnet are valid host addresses.
+	 */
+	struct in_addr hostmask;
+	/** Gateway address, or zero for no gateway */
 	struct in_addr gateway;
 };
 
diff --git a/src/net/ipv4.c b/src/net/ipv4.c
index b91fa2ad0..5d0cb0f9a 100644
--- a/src/net/ipv4.c
+++ b/src/net/ipv4.c
@@ -85,9 +85,18 @@ static int add_ipv4_miniroute ( struct net_device *netdev,
 				struct in_addr address, struct in_addr netmask,
 				struct in_addr gateway ) {
 	struct ipv4_miniroute *miniroute;
+	struct in_addr hostmask;
+	struct in_addr broadcast;
 
+	/* Calculate host mask */
+	hostmask.s_addr = ( IN_IS_SMALL ( netmask.s_addr ) ?
+			    INADDR_NONE : ~netmask.s_addr );
+	broadcast.s_addr = ( address.s_addr | hostmask.s_addr );
+
+	/* Print debugging information */
 	DBGC ( netdev, "IPv4 add %s", inet_ntoa ( address ) );
 	DBGC ( netdev, "/%s ", inet_ntoa ( netmask ) );
+	DBGC ( netdev, "bc %s ", inet_ntoa ( broadcast ) );
 	if ( gateway.s_addr )
 		DBGC ( netdev, "gw %s ", inet_ntoa ( gateway ) );
 	DBGC ( netdev, "via %s\n", netdev->name );
@@ -103,8 +112,9 @@ static int add_ipv4_miniroute ( struct net_device *netdev,
 	miniroute->netdev = netdev_get ( netdev );
 	miniroute->address = address;
 	miniroute->netmask = netmask;
+	miniroute->hostmask = hostmask;
 	miniroute->gateway = gateway;
-		
+
 	/* Add to end of list if we have a gateway, otherwise
 	 * to start of list.
 	 */
@@ -310,7 +320,7 @@ static int ipv4_tx ( struct io_buffer *iobuf,
 	struct sockaddr_in *sin_dest = ( ( struct sockaddr_in * ) st_dest );
 	struct ipv4_miniroute *miniroute;
 	struct in_addr next_hop;
-	struct in_addr netmask = { .s_addr = 0 };
+	struct in_addr hostmask = { .s_addr = INADDR_NONE };
 	uint8_t ll_dest_buf[MAX_LL_ADDR_LEN];
 	const void *ll_dest;
 	int rc;
@@ -338,7 +348,7 @@ static int ipv4_tx ( struct io_buffer *iobuf,
 	     ( ( miniroute = ipv4_route ( sin_dest->sin_scope_id,
 					  &next_hop ) ) != NULL ) ) {
 		iphdr->src = miniroute->address;
-		netmask = miniroute->netmask;
+		hostmask = miniroute->hostmask;
 		netdev = miniroute->netdev;
 	}
 	if ( ! netdev ) {
@@ -373,7 +383,7 @@ static int ipv4_tx ( struct io_buffer *iobuf,
 		ntohs ( iphdr->chksum ) );
 
 	/* Calculate link-layer destination address, if possible */
-	if ( ( ( next_hop.s_addr ^ INADDR_BROADCAST ) & ~netmask.s_addr ) == 0){
+	if ( ( ( ~next_hop.s_addr ) & hostmask.s_addr ) == 0 ) {
 		/* Broadcast address */
 		ipv4_stats.out_bcast_pkts++;
 		ll_dest = netdev->ll_broadcast;

From b66e27d9b29a172a097c737ab4d378d60fe01b05 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 27 Jun 2024 13:26:39 +0100
Subject: [PATCH 152/237] [ipv6] Expose router address for DHCPv6 leased
 addresses

The DHCPv6 protocol does not itself provide a router address or a
prefix length.  This information is instead obtained from the router
advertisements.

Our IPv6 minirouting table construction logic will first construct an
entry for each advertised prefix, and later update the entry to
include an address assigned within that prefix via stateful DHCPv6 (if
applicable).

This logic fails if the address assigned via stateful DHCPv6 does not
fall within any of the advertised prefixes (e.g. if the network is
configured to use DHCPv6-assigned /128 addresses with no advertised
on-link prefixes).  We will currently treat this situation as
equivalent to having a manually assigned address with no corresponding
router address or prefix length: the routing table entry will use the
default /64 prefix length and will not include the router address.

DHCPv6 is triggered only in response to a router advertisement with
the "Managed Address Configuration (M)" or "Other Configuration (O)"
flags set, and a router address is therefore available at the point
that we initiate DHCPv6.

Record the router address when initiating DHCPv6, and expose this
router address as part of the DHCPv6 settings block.  This allows the
routing table entry for any address assigned via stateful DHCPv6 to
correctly include the router address, even if the assigned address
does not fall within an advertised prefix.

Also provide a fixed /128 prefix length as part of the DHCPv6 settings
block.  When an address assigned via stateful DHCPv6 does not fall
within an advertised prefix, this will cause the routing table entry
to have a /128 prefix length as expected.  (When such an address does
fall within an advertised prefix, it will continue to use the
advertised prefix length.)

Originally-fixed-by: Guvenc Gulce <guevenc.guelce@sap.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/dhcpv6.h |   2 +-
 src/net/ndp.c             |   2 +-
 src/net/udp/dhcpv6.c      | 109 ++++++++++++++++++++++++++++++++------
 3 files changed, 96 insertions(+), 17 deletions(-)

diff --git a/src/include/ipxe/dhcpv6.h b/src/include/ipxe/dhcpv6.h
index 6e70f7e63..065e9c376 100644
--- a/src/include/ipxe/dhcpv6.h
+++ b/src/include/ipxe/dhcpv6.h
@@ -276,6 +276,6 @@ static inline void ipv6_all_dhcp_relay_and_servers ( struct in6_addr *addr ) {
 }
 
 extern int start_dhcpv6 ( struct interface *job, struct net_device *netdev,
-			  int stateful );
+			  struct in6_addr *router, int stateful );
 
 #endif /* _IPXE_DHCPV6_H */
diff --git a/src/net/ndp.c b/src/net/ndp.c
index 373a9360b..3c555f4a3 100644
--- a/src/net/ndp.c
+++ b/src/net/ndp.c
@@ -1221,7 +1221,7 @@ ipv6conf_rx_router_advertisement ( struct net_device *netdev,
 	/* Start DHCPv6 if required */
 	if ( radv->flags & ( NDP_ROUTER_MANAGED | NDP_ROUTER_OTHER ) ) {
 		stateful = ( radv->flags & NDP_ROUTER_MANAGED );
-		if ( ( rc = start_dhcpv6 ( &ipv6conf->dhcp, netdev,
+		if ( ( rc = start_dhcpv6 ( &ipv6conf->dhcp, netdev, router,
 					   stateful ) ) != 0 ) {
 			DBGC ( netdev, "NDP %s could not start state%s DHCPv6: "
 			       "%s\n", netdev->name,
diff --git a/src/net/udp/dhcpv6.c b/src/net/udp/dhcpv6.c
index 9e27dec6f..a49109894 100644
--- a/src/net/udp/dhcpv6.c
+++ b/src/net/udp/dhcpv6.c
@@ -268,6 +268,8 @@ struct dhcpv6_settings {
 	struct settings settings;
 	/** Leased address */
 	struct in6_addr lease;
+	/** Router address */
+	struct in6_addr router;
 	/** Option list */
 	struct dhcpv6_option_list options;
 };
@@ -283,25 +285,21 @@ static int dhcpv6_applies ( struct settings *settings __unused,
 			    const struct setting *setting ) {
 
 	return ( ( setting->scope == &dhcpv6_scope ) ||
-		 ( setting_cmp ( setting, &ip6_setting ) == 0 ) );
+		 ( setting->scope == &ipv6_settings_scope ) );
 }
 
 /**
  * Fetch value of DHCPv6 leased address
  *
- * @v dhcpset		DHCPv6 settings
+ * @v dhcpv6set		DHCPv6 settings
  * @v data		Buffer to fill with setting data
  * @v len		Length of buffer
  * @ret len		Length of setting data, or negative error
  */
-static int dhcpv6_fetch_lease ( struct dhcpv6_settings *dhcpv6set,
-				void *data, size_t len ) {
+static int dhcpv6_fetch_ip6 ( struct dhcpv6_settings *dhcpv6set,
+			      void *data, size_t len ) {
 	struct in6_addr *lease = &dhcpv6set->lease;
 
-	/* Do nothing unless a leased address exists */
-	if ( IN6_IS_ADDR_UNSPECIFIED ( lease ) )
-		return -ENOENT;
-
 	/* Copy leased address */
 	if ( len > sizeof ( *lease ) )
 		len = sizeof ( *lease );
@@ -310,6 +308,72 @@ static int dhcpv6_fetch_lease ( struct dhcpv6_settings *dhcpv6set,
 	return sizeof ( *lease );
 }
 
+/**
+ * Fetch value of DHCPv6 implicit address prefix length
+ *
+ * @v dhcpv6set		DHCPv6 settings
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int dhcpv6_fetch_len6 ( struct dhcpv6_settings *dhcpv6set __unused,
+			       void *data, size_t len ) {
+	uint8_t *len6 = data;
+
+	/* Default to assuming this is the only address on the link.
+	 * If the address falls within a known prefix, then the IPv6
+	 * routing table construction logic will match it against that
+	 * prefix.
+	 */
+	if ( len )
+		*len6 = IPV6_MAX_PREFIX_LEN;
+
+	return sizeof ( *len6 );
+}
+
+/**
+ * Fetch value of DHCPv6 router address
+ *
+ * @v dhcpv6set		DHCPv6 settings
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int dhcpv6_fetch_gateway6 ( struct dhcpv6_settings *dhcpv6set,
+				   void *data, size_t len ) {
+	struct in6_addr *router = &dhcpv6set->router;
+
+	/* Copy router address */
+	if ( len > sizeof ( *router ) )
+		len = sizeof ( *router );
+	memcpy ( data, router, len );
+
+	return sizeof ( *router );
+}
+
+/** A DHCPv6 address setting operation */
+struct dhcpv6_address_operation {
+	/** Generic setting */
+	const struct setting *setting;
+	/**
+	 * Fetch value of setting
+	 *
+	 * @v dhcpv6set		DHCPv6 settings
+	 * @v data		Buffer to fill with setting data
+	 * @v len		Length of buffer
+	 * @ret len		Length of setting data, or negative error
+	 */
+	int ( * fetch ) ( struct dhcpv6_settings *dhcpv6set,
+			  void *data, size_t len );
+};
+
+/** DHCPv6 address settings operations */
+static struct dhcpv6_address_operation dhcpv6_address_operations[] = {
+	{ &ip6_setting, dhcpv6_fetch_ip6 },
+	{ &len6_setting, dhcpv6_fetch_len6 },
+	{ &gateway6_setting, dhcpv6_fetch_gateway6 },
+};
+
 /**
  * Fetch value of DHCPv6 setting
  *
@@ -325,11 +389,20 @@ static int dhcpv6_fetch ( struct settings *settings,
 	struct dhcpv6_settings *dhcpv6set =
 		container_of ( settings, struct dhcpv6_settings, settings );
 	const union dhcpv6_any_option *option;
+	struct dhcpv6_address_operation *op;
 	size_t option_len;
+	unsigned int i;
 
-	/* Handle leased address */
-	if ( setting_cmp ( setting, &ip6_setting ) == 0 )
-		return dhcpv6_fetch_lease ( dhcpv6set, data, len );
+	/* Handle address settings */
+	for ( i = 0 ; i < ( sizeof ( dhcpv6_address_operations ) /
+			    sizeof ( dhcpv6_address_operations[0] ) ) ; i++ ) {
+		op = &dhcpv6_address_operations[i];
+		if ( setting_cmp ( setting, op->setting ) != 0 )
+			continue;
+		if ( IN6_IS_ADDR_UNSPECIFIED ( &dhcpv6set->lease ) )
+			return -ENOENT;
+		return op->fetch ( dhcpv6set, data, len );
+	}
 
 	/* Find option */
 	option = dhcpv6_option ( &dhcpv6set->options, setting->tag );
@@ -354,11 +427,12 @@ static struct settings_operations dhcpv6_settings_operations = {
  * Register DHCPv6 options as network device settings
  *
  * @v lease		DHCPv6 leased address
+ * @v router		DHCPv6 router address
  * @v options		DHCPv6 option list
  * @v parent		Parent settings block
  * @ret rc		Return status code
  */
-static int dhcpv6_register ( struct in6_addr *lease,
+static int dhcpv6_register ( struct in6_addr *lease, struct in6_addr *router,
 			     struct dhcpv6_option_list *options,
 			     struct settings *parent ) {
 	struct dhcpv6_settings *dhcpv6set;
@@ -382,6 +456,7 @@ static int dhcpv6_register ( struct in6_addr *lease,
 	dhcpv6set->options.data = data;
 	dhcpv6set->options.len = len;
 	memcpy ( &dhcpv6set->lease, lease, sizeof ( dhcpv6set->lease ) );
+	memcpy ( &dhcpv6set->router, router, sizeof ( dhcpv6set->router ) );
 
 	/* Register settings */
 	if ( ( rc = register_settings ( &dhcpv6set->settings, parent,
@@ -501,6 +576,8 @@ struct dhcpv6_session {
 
 	/** Network device being configured */
 	struct net_device *netdev;
+	/** Router address */
+	struct in6_addr router;
 	/** Transaction ID */
 	uint8_t xid[3];
 	/** Identity association ID */
@@ -876,8 +953,8 @@ static int dhcpv6_rx ( struct dhcpv6_session *dhcpv6,
 	}
 
 	/* Register settings */
-	if ( ( rc = dhcpv6_register ( &dhcpv6->lease, &options,
-				      parent ) ) != 0 ) {
+	if ( ( rc = dhcpv6_register ( &dhcpv6->lease, &dhcpv6->router,
+				      &options, parent ) ) != 0 ) {
 		DBGC ( dhcpv6, "DHCPv6 %s could not register settings: %s\n",
 		       dhcpv6->netdev->name, strerror ( rc ) );
 		goto done;
@@ -915,11 +992,12 @@ static struct interface_descriptor dhcpv6_xfer_desc =
  *
  * @v job		Job control interface
  * @v netdev		Network device
+ * @v router		Router address
  * @v stateful		Perform stateful address autoconfiguration
  * @ret rc		Return status code
  */
 int start_dhcpv6 ( struct interface *job, struct net_device *netdev,
-		   int stateful ) {
+		   struct in6_addr *router, int stateful ) {
 	struct ll_protocol *ll_protocol = netdev->ll_protocol;
 	struct dhcpv6_session *dhcpv6;
 	struct {
@@ -944,6 +1022,7 @@ int start_dhcpv6 ( struct interface *job, struct net_device *netdev,
 	intf_init ( &dhcpv6->job, &dhcpv6_job_desc, &dhcpv6->refcnt );
 	intf_init ( &dhcpv6->xfer, &dhcpv6_xfer_desc, &dhcpv6->refcnt );
 	dhcpv6->netdev = netdev_get ( netdev );
+	memcpy ( &dhcpv6->router, router, sizeof ( dhcpv6->router ) );
 	xid = random();
 	memcpy ( dhcpv6->xid, &xid, sizeof ( dhcpv6->xid ) );
 	dhcpv6->start = currticks();

From 5a9f476d4f1395e69cbb845d7379b0e3591028c0 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 5 Jul 2024 23:54:50 +0100
Subject: [PATCH 153/237] [cloud] Add utility for importing images to Google
 Compute Engine

Following the example of aws-import, add a utility that can be used to
upload an iPXE disk image to Google Compute Engine as a bootable
image.  For example:

  make CONFIG=cloud EMBED=config/cloud/gce.ipxe \
       bin-x86_64-pcbios/ipxe.usb bin-x86_64-efi/ipxe.usb

  make CONFIG=cloud EMBED=config/cloud/gce.ipxe \
       CROSS=aarch64-linux-gnu- bin-arm64-efi/ipxe.usb

  ../contrib/cloud/gce-import -p \
       bin-x86_64-pcbios/ipxe.usb \
       bin-x86_64-efi/ipxe.usb \
       bin-arm64-efi/ipxe.usb

The iPXE disk image is automatically wrapped into a tarball containing
a single file named "disk.raw", uploaded to a temporary bucket in
Google Cloud Storage, and used to create a bootable image.  The
temporary bucket is deleted after use.

An appropriate image family name is identified automatically: "ipxe"
for BIOS images, "ipxe-uefi-x86-64" for x86_64 UEFI images, and
"ipxe-uefi-arm64" for AArch64 UEFI images.  This allows the latest
image within each family to be launched within needing to know the
precise image name.

Google Compute Engine images are globally scoped and are available
(and cached upon first use) in all regions.  The initial placement of
the image may be controlled indirectly by using the "--location"
option to specify the Google Cloud Storage location used for the
temporary upload bucket: the image will then be created in the closest
multi-region to the storage location.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 contrib/cloud/gce-import | 167 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 167 insertions(+)
 create mode 100755 contrib/cloud/gce-import

diff --git a/contrib/cloud/gce-import b/contrib/cloud/gce-import
new file mode 100755
index 000000000..e7adfee84
--- /dev/null
+++ b/contrib/cloud/gce-import
@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import date
+import io
+import subprocess
+import tarfile
+from uuid import uuid4
+
+from google.cloud import compute
+from google.cloud import exceptions
+from google.cloud import storage
+
+IPXE_STORAGE_PREFIX = 'ipxe-upload-temp-'
+
+FEATURE_GVNIC = compute.GuestOsFeature(type_="GVNIC")
+FEATURE_IDPF = compute.GuestOsFeature(type_="IDPF")
+FEATURE_UEFI = compute.GuestOsFeature(type_="UEFI_COMPATIBLE")
+
+POLICY_PUBLIC = compute.Policy(bindings=[{
+    "role": "roles/compute.imageUser",
+    "members": ["allAuthenticatedUsers"],
+}])
+
+def delete_temp_bucket(bucket):
+    """Remove temporary bucket"""
+    assert bucket.name.startswith(IPXE_STORAGE_PREFIX)
+    for blob in bucket.list_blobs(prefix=IPXE_STORAGE_PREFIX):
+        assert blob.name.startswith(IPXE_STORAGE_PREFIX)
+        blob.delete()
+    if not list(bucket.list_blobs()):
+        bucket.delete()
+
+def create_temp_bucket(location):
+    """Create temporary bucket (and remove any stale temporary buckets)"""
+    client = storage.Client()
+    for bucket in client.list_buckets(prefix=IPXE_STORAGE_PREFIX):
+        delete_temp_bucket(bucket)
+    name = '%s%s' % (IPXE_STORAGE_PREFIX, uuid4())
+    return client.create_bucket(name, location=location)
+
+def create_tarball(image):
+    """Create raw disk image tarball"""
+    tarball = io.BytesIO()
+    with tarfile.open(fileobj=tarball, mode='w:gz',
+                      format=tarfile.GNU_FORMAT) as tar:
+        tar.add(image, arcname='disk.raw')
+    tarball.seek(0)
+    return tarball
+
+def upload_blob(bucket, image):
+    """Upload raw disk image blob"""
+    blob = bucket.blob('%s%s.tar.gz' % (IPXE_STORAGE_PREFIX, uuid4()))
+    tarball = create_tarball(image)
+    blob.upload_from_file(tarball)
+    return blob
+
+def detect_uefi(image):
+    """Identify UEFI CPU architecture(s)"""
+    mdir = subprocess.run(['mdir', '-b', '-i', image, '::/EFI/BOOT'],
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                          check=False)
+    mapping = {
+        b'BOOTX64.EFI': 'x86_64',
+        b'BOOTAA64.EFI': 'arm64',
+    }
+    uefi = [
+        arch
+        for filename, arch in mapping.items()
+        if filename in mdir.stdout
+    ]
+    return uefi
+
+def image_architecture(uefi):
+    """Get image architecture"""
+    return uefi[0] if len(uefi) == 1 else None if uefi else 'x86_64'
+
+def image_features(uefi):
+    """Get image feature list"""
+    features = [FEATURE_GVNIC, FEATURE_IDPF]
+    if uefi:
+        features.append(FEATURE_UEFI)
+    return features
+
+def image_name(base, uefi):
+    """Calculate image name or family name"""
+    suffix = ('-uefi-%s' % uefi[0].replace('_', '-') if len(uefi) == 1 else
+              '-uefi-multi' if uefi else '')
+    return '%s%s' % (base, suffix)
+
+def create_image(project, basename, basefamily, overwrite, public, bucket,
+                 image):
+    """Create image"""
+    client = compute.ImagesClient()
+    uefi = detect_uefi(image)
+    architecture = image_architecture(uefi)
+    features = image_features(uefi)
+    name = image_name(basename, uefi)
+    family = image_name(basefamily, uefi)
+    if overwrite:
+        try:
+            client.delete(project=project, image=name).result()
+        except exceptions.NotFound:
+            pass
+    blob = upload_blob(bucket, image)
+    disk = compute.RawDisk(source=blob.public_url)
+    image = compute.Image(name=name, family=family, architecture=architecture,
+                          guest_os_features=features, raw_disk=disk)
+    client.insert(project=project, image_resource=image).result()
+    if public:
+        request = compute.GlobalSetPolicyRequest(policy=POLICY_PUBLIC)
+        client.set_iam_policy(project=project, resource=name,
+                              global_set_policy_request_resource=request)
+    image = client.get(project=project, image=name)
+    return image
+
+# Parse command-line arguments
+#
+parser = argparse.ArgumentParser(description="Import Google Cloud image")
+parser.add_argument('--name', '-n',
+                    help="Base image name")
+parser.add_argument('--family', '-f',
+                    help="Base family name")
+parser.add_argument('--public', '-p', action='store_true',
+                    help="Make image public")
+parser.add_argument('--overwrite', action='store_true',
+                    help="Overwrite any existing image with same name")
+parser.add_argument('--project', '-j', default="ipxe-images",
+                    help="Google Cloud project")
+parser.add_argument('--location', '-l',
+                    help="Google Cloud Storage initial location")
+parser.add_argument('image', nargs='+', help="iPXE disk image")
+args = parser.parse_args()
+
+# Use default family name if none specified
+if not args.family:
+    args.family = 'ipxe'
+
+# Use default name if none specified
+if not args.name:
+    args.name = '%s-%s' % (args.family, date.today().strftime('%Y%m%d'))
+
+# Create temporary upload bucket
+bucket = create_temp_bucket(args.location)
+
+# Use one thread per image to maximise parallelism
+with ThreadPoolExecutor(max_workers=len(args.image)) as executor:
+    futures = {executor.submit(create_image,
+                               project=args.project,
+                               basename=args.name,
+                               basefamily=args.family,
+                               overwrite=args.overwrite,
+                               public=args.public,
+                               bucket=bucket,
+                               image=image): image
+               for image in args.image}
+    results = {futures[future]: future.result()
+               for future in as_completed(futures)}
+
+# Delete temporary upload bucket
+delete_temp_bucket(bucket)
+
+# Show created images
+for image in args.image:
+    result = results[image]
+    print("%s (%s) %s" % (result.name, result.family, result.status))

From c7b76e3adc3b4365aa3b490f24ae22375901c559 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 24 Jul 2024 14:30:58 +0100
Subject: [PATCH 154/237] [gve] Add driver for Google Virtual Ethernet NIC

The Google Virtual Ethernet NIC (GVE or gVNIC) is found only in Google
Cloud instances.  There is essentially zero documentation available
beyond the mostly uncommented source code in the Linux kernel.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/fault.h         |    3 +
 src/drivers/net/gve.c      | 1607 ++++++++++++++++++++++++++++++++++++
 src/drivers/net/gve.h      |  702 ++++++++++++++++
 src/include/ipxe/errfile.h |    1 +
 4 files changed, 2313 insertions(+)
 create mode 100644 src/drivers/net/gve.c
 create mode 100644 src/drivers/net/gve.h

diff --git a/src/config/fault.h b/src/config/fault.h
index 5024a8ff3..b6ee3c934 100644
--- a/src/config/fault.h
+++ b/src/config/fault.h
@@ -29,6 +29,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /* Corrupt every N received PeerDist packets */
 #define PEERBLK_CORRUPT_RATE 0
 
+/* Experience virtual machine migration on every N watchdog checks */
+#define VM_MIGRATED_RATE 0
+
 #include <config/local/fault.h>
 
 #endif /* CONFIG_FAULT_H */
diff --git a/src/drivers/net/gve.c b/src/drivers/net/gve.c
new file mode 100644
index 000000000..03edc0899
--- /dev/null
+++ b/src/drivers/net/gve.c
@@ -0,0 +1,1607 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/dma.h>
+#include <ipxe/pci.h>
+#include <ipxe/fault.h>
+#include "gve.h"
+
+/** @file
+ *
+ * Google Virtual Ethernet network driver
+ *
+ */
+
+/* Disambiguate the various error causes */
+#define EINFO_EIO_ADMIN_UNSET						\
+	__einfo_uniqify ( EINFO_EIO, 0x00, "Uncompleted" )
+#define EIO_ADMIN_UNSET							\
+	__einfo_error ( EINFO_EIO_ADMIN_UNSET )
+#define EINFO_EIO_ADMIN_ABORTED						\
+	__einfo_uniqify ( EINFO_EIO, 0x10, "Aborted" )
+#define EIO_ADMIN_ABORTED						\
+	__einfo_error ( EINFO_EIO_ADMIN_ABORTED )
+#define EINFO_EIO_ADMIN_EXISTS						\
+	__einfo_uniqify ( EINFO_EIO, 0x11, "Already exists" )
+#define EIO_ADMIN_EXISTS						\
+	__einfo_error ( EINFO_EIO_ADMIN_EXISTS )
+#define EINFO_EIO_ADMIN_CANCELLED					\
+	__einfo_uniqify ( EINFO_EIO, 0x12, "Cancelled" )
+#define EIO_ADMIN_CANCELLED						\
+	__einfo_error ( EINFO_EIO_ADMIN_CANCELLED )
+#define EINFO_EIO_ADMIN_DATALOSS					\
+	__einfo_uniqify ( EINFO_EIO, 0x13, "Data loss" )
+#define EIO_ADMIN_DATALOSS						\
+	__einfo_error ( EINFO_EIO_ADMIN_DATALOSS )
+#define EINFO_EIO_ADMIN_DEADLINE					\
+	__einfo_uniqify ( EINFO_EIO, 0x14, "Deadline exceeded" )
+#define EIO_ADMIN_DEADLINE						\
+	__einfo_error ( EINFO_EIO_ADMIN_DEADLINE )
+#define EINFO_EIO_ADMIN_PRECONDITION					\
+	__einfo_uniqify ( EINFO_EIO, 0x15, "Failed precondition" )
+#define EIO_ADMIN_PRECONDITION						\
+	__einfo_error ( EINFO_EIO_ADMIN_PRECONDITION )
+#define EINFO_EIO_ADMIN_INTERNAL					\
+	__einfo_uniqify ( EINFO_EIO, 0x16, "Internal error" )
+#define EIO_ADMIN_INTERNAL						\
+	__einfo_error ( EINFO_EIO_ADMIN_INTERNAL )
+#define EINFO_EIO_ADMIN_INVAL						\
+	__einfo_uniqify ( EINFO_EIO, 0x17, "Invalid argument" )
+#define EIO_ADMIN_INVAL							\
+	__einfo_error ( EINFO_EIO_ADMIN_INVAL )
+#define EINFO_EIO_ADMIN_NOT_FOUND					\
+	__einfo_uniqify ( EINFO_EIO, 0x18, "Not found" )
+#define EIO_ADMIN_NOT_FOUND						\
+	__einfo_error ( EINFO_EIO_ADMIN_NOT_FOUND )
+#define EINFO_EIO_ADMIN_RANGE						\
+	__einfo_uniqify ( EINFO_EIO, 0x19, "Out of range" )
+#define EIO_ADMIN_RANGE							\
+	__einfo_error ( EINFO_EIO_ADMIN_RANGE )
+#define EINFO_EIO_ADMIN_PERM						\
+	__einfo_uniqify ( EINFO_EIO, 0x1a, "Permission denied" )
+#define EIO_ADMIN_PERM							\
+	__einfo_error ( EINFO_EIO_ADMIN_PERM )
+#define EINFO_EIO_ADMIN_UNAUTH						\
+	__einfo_uniqify ( EINFO_EIO, 0x1b, "Unauthenticated" )
+#define EIO_ADMIN_UNAUTH						\
+	__einfo_error ( EINFO_EIO_ADMIN_UNAUTH )
+#define EINFO_EIO_ADMIN_RESOURCE					\
+	__einfo_uniqify ( EINFO_EIO, 0x1c, "Resource exhausted" )
+#define EIO_ADMIN_RESOURCE						\
+	__einfo_error ( EINFO_EIO_ADMIN_RESOURCE )
+#define EINFO_EIO_ADMIN_UNAVAIL						\
+	__einfo_uniqify ( EINFO_EIO, 0x1d, "Unavailable" )
+#define EIO_ADMIN_UNAVAIL						\
+	__einfo_error ( EINFO_EIO_ADMIN_UNAVAIL )
+#define EINFO_EIO_ADMIN_NOTSUP						\
+	__einfo_uniqify ( EINFO_EIO, 0x1e, "Unimplemented" )
+#define EIO_ADMIN_NOTSUP	       					\
+	__einfo_error ( EINFO_EIO_ADMIN_NOTSUP )
+#define EINFO_EIO_ADMIN_UNKNOWN						\
+	__einfo_uniqify ( EINFO_EIO, 0x1f, "Unknown error" )
+#define EIO_ADMIN_UNKNOWN						\
+	__einfo_error ( EINFO_EIO_ADMIN_UNKNOWN )
+#define EIO_ADMIN( status )						\
+	EUNIQ ( EINFO_EIO, ( (status) & 0x1f ),				\
+		EIO_ADMIN_UNSET, EIO_ADMIN_ABORTED, EIO_ADMIN_EXISTS,	\
+		EIO_ADMIN_CANCELLED, EIO_ADMIN_DATALOSS,		\
+		EIO_ADMIN_DEADLINE, EIO_ADMIN_NOT_FOUND,		\
+		EIO_ADMIN_RANGE, EIO_ADMIN_PERM, EIO_ADMIN_UNAUTH,	\
+		EIO_ADMIN_RESOURCE, EIO_ADMIN_UNAVAIL,			\
+		EIO_ADMIN_NOTSUP, EIO_ADMIN_UNKNOWN )
+
+/******************************************************************************
+ *
+ * Device reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Reset hardware
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_reset ( struct gve_nic *gve ) {
+	uint32_t pfn;
+	unsigned int i;
+
+	/* Skip reset if admin queue page frame number is already
+	 * clear.  Triggering a reset on an already-reset device seems
+	 * to cause a delayed reset to be scheduled.  This can cause
+	 * the device to end up in a reset loop, where each attempt to
+	 * recover from reset triggers another reset a few seconds
+	 * later.
+	 */
+	pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
+	if ( ! pfn ) {
+		DBGC ( gve, "GVE %p skipping reset\n", gve );
+		return 0;
+	}
+
+	/* Clear admin queue page frame number */
+	writel ( 0, gve->cfg + GVE_CFG_ADMIN_PFN );
+	wmb();
+
+	/* Wait for device to reset */
+	for ( i = 0 ; i < GVE_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Delay */
+		mdelay ( 1 );
+
+		/* Check for reset completion */
+		pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
+		if ( ! pfn )
+			return 0;
+	}
+
+	DBGC ( gve, "GVE %p reset timed out (PFN %#08x devstat %#08x)\n",
+	       gve, bswap_32 ( pfn ),
+	       bswap_32 ( readl ( gve->cfg + GVE_CFG_DEVSTAT ) ) );
+	return -ETIMEDOUT;
+}
+
+/******************************************************************************
+ *
+ * Admin queue
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate admin queue
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_admin_alloc ( struct gve_nic *gve ) {
+	struct dma_device *dma = gve->dma;
+	struct gve_admin *admin = &gve->admin;
+	struct gve_irqs *irqs = &gve->irqs;
+	struct gve_events *events = &gve->events;
+	struct gve_scratch *scratch = &gve->scratch;
+	size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
+	size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
+	size_t events_len = ( GVE_EVENT_MAX * sizeof ( events->event[0] ) );
+	size_t scratch_len = sizeof ( *scratch->buf );
+	int rc;
+
+	/* Allocate admin queue */
+	admin->cmd = dma_alloc ( dma, &admin->map, admin_len, GVE_ALIGN );
+	if ( ! admin->cmd ) {
+		rc = -ENOMEM;
+		goto err_admin;
+	}
+
+	/* Allocate interrupt channels */
+	irqs->irq = dma_alloc ( dma, &irqs->map, irqs_len, GVE_ALIGN );
+	if ( ! irqs->irq ) {
+		rc = -ENOMEM;
+		goto err_irqs;
+	}
+
+	/* Allocate event counters */
+	events->event = dma_alloc ( dma, &events->map, events_len, GVE_ALIGN );
+	if ( ! events->event ) {
+		rc = -ENOMEM;
+		goto err_events;
+	}
+
+	/* Allocate scratch buffer */
+	scratch->buf = dma_alloc ( dma, &scratch->map, scratch_len, GVE_ALIGN );
+	if ( ! scratch->buf ) {
+		rc = -ENOMEM;
+		goto err_scratch;
+	}
+
+	DBGC ( gve, "GVE %p AQ at [%08lx,%08lx)\n",
+	       gve, virt_to_phys ( admin->cmd ),
+	       ( virt_to_phys ( admin->cmd ) + admin_len ) );
+	return 0;
+
+	dma_free ( &scratch->map, scratch->buf, scratch_len );
+ err_scratch:
+	dma_free ( &events->map, events->event, events_len );
+ err_events:
+	dma_free ( &irqs->map, irqs->irq, irqs_len );
+ err_irqs:
+	dma_free ( &admin->map, admin->cmd, admin_len );
+ err_admin:
+	return rc;
+}
+
+/**
+ * Free admin queue
+ *
+ * @v gve		GVE device
+ */
+static void gve_admin_free ( struct gve_nic *gve ) {
+	struct gve_admin *admin = &gve->admin;
+	struct gve_irqs *irqs = &gve->irqs;
+	struct gve_events *events = &gve->events;
+	struct gve_scratch *scratch = &gve->scratch;
+	size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
+	size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
+	size_t events_len = ( GVE_EVENT_MAX * sizeof ( events->event[0] ) );
+	size_t scratch_len = sizeof ( *scratch->buf );
+
+	/* Free scratch buffer */
+	dma_free ( &scratch->map, scratch->buf, scratch_len );
+
+	/* Free event counter */
+	dma_free ( &events->map, events->event, events_len );
+
+	/* Free interrupt channels */
+	dma_free ( &irqs->map, irqs->irq, irqs_len );
+
+	/* Free admin queue */
+	dma_free ( &admin->map, admin->cmd, admin_len );
+}
+
+/**
+ * Enable admin queue
+ *
+ * @v gve		GVE device
+ */
+static void gve_admin_enable ( struct gve_nic *gve ) {
+	struct gve_admin *admin = &gve->admin;
+	size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
+	physaddr_t base;
+
+	/* Reset queue */
+	admin->prod = 0;
+
+	/* Program queue addresses and capabilities */
+	base = dma ( &admin->map, admin->cmd );
+	writel ( bswap_32 ( base / GVE_PAGE_SIZE ),
+		 gve->cfg + GVE_CFG_ADMIN_PFN );
+	writel ( bswap_32 ( base & 0xffffffffUL ),
+		 gve->cfg + GVE_CFG_ADMIN_BASE_LO );
+	if ( sizeof ( base ) > sizeof ( uint32_t ) ) {
+		writel ( bswap_32 ( ( ( uint64_t ) base ) >> 32 ),
+			 gve->cfg + GVE_CFG_ADMIN_BASE_HI );
+	} else {
+		writel ( 0, gve->cfg + GVE_CFG_ADMIN_BASE_HI );
+	}
+	writel ( bswap_16 ( admin_len ), gve->cfg + GVE_CFG_ADMIN_LEN );
+	writel ( bswap_32 ( GVE_CFG_DRVSTAT_RUN ), gve->cfg + GVE_CFG_DRVSTAT );
+}
+
+/**
+ * Get next available admin queue command slot
+ *
+ * @v gve		GVE device
+ * @ret cmd		Admin queue command
+ */
+static union gve_admin_command * gve_admin_command ( struct gve_nic *gve ) {
+	struct gve_admin *admin = &gve->admin;
+	union gve_admin_command *cmd;
+	unsigned int index;
+
+	/* Get next command slot */
+	index = admin->prod;
+	cmd = &admin->cmd[ index % GVE_ADMIN_COUNT ];
+
+	/* Initialise request */
+	memset ( cmd, 0, sizeof ( *cmd ) );
+
+	return cmd;
+}
+
+/**
+ * Wait for admin queue command to complete
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_admin_wait ( struct gve_nic *gve ) {
+	struct gve_admin *admin = &gve->admin;
+	uint32_t evt;
+	uint32_t pfn;
+	unsigned int i;
+
+	/* Wait for any outstanding commands to complete */
+	for ( i = 0 ; i < GVE_ADMIN_MAX_WAIT_MS ; i++ ) {
+
+		/* Check event counter */
+		rmb();
+		evt = bswap_32 ( readl ( gve->cfg + GVE_CFG_ADMIN_EVT ) );
+		if ( evt == admin->prod )
+			return 0;
+
+		/* Check for device reset */
+		pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
+		if ( ! pfn )
+			break;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( gve, "GVE %p AQ %#02x %s (completed %#02x, status %#08x)\n",
+	       gve, admin->prod, ( pfn ? "timed out" : "saw reset" ), evt,
+	       bswap_32 ( readl ( gve->cfg + GVE_CFG_DEVSTAT ) ) );
+	return ( pfn ? -ETIMEDOUT : -ECONNRESET );
+}
+
+/**
+ * Issue admin queue command
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_admin ( struct gve_nic *gve ) {
+	struct gve_admin *admin = &gve->admin;
+	union gve_admin_command *cmd;
+	unsigned int index;
+	uint32_t opcode;
+	uint32_t status;
+	int rc;
+
+	/* Ensure admin queue is idle */
+	if ( ( rc = gve_admin_wait ( gve ) ) != 0 )
+		return rc;
+
+	/* Get next command slot */
+	index = admin->prod;
+	cmd = &admin->cmd[ index % GVE_ADMIN_COUNT ];
+	opcode = cmd->hdr.opcode;
+	DBGC2 ( gve, "GVE %p AQ %#02x command %#04x request:\n",
+		gve, index, opcode );
+	DBGC2_HDA ( gve, 0, cmd, sizeof ( *cmd ) );
+
+	/* Increment producer counter */
+	admin->prod++;
+
+	/* Ring doorbell */
+	wmb();
+	writel ( bswap_32 ( admin->prod ), gve->cfg + GVE_CFG_ADMIN_DB );
+
+	/* Wait for command to complete */
+	if ( ( rc = gve_admin_wait ( gve ) ) != 0 )
+		return rc;
+
+	/* Check command status */
+	status = be32_to_cpu ( cmd->hdr.status );
+	if ( status != GVE_ADMIN_STATUS_OK ) {
+		rc = -EIO_ADMIN ( status );
+		DBGC ( gve, "GVE %p AQ %#02x command %#04x failed: %#08x\n",
+		       gve, index, opcode, status );
+		DBGC_HDA ( gve, 0, cmd, sizeof ( *cmd ) );
+		DBGC ( gve, "GVE %p AQ error: %s\n", gve, strerror ( rc ) );
+		return rc;
+	}
+
+	DBGC2 ( gve, "GVE %p AQ %#02x command %#04x result:\n",
+		gve, index, opcode );
+	DBGC2_HDA ( gve, 0, cmd, sizeof ( *cmd ) );
+	return 0;
+}
+
+/**
+ * Issue simple admin queue command
+ *
+ * @v gve		GVE device
+ * @v opcode		Operation code
+ * @v id		ID parameter (or zero if not applicable)
+ * @ret rc		Return status code
+ *
+ * Several admin queue commands take either an empty parameter list or
+ * a single 32-bit ID parameter.
+ */
+static int gve_admin_simple ( struct gve_nic *gve, unsigned int opcode,
+			      unsigned int id ) {
+	union gve_admin_command *cmd;
+	int rc;
+
+	/* Construct request */
+	cmd = gve_admin_command ( gve );
+	cmd->hdr.opcode = opcode;
+	cmd->simple.id = cpu_to_be32 ( id );
+
+	/* Issue command */
+	if ( ( rc = gve_admin ( gve ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Get device descriptor
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_describe ( struct gve_nic *gve ) {
+	struct net_device *netdev = gve->netdev;
+	struct gve_device_descriptor *desc = &gve->scratch.buf->desc;
+	union gve_admin_command *cmd;
+	int rc;
+
+	/* Construct request */
+	cmd = gve_admin_command ( gve );
+	cmd->hdr.opcode = GVE_ADMIN_DESCRIBE;
+	cmd->desc.addr = cpu_to_be64 ( dma ( &gve->scratch.map, desc ) );
+	cmd->desc.ver = cpu_to_be32 ( GVE_ADMIN_DESCRIBE_VER );
+	cmd->desc.len = cpu_to_be32 ( sizeof ( *desc ) );
+
+	/* Issue command */
+	if ( ( rc = gve_admin ( gve ) ) != 0 )
+		return rc;
+	DBGC2 ( gve, "GVE %p device descriptor:\n", gve );
+	DBGC2_HDA ( gve, 0, desc, sizeof ( *desc ) );
+
+	/* Extract queue parameters */
+	gve->events.count = be16_to_cpu ( desc->counters );
+	if ( gve->events.count > GVE_EVENT_MAX )
+		gve->events.count = GVE_EVENT_MAX;
+	gve->tx.count = be16_to_cpu ( desc->tx_count );
+	gve->rx.count = be16_to_cpu ( desc->rx_count );
+	DBGC ( gve, "GVE %p using %d TX, %d RX, %d/%d events\n",
+	       gve, gve->tx.count, gve->rx.count, gve->events.count,
+	       be16_to_cpu ( desc->counters ) );
+
+	/* Extract network parameters */
+	build_assert ( sizeof ( desc->mac ) == ETH_ALEN );
+	memcpy ( netdev->hw_addr, &desc->mac, sizeof ( desc->mac ) );
+	netdev->mtu = be16_to_cpu ( desc->mtu );
+	netdev->max_pkt_len = ( netdev->mtu + ETH_HLEN );
+	DBGC ( gve, "GVE %p MAC %s (\"%s\") MTU %zd\n",
+	       gve, eth_ntoa ( netdev->hw_addr ),
+	       inet_ntoa ( desc->mac.in ), netdev->mtu );
+
+	return 0;
+}
+
+/**
+ * Configure device resources
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_configure ( struct gve_nic *gve ) {
+	struct gve_events *events = &gve->events;
+	struct gve_irqs *irqs = &gve->irqs;
+	union gve_admin_command *cmd;
+	unsigned int db_off;
+	unsigned int i;
+	int rc;
+
+	/* Construct request */
+	cmd = gve_admin_command ( gve );
+	cmd->hdr.opcode = GVE_ADMIN_CONFIGURE;
+	cmd->conf.events =
+		cpu_to_be64 ( dma ( &events->map, events->event ) );
+	cmd->conf.irqs =
+		cpu_to_be64 ( dma ( &irqs->map, irqs->irq ) );
+	cmd->conf.num_events = cpu_to_be32 ( events->count );
+	cmd->conf.num_irqs = cpu_to_be32 ( GVE_IRQ_COUNT );
+	cmd->conf.irq_stride = cpu_to_be32 ( sizeof ( irqs->irq[0] ) );
+
+	/* Issue command */
+	if ( ( rc = gve_admin ( gve ) ) != 0 )
+		return rc;
+
+	/* Disable all interrupts */
+	for ( i = 0 ; i < GVE_IRQ_COUNT ; i++ ) {
+		db_off = ( be32_to_cpu ( irqs->irq[i].db_idx ) *
+			   sizeof ( uint32_t ) );
+		DBGC ( gve, "GVE %p IRQ %d doorbell +%#04x\n", gve, i, db_off );
+		irqs->db[i] = ( gve->db + db_off );
+		writel ( bswap_32 ( GVE_IRQ_DISABLE ), irqs->db[i] );
+	}
+
+	return 0;
+}
+
+/**
+ * Deconfigure device resources
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_deconfigure ( struct gve_nic *gve ) {
+	int rc;
+
+	/* Issue command (with meaningless ID) */
+	if ( ( rc = gve_admin_simple ( gve, GVE_ADMIN_DECONFIGURE, 0 ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Register queue page list
+ *
+ * @v gve		GVE device
+ * @v qpl		Queue page list
+ * @ret rc		Return status code
+ */
+static int gve_register ( struct gve_nic *gve, struct gve_qpl *qpl ) {
+	struct gve_pages *pages = &gve->scratch.buf->pages;
+	union gve_admin_command *cmd;
+	physaddr_t addr;
+	unsigned int i;
+	int rc;
+
+	/* Build page address list */
+	for ( i = 0 ; i < qpl->count ; i++ ) {
+		addr = user_to_phys ( qpl->data, ( i * GVE_PAGE_SIZE ) );
+		pages->addr[i] = cpu_to_be64 ( dma_phys ( &qpl->map, addr ) );
+	}
+
+	/* Construct request */
+	cmd = gve_admin_command ( gve );
+	cmd->hdr.opcode = GVE_ADMIN_REGISTER;
+	cmd->reg.id = cpu_to_be32 ( qpl->id );
+	cmd->reg.count = cpu_to_be32 ( qpl->count );
+	cmd->reg.addr = cpu_to_be64 ( dma ( &gve->scratch.map, pages ) );
+	cmd->reg.size = cpu_to_be64 ( GVE_PAGE_SIZE );
+
+	/* Issue command */
+	if ( ( rc = gve_admin ( gve ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Unregister page list
+ *
+ * @v gve		GVE device
+ * @v qpl		Queue page list
+ * @ret rc		Return status code
+ */
+static int gve_unregister ( struct gve_nic *gve, struct gve_qpl *qpl ) {
+	int rc;
+
+	/* Issue command */
+	if ( ( rc = gve_admin_simple ( gve, GVE_ADMIN_UNREGISTER,
+				       qpl->id ) ) != 0 ) {
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Construct command to create transmit queue
+ *
+ * @v queue		Transmit queue
+ * @v cmd		Admin queue command
+ */
+static void gve_create_tx_param ( struct gve_queue *queue,
+				  union gve_admin_command *cmd ) {
+	struct gve_admin_create_tx *create = &cmd->create_tx;
+	const struct gve_queue_type *type = queue->type;
+	physaddr_t desc = user_to_phys ( queue->desc, 0 );
+
+	/* Construct request parameters */
+	create->res = cpu_to_be64 ( dma ( &queue->res_map, queue->res ) );
+	create->desc = cpu_to_be64 ( dma_phys ( &queue->desc_map, desc ) );
+	create->qpl_id = cpu_to_be32 ( type->qpl );
+	create->notify_id = cpu_to_be32 ( type->irq );
+}
+
+/**
+ * Construct command to create receive queue
+ *
+ * @v queue		Receive queue
+ * @v cmd		Admin queue command
+ */
+static void gve_create_rx_param ( struct gve_queue *queue,
+				  union gve_admin_command *cmd ) {
+	struct gve_admin_create_rx *create = &cmd->create_rx;
+	const struct gve_queue_type *type = queue->type;
+	physaddr_t desc = user_to_phys ( queue->desc, 0 );
+	physaddr_t cmplt = user_to_phys ( queue->cmplt, 0 );
+
+	/* Construct request parameters */
+	create->notify_id = cpu_to_be32 ( type->irq );
+	create->res = cpu_to_be64 ( dma ( &queue->res_map, queue->res ) );
+	create->desc = cpu_to_be64 ( dma_phys ( &queue->desc_map, desc ) );
+	create->cmplt = cpu_to_be64 ( dma_phys ( &queue->cmplt_map, cmplt ) );
+	create->qpl_id = cpu_to_be32 ( type->qpl );
+	create->bufsz = cpu_to_be16 ( GVE_BUF_SIZE );
+}
+
+/**
+ * Create transmit or receive queue
+ *
+ * @v gve		GVE device
+ * @v queue		Descriptor queue
+ * @ret rc		Return status code
+ */
+static int gve_create_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
+	const struct gve_queue_type *type = queue->type;
+	union gve_admin_command *cmd;
+	unsigned int db_off;
+	unsigned int evt_idx;
+	int rc;
+
+	/* Reset queue */
+	queue->prod = 0;
+	queue->cons = 0;
+
+	/* Construct request */
+	cmd = gve_admin_command ( gve );
+	cmd->hdr.opcode = type->create;
+	type->param ( queue, cmd );
+
+	/* Issue command */
+	if ( ( rc = gve_admin ( gve ) ) != 0 )
+		return rc;
+
+	/* Record indices */
+	db_off = ( be32_to_cpu ( queue->res->db_idx ) * sizeof ( uint32_t ) );
+	evt_idx = be32_to_cpu ( queue->res->evt_idx );
+	DBGC ( gve, "GVE %p %s doorbell +%#04x event counter %d\n",
+	       gve, type->name, db_off, evt_idx );
+	queue->db = ( gve->db + db_off );
+	assert ( evt_idx < gve->events.count );
+	queue->event = &gve->events.event[evt_idx];
+	assert ( queue->event->count == 0 );
+
+	return 0;
+}
+
+/**
+ * Destroy transmit or receive queue
+ *
+ * @v gve		GVE device
+ * @v queue		Descriptor queue
+ * @ret rc		Return status code
+ */
+static int gve_destroy_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
+	const struct gve_queue_type *type = queue->type;
+	int rc;
+
+	/* Issue command */
+	if ( ( rc = gve_admin_simple ( gve, type->destroy, 0 ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate queue page list
+ *
+ * @v gve		GVE device
+ * @v qpl		Queue page list
+ * @v id		Queue page list ID
+ * @v buffers		Number of data buffers
+ * @ret rc		Return status code
+ */
+static int gve_alloc_qpl ( struct gve_nic *gve, struct gve_qpl *qpl,
+			   uint32_t id, unsigned int buffers ) {
+	size_t len;
+
+	/* Record ID */
+	qpl->id = id;
+
+	/* Calculate number of pages required */
+	build_assert ( GVE_BUF_SIZE <= GVE_PAGE_SIZE );
+	qpl->count = ( ( buffers + GVE_BUF_PER_PAGE - 1 ) / GVE_BUF_PER_PAGE );
+
+	/* Allocate pages (as a single block) */
+	len = ( qpl->count * GVE_PAGE_SIZE );
+	qpl->data = dma_umalloc ( gve->dma, &qpl->map, len, GVE_ALIGN );
+	if ( ! qpl->data )
+		return -ENOMEM;
+
+	DBGC ( gve, "GVE %p QPL %#08x at [%08lx,%08lx)\n",
+	       gve, qpl->id, user_to_phys ( qpl->data, 0 ),
+	       user_to_phys ( qpl->data, len ) );
+	return 0;
+}
+
+/**
+ * Free queue page list
+ *
+ * @v gve		GVE device
+ * @v qpl		Queue page list
+ */
+static void gve_free_qpl ( struct gve_nic *nic __unused,
+			   struct gve_qpl *qpl ) {
+	size_t len = ( qpl->count * GVE_PAGE_SIZE );
+
+	/* Free pages */
+	dma_ufree ( &qpl->map, qpl->data, len );
+}
+
+/**
+ * Get buffer address (within queue page list address space)
+ *
+ * @v queue		Descriptor queue
+ * @v index		Buffer index
+ * @ret addr		Buffer address within queue page list address space
+ */
+static inline __attribute__ (( always_inline)) size_t
+gve_address ( struct gve_queue *queue, unsigned int index ) {
+
+	/* We allocate sufficient pages for the maximum fill level of
+	 * buffers, and reuse the pages in strict rotation as we
+	 * progress through the queue.
+	 */
+	return ( ( index & ( queue->fill - 1 ) ) * GVE_BUF_SIZE );
+}
+
+/**
+ * Get buffer address
+ *
+ * @v queue		Descriptor queue
+ * @v index		Buffer index
+ * @ret addr		Buffer address
+ */
+static inline __attribute__ (( always_inline )) userptr_t
+gve_buffer ( struct gve_queue *queue, unsigned int index ) {
+
+	/* Pages are currently allocated as a single contiguous block */
+	return userptr_add ( queue->qpl.data, gve_address ( queue, index ) );
+}
+
+/**
+ * Calculate next receive sequence number
+ *
+ * @v seq		Current sequence number, or zero to start sequence
+ * @ret next		Next sequence number
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+gve_next ( unsigned int seq ) {
+
+	/* The receive completion sequence number is a modulo 7
+	 * counter that cycles through the non-zero three-bit values 1
+	 * to 7 inclusive.
+	 *
+	 * Since 7 is coprime to 2^n, this ensures that the sequence
+	 * number changes each time that a new completion is written
+	 * to memory.
+	 *
+	 * Since the counter takes only non-zero values, this ensures
+	 * that the sequence number changes whenever a new completion
+	 * is first written to a zero-initialised completion ring.
+	 */
+	seq = ( ( seq + 1 ) & GVE_RX_SEQ_MASK );
+	return ( seq ? seq : 1 );
+}
+
+/**
+ * Allocate descriptor queue
+ *
+ * @v gve		GVE device
+ * @v queue		Descriptor queue
+ * @ret rc		Return status code
+ */
+static int gve_alloc_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
+	const struct gve_queue_type *type = queue->type;
+	struct dma_device *dma = gve->dma;
+	size_t desc_len = ( queue->count * type->desc_len );
+	size_t cmplt_len = ( queue->count * type->cmplt_len );
+	size_t res_len = sizeof ( *queue->res );
+	struct gve_buffer buf;
+	size_t offset;
+	unsigned int i;
+	int rc;
+
+	/* Sanity checks */
+	if ( ( queue->count == 0 ) ||
+	     ( queue->count & ( queue->count - 1 ) ) ) {
+		DBGC ( gve, "GVE %p %s invalid queue size %d\n",
+		       gve, type->name, queue->count );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+
+	/* Calculate maximum fill level */
+	assert ( ( type->fill & ( type->fill - 1 ) ) == 0 );
+	queue->fill = type->fill;
+	if ( queue->fill > queue->count )
+		queue->fill = queue->count;
+	DBGC ( gve, "GVE %p %s using QPL %#08x with %d/%d descriptors\n",
+	       gve, type->name, type->qpl, queue->fill, queue->count );
+
+	/* Allocate queue page list */
+	if ( ( rc = gve_alloc_qpl ( gve, &queue->qpl, type->qpl,
+				    queue->fill ) ) != 0 )
+		goto err_qpl;
+
+	/* Allocate descriptors */
+	queue->desc = dma_umalloc ( dma, &queue->desc_map, desc_len,
+				    GVE_ALIGN );
+	if ( ! queue->desc ) {
+		rc = -ENOMEM;
+		goto err_desc;
+	}
+	DBGC ( gve, "GVE %p %s descriptors at [%08lx,%08lx)\n",
+	       gve, type->name, user_to_phys ( queue->desc, 0 ),
+	       user_to_phys ( queue->desc, desc_len ) );
+
+	/* Allocate completions */
+	if ( cmplt_len ) {
+		queue->cmplt = dma_umalloc ( dma, &queue->cmplt_map, cmplt_len,
+					     GVE_ALIGN );
+		if ( ! queue->cmplt ) {
+			rc = -ENOMEM;
+			goto err_cmplt;
+		}
+		DBGC ( gve, "GVE %p %s completions at [%08lx,%08lx)\n",
+		       gve, type->name, user_to_phys ( queue->cmplt, 0 ),
+		       user_to_phys ( queue->cmplt, cmplt_len ) );
+	}
+
+	/* Allocate queue resources */
+	queue->res = dma_alloc ( dma, &queue->res_map, res_len, GVE_ALIGN );
+	if ( ! queue->res ) {
+		rc = -ENOMEM;
+		goto err_res;
+	}
+	memset ( queue->res, 0, res_len );
+
+	/* Populate descriptor offsets */
+	offset = ( type->desc_len - sizeof ( buf ) );
+	for ( i = 0 ; i < queue->count ; i++ ) {
+		buf.addr = cpu_to_be64 ( gve_address ( queue, i ) );
+		copy_to_user ( queue->desc, offset, &buf, sizeof ( buf ) );
+		offset += type->desc_len;
+	}
+
+	return 0;
+
+	dma_free ( &queue->res_map, queue->res, res_len );
+ err_res:
+	if ( cmplt_len )
+		dma_ufree ( &queue->cmplt_map, queue->cmplt, cmplt_len );
+ err_cmplt:
+	dma_ufree ( &queue->desc_map, queue->desc, desc_len );
+ err_desc:
+	gve_free_qpl ( gve, &queue->qpl );
+ err_qpl:
+ err_sanity:
+	return rc;
+}
+
+/**
+ * Free descriptor queue
+ *
+ * @v gve		GVE device
+ * @v queue		Descriptor queue
+ */
+static void gve_free_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
+	const struct gve_queue_type *type = queue->type;
+	size_t desc_len = ( queue->count * type->desc_len );
+	size_t cmplt_len = ( queue->count * type->cmplt_len );
+	size_t res_len = sizeof ( *queue->res );
+
+	/* Free queue resources */
+	dma_free ( &queue->res_map, queue->res, res_len );
+
+	/* Free completions, if applicable */
+	if ( cmplt_len )
+		dma_ufree ( &queue->cmplt_map, queue->cmplt, cmplt_len );
+
+	/* Free descriptors */
+	dma_ufree ( &queue->desc_map, queue->desc, desc_len );
+
+	/* Free queue page list */
+	gve_free_qpl ( gve, &queue->qpl );
+}
+
+/**
+ * Start up device
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_start ( struct gve_nic *gve ) {
+	struct net_device *netdev = gve->netdev;
+	struct gve_queue *tx = &gve->tx;
+	struct gve_queue *rx = &gve->rx;
+	struct io_buffer *iobuf;
+	unsigned int i;
+	int rc;
+
+	/* Cancel any pending transmissions */
+	for ( i = 0 ; i < ( sizeof ( gve->tx_iobuf ) /
+			    sizeof ( gve->tx_iobuf[0] ) ) ; i++ ) {
+		iobuf = gve->tx_iobuf[i];
+		gve->tx_iobuf[i] = NULL;
+		if ( iobuf )
+			netdev_tx_complete_err ( netdev, iobuf, -ECANCELED );
+	}
+
+	/* Invalidate receive completions */
+	memset_user ( rx->cmplt, 0, 0, ( rx->count * rx->type->cmplt_len ) );
+
+	/* Reset receive sequence */
+	gve->seq = gve_next ( 0 );
+
+	/* Configure device resources */
+	if ( ( rc = gve_configure ( gve ) ) != 0 )
+		goto err_configure;
+
+	/* Register transmit queue page list */
+	if ( ( rc = gve_register ( gve, &tx->qpl ) ) != 0 )
+		goto err_register_tx;
+
+	/* Register receive queue page list */
+	if ( ( rc = gve_register ( gve, &rx->qpl ) ) != 0 )
+		goto err_register_rx;
+
+	/* Create transmit queue */
+	if ( ( rc = gve_create_queue ( gve, tx ) ) != 0 )
+		goto err_create_tx;
+
+	/* Create receive queue */
+	if ( ( rc = gve_create_queue ( gve, rx ) ) != 0 )
+		goto err_create_rx;
+
+	return 0;
+
+	gve_destroy_queue ( gve, rx );
+ err_create_rx:
+	gve_destroy_queue ( gve, tx );
+ err_create_tx:
+	gve_unregister ( gve, &rx->qpl );
+ err_register_rx:
+	gve_unregister ( gve, &tx->qpl );
+ err_register_tx:
+	gve_deconfigure ( gve );
+ err_configure:
+	return rc;
+}
+
+/**
+ * Stop device
+ *
+ * @v gve		GVE device
+ */
+static void gve_stop ( struct gve_nic *gve ) {
+	struct gve_queue *tx = &gve->tx;
+	struct gve_queue *rx = &gve->rx;
+
+	/* Destroy queues */
+	gve_destroy_queue ( gve, rx );
+	gve_destroy_queue ( gve, tx );
+
+	/* Unregister page lists */
+	gve_unregister ( gve, &rx->qpl );
+	gve_unregister ( gve, &tx->qpl );
+
+	/* Deconfigure device */
+	gve_deconfigure ( gve );
+}
+
+/**
+ * Device startup process
+ *
+ * @v gve		GVE device
+ */
+static void gve_startup ( struct gve_nic *gve ) {
+	struct net_device *netdev = gve->netdev;
+	int rc;
+
+	/* Reset device */
+	if ( ( rc = gve_reset ( gve ) ) != 0 )
+		goto err_reset;
+
+	/* Enable admin queue */
+	gve_admin_enable ( gve );
+
+	/* Start device */
+	if ( ( rc = gve_start ( gve ) ) != 0 )
+		goto err_start;
+
+	/* Reset retry count */
+	gve->retries = 0;
+
+	/* (Ab)use link status to report startup status */
+	netdev_link_up ( netdev );
+
+	return;
+
+	gve_stop ( gve );
+ err_start:
+ err_reset:
+	DBGC ( gve, "GVE %p startup failed: %s\n", gve, strerror ( rc ) );
+	netdev_link_err ( netdev, rc );
+	if ( gve->retries++ < GVE_RESET_MAX_RETRY )
+		process_add ( &gve->startup );
+}
+
+/**
+ * Trigger startup process
+ *
+ * @v gve		GVE device
+ */
+static void gve_restart ( struct gve_nic *gve ) {
+	struct net_device *netdev = gve->netdev;
+
+	/* Mark link down to inhibit polling and transmit activity */
+	netdev_link_down ( netdev );
+
+	/* Schedule startup process */
+	process_add ( &gve->startup );
+}
+
+/**
+ * Reset recovery watchdog
+ *
+ * @v timer		Reset recovery watchdog timer
+ * @v over		Failure indicator
+ */
+static void gve_watchdog ( struct retry_timer *timer, int over __unused ) {
+	struct gve_nic *gve = container_of ( timer, struct gve_nic, watchdog );
+	uint32_t activity;
+	uint32_t pfn;
+	int rc;
+
+	/* Reschedule watchdog */
+	start_timer_fixed ( &gve->watchdog, GVE_WATCHDOG_TIMEOUT );
+
+	/* Reset device (for test purposes) if applicable */
+	if ( ( rc = inject_fault ( VM_MIGRATED_RATE ) ) != 0 ) {
+		DBGC ( gve, "GVE %p synthesising host reset\n", gve );
+		writel ( 0, gve->cfg + GVE_CFG_ADMIN_PFN );
+	}
+
+	/* Check for activity since last timer invocation */
+	activity = ( gve->tx.cons + gve->rx.cons );
+	if ( activity != gve->activity ) {
+		gve->activity = activity;
+		return;
+	}
+
+	/* Check for reset */
+	pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
+	if ( pfn ) {
+		DBGC2 ( gve, "GVE %p idle but not in reset\n", gve );
+		return;
+	}
+
+	/* Schedule restart */
+	DBGC ( gve, "GVE %p watchdog detected reset by host\n", gve );
+	gve_restart ( gve );
+}
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int gve_open ( struct net_device *netdev ) {
+	struct gve_nic *gve = netdev->priv;
+	struct gve_queue *tx = &gve->tx;
+	struct gve_queue *rx = &gve->rx;
+	int rc;
+
+	/* Allocate and prepopulate transmit queue */
+	if ( ( rc = gve_alloc_queue ( gve, tx ) ) != 0 )
+		goto err_alloc_tx;
+
+	/* Allocate and prepopulate receive queue */
+	if ( ( rc = gve_alloc_queue ( gve, rx ) ) != 0 )
+		goto err_alloc_rx;
+
+	/* Trigger startup */
+	gve_restart ( gve );
+
+	/* Start reset recovery watchdog timer */
+	start_timer_fixed ( &gve->watchdog, GVE_WATCHDOG_TIMEOUT );
+
+	return 0;
+
+	gve_free_queue ( gve, rx );
+ err_alloc_rx:
+	gve_free_queue ( gve, tx );
+ err_alloc_tx:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void gve_close ( struct net_device *netdev ) {
+	struct gve_nic *gve = netdev->priv;
+	struct gve_queue *tx = &gve->tx;
+	struct gve_queue *rx = &gve->rx;
+
+	/* Stop reset recovery timer */
+	stop_timer ( &gve->watchdog );
+
+	/* Terminate startup process */
+	process_del ( &gve->startup );
+
+	/* Stop and reset device */
+	gve_stop ( gve );
+	gve_reset ( gve );
+
+	/* Free queues */
+	gve_free_queue ( gve, rx );
+	gve_free_queue ( gve, tx );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int gve_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
+	struct gve_nic *gve = netdev->priv;
+	struct gve_queue *tx = &gve->tx;
+	struct gve_tx_descriptor desc;
+	unsigned int count;
+	unsigned int index;
+	size_t frag_len;
+	size_t offset;
+	size_t len;
+
+	/* Do nothing if queues are not yet set up */
+	if ( ! netdev_link_ok ( netdev ) )
+		return -ENETDOWN;
+
+	/* Defer packet if there is no space in the transmit ring */
+	len = iob_len ( iobuf );
+	count = ( ( len + GVE_BUF_SIZE - 1 ) / GVE_BUF_SIZE );
+	if ( ( ( tx->prod - tx->cons ) + count ) > tx->fill ) {
+		netdev_tx_defer ( netdev, iobuf );
+		return 0;
+	}
+
+	/* Copy packet to queue pages and populate descriptors */
+	for ( offset = 0 ; offset < len ; offset += frag_len ) {
+
+		/* Sanity check */
+		assert ( gve->tx_iobuf[ tx->prod % GVE_TX_FILL ] == NULL );
+
+		/* Copy packet fragment */
+		frag_len = ( len - offset );
+		if ( frag_len > GVE_BUF_SIZE )
+			frag_len = GVE_BUF_SIZE;
+		copy_to_user ( gve_buffer ( tx, tx->prod ), 0,
+			       ( iobuf->data + offset ), frag_len );
+
+		/* Populate descriptor */
+		index = ( tx->prod++ & ( tx->count - 1 ) );
+		memset ( &desc.pkt, 0, sizeof ( desc.pkt ) );
+		if ( offset ) {
+			desc.pkt.type = GVE_TX_TYPE_CONT;
+		} else {
+			desc.pkt.type = GVE_TX_TYPE_START;
+			desc.pkt.count = count;
+			desc.pkt.total = cpu_to_be16 ( len );
+		}
+		desc.pkt.len = cpu_to_be16 ( frag_len );
+		copy_to_user ( tx->desc, ( index * sizeof ( desc ) ), &desc,
+			       sizeof ( desc.pkt ) );
+		DBGC2 ( gve, "GVE %p TX %#04x %#02x:%#02x len %#04x/%#04x at "
+			"%#08zx\n", gve, index, desc.pkt.type, desc.pkt.count,
+			be16_to_cpu ( desc.pkt.len ),
+			be16_to_cpu ( desc.pkt.total ),
+			gve_address ( tx, index ) );
+	}
+	assert ( ( tx->prod - tx->cons ) <= tx->fill );
+
+	/* Record I/O buffer against final descriptor */
+	gve->tx_iobuf[ ( tx->prod - 1U ) % GVE_TX_FILL ] = iobuf;
+
+	/* Ring doorbell */
+	wmb();
+	writel ( bswap_32 ( tx->prod ), tx->db );
+
+	return 0;
+}
+
+/**
+ * Poll for completed transmissions
+ *
+ * @v netdev		Network device
+ */
+static void gve_poll_tx ( struct net_device *netdev ) {
+	struct gve_nic *gve = netdev->priv;
+	struct gve_queue *tx = &gve->tx;
+	struct io_buffer *iobuf;
+	uint32_t count;
+
+	/* Read event counter */
+	count = be32_to_cpu ( tx->event->count );
+
+	/* Process transmit completions */
+	while ( count != tx->cons ) {
+		DBGC2 ( gve, "GVE %p TX %#04x complete\n", gve, tx->cons );
+		iobuf = gve->tx_iobuf[ tx->cons % GVE_TX_FILL ];
+		gve->tx_iobuf[ tx->cons % GVE_TX_FILL ] = NULL;
+		tx->cons++;
+		if ( iobuf )
+			netdev_tx_complete ( netdev, iobuf );
+	}
+}
+
+/**
+ * Poll for received packets
+ *
+ * @v netdev		Network device
+ */
+static void gve_poll_rx ( struct net_device *netdev ) {
+	struct gve_nic *gve = netdev->priv;
+	struct gve_queue *rx = &gve->rx;
+	struct gve_rx_completion cmplt;
+	struct io_buffer *iobuf;
+	unsigned int index;
+	unsigned int seq;
+	uint32_t cons;
+	size_t offset;
+	size_t total;
+	size_t len;
+	int rc;
+
+	/* Process receive completions */
+	cons = rx->cons;
+	seq = gve->seq;
+	total = 0;
+	while ( 1 ) {
+
+		/* Read next possible completion */
+		index = ( cons++ & ( rx->count - 1 ) );
+		offset = ( ( index * sizeof ( cmplt ) ) +
+			   offsetof ( typeof ( cmplt ), pkt ) );
+		copy_from_user ( &cmplt.pkt, rx->cmplt, offset,
+				 sizeof ( cmplt.pkt ) );
+
+		/* Check sequence number */
+		if ( ( cmplt.pkt.seq & GVE_RX_SEQ_MASK ) != seq )
+			break;
+		seq = gve_next ( seq );
+
+		/* Parse completion */
+		len = be16_to_cpu ( cmplt.pkt.len );
+		DBGC2 ( gve, "GVE %p RX %#04x %#02x:%#02x len %#04zx at "
+			"%#08zx\n", gve, index, cmplt.pkt.seq, cmplt.pkt.flags,
+			len, gve_address ( rx, index ) );
+
+		/* Accumulate a complete packet */
+		if ( cmplt.pkt.flags & GVE_RXF_ERROR ) {
+			total = 0;
+		} else {
+			total += len;
+			if ( cmplt.pkt.flags & GVE_RXF_MORE )
+				continue;
+		}
+		gve->seq = seq;
+
+		/* Allocate and populate I/O buffer */
+		iobuf = ( total ? alloc_iob ( total ) : NULL );
+		for ( ; rx->cons != cons ; rx->cons++ ) {
+
+			/* Re-read completion length */
+			index = ( rx->cons & ( rx->count - 1 ) );
+			offset = ( ( index * sizeof ( cmplt ) ) +
+				   offsetof ( typeof ( cmplt ), pkt.len ) );
+			copy_from_user ( &cmplt.pkt, rx->cmplt, offset,
+					 sizeof ( cmplt.pkt.len ) );
+
+			/* Copy data */
+			if ( iobuf ) {
+				len = be16_to_cpu ( cmplt.pkt.len );
+				copy_from_user ( iob_put ( iobuf, len ),
+						 gve_buffer ( rx, rx->cons ),
+						 0, len );
+			}
+		}
+		assert ( ( iobuf == NULL ) || ( iob_len ( iobuf ) == total ) );
+		total = 0;
+
+		/* Hand off packet to network stack */
+		if ( iobuf ) {
+			iob_pull ( iobuf, GVE_RX_PAD );
+			netdev_rx ( netdev, iobuf );
+		} else {
+			rc = ( ( cmplt.pkt.flags & GVE_RXF_ERROR ) ?
+			       -EIO : -ENOMEM );
+			netdev_rx_err ( netdev, NULL, rc );
+		}
+
+		/* Sanity check */
+		assert ( rx->cons == cons );
+		assert ( gve->seq == seq );
+		assert ( total == 0 );
+	}
+}
+
+/**
+ * Refill receive queue
+ *
+ * @v netdev		Network device
+ */
+static void gve_refill_rx ( struct net_device *netdev ) {
+	struct gve_nic *gve = netdev->priv;
+	struct gve_queue *rx = &gve->rx;
+	unsigned int prod;
+
+	/* The receive descriptors are prepopulated at the time of
+	 * creating the receive queue (pointing to the preallocated
+	 * queue pages).  Refilling is therefore just a case of
+	 * ringing the doorbell if the device is not yet aware of any
+	 * available descriptors.
+	 */
+	prod = ( rx->cons + rx->fill );
+	if ( prod != rx->prod ) {
+		rx->prod = prod;
+		writel ( bswap_32 ( prod ), rx->db );
+		DBGC2 ( gve, "GVE %p RX %#04x ready\n", gve, rx->prod );
+	}
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void gve_poll ( struct net_device *netdev ) {
+
+	/* Do nothing if queues are not yet set up */
+	if ( ! netdev_link_ok ( netdev ) )
+		return;
+
+	/* Poll for transmit completions */
+	gve_poll_tx ( netdev );
+
+	/* Poll for receive completions */
+	gve_poll_rx ( netdev );
+
+	/* Refill receive queue */
+	gve_refill_rx ( netdev );
+}
+
+/** GVE network device operations */
+static struct net_device_operations gve_operations = {
+	.open		= gve_open,
+	.close		= gve_close,
+	.transmit	= gve_transmit,
+	.poll		= gve_poll,
+};
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/** Transmit descriptor queue type */
+static const struct gve_queue_type gve_tx_type = {
+	.name = "TX",
+	.param = gve_create_tx_param,
+	.qpl = GVE_TX_QPL,
+	.irq = GVE_TX_IRQ,
+	.fill = GVE_TX_FILL,
+	.desc_len = sizeof ( struct gve_tx_descriptor ),
+	.create = GVE_ADMIN_CREATE_TX,
+	.destroy = GVE_ADMIN_DESTROY_TX,
+};
+
+/** Receive descriptor queue type */
+static const struct gve_queue_type gve_rx_type = {
+	.name = "RX",
+	.param = gve_create_rx_param,
+	.qpl = GVE_RX_QPL,
+	.irq = GVE_RX_IRQ,
+	.fill = GVE_RX_FILL,
+	.desc_len = sizeof ( struct gve_rx_descriptor ),
+	.cmplt_len = sizeof ( struct gve_rx_completion ),
+	.create = GVE_ADMIN_CREATE_RX,
+	.destroy = GVE_ADMIN_DESTROY_RX,
+};
+
+/**
+ * Set up admin queue and get device description
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_setup ( struct gve_nic *gve ) {
+	unsigned int i;
+	int rc;
+
+	/* Attempt several times, since the device may decide to add
+	 * in a few spurious resets.
+	 */
+	for ( i = 0 ; i < GVE_RESET_MAX_RETRY ; i++ ) {
+
+		/* Reset device */
+		if ( ( rc = gve_reset ( gve ) ) != 0 )
+			continue;
+
+		/* Enable admin queue */
+		gve_admin_enable ( gve );
+
+		/* Fetch MAC address */
+		if ( ( rc = gve_describe ( gve ) ) != 0 )
+			continue;
+
+		/* Success */
+		return 0;
+	}
+
+	DBGC ( gve, "GVE %p failed to get device description: %s\n",
+	       gve, strerror ( rc ) );
+	return rc;
+}
+
+/** Device startup process descriptor */
+static struct process_descriptor gve_startup_desc =
+	PROC_DESC_ONCE ( struct gve_nic, startup, gve_startup );
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int gve_probe ( struct pci_device *pci ) {
+	struct net_device *netdev;
+	struct gve_nic *gve;
+	unsigned long cfg_start;
+	unsigned long db_start;
+	unsigned long db_size;
+	int rc;
+
+	/* Allocate and initialise net device */
+	netdev = alloc_etherdev ( sizeof ( *gve ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &gve_operations );
+	gve = netdev->priv;
+	pci_set_drvdata ( pci, netdev );
+	netdev->dev = &pci->dev;
+	memset ( gve, 0, sizeof ( *gve ) );
+	gve->netdev = netdev;
+	gve->tx.type = &gve_tx_type;
+	gve->rx.type = &gve_rx_type;
+	process_init ( &gve->startup, &gve_startup_desc, &netdev->refcnt );
+	timer_init ( &gve->watchdog, gve_watchdog, &netdev->refcnt );
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Check PCI revision */
+	pci_read_config_byte ( pci, PCI_REVISION, &gve->revision );
+	DBGC ( gve, "GVE %p is revision %#02x\n", gve, gve->revision );
+
+	/* Map configuration registers */
+	cfg_start = pci_bar_start ( pci, GVE_CFG_BAR );
+	gve->cfg = pci_ioremap ( pci, cfg_start, GVE_CFG_SIZE );
+	if ( ! gve->cfg ) {
+		rc = -ENODEV;
+		goto err_cfg;
+	}
+
+	/* Map doorbell registers */
+	db_start = pci_bar_start ( pci, GVE_DB_BAR );
+	db_size = pci_bar_size ( pci, GVE_DB_BAR );
+	gve->db = pci_ioremap ( pci, db_start, db_size );
+	if ( ! gve->db ) {
+		rc = -ENODEV;
+		goto err_db;
+	}
+
+	/* Configure DMA */
+	gve->dma = &pci->dma;
+	dma_set_mask_64bit ( gve->dma );
+	assert ( netdev->dma == NULL );
+
+	/* Allocate admin queue */
+	if ( ( rc = gve_admin_alloc ( gve ) ) != 0 )
+		goto err_admin;
+
+	/* Set up the device */
+	if ( ( rc = gve_setup ( gve ) ) != 0 )
+		goto err_setup;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register_netdev;
+
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register_netdev:
+ err_setup:
+	gve_reset ( gve );
+	gve_admin_free ( gve );
+ err_admin:
+	iounmap ( gve->db );
+ err_db:
+	iounmap ( gve->cfg );
+ err_cfg:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void gve_remove ( struct pci_device *pci ) {
+	struct net_device *netdev = pci_get_drvdata ( pci );
+	struct gve_nic *gve = netdev->priv;
+
+	/* Unregister network device */
+	unregister_netdev ( netdev );
+
+	/* Reset device */
+	gve_reset ( gve );
+
+	/* Free admin queue */
+	gve_admin_free ( gve );
+
+	/* Unmap registers */
+	iounmap ( gve->db );
+	iounmap ( gve->cfg );
+
+	/* Free network device */
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** GVE PCI device IDs */
+static struct pci_device_id gve_nics[] = {
+	PCI_ROM ( 0x1ae0, 0x0042, "gve", "gVNIC", 0 ),
+};
+
+/** GVE PCI driver */
+struct pci_driver gve_driver __pci_driver = {
+	.ids = gve_nics,
+	.id_count = ( sizeof ( gve_nics ) / sizeof ( gve_nics[0] ) ),
+	.probe = gve_probe,
+	.remove = gve_remove,
+};
diff --git a/src/drivers/net/gve.h b/src/drivers/net/gve.h
new file mode 100644
index 000000000..2845699ac
--- /dev/null
+++ b/src/drivers/net/gve.h
@@ -0,0 +1,702 @@
+#ifndef _GVE_H
+#define _GVE_H
+
+/** @file
+ *
+ * Google Virtual Ethernet network driver
+ *
+ * The Google Virtual Ethernet NIC (GVE or gVNIC) is found only in
+ * Google Cloud instances.  There is essentially zero documentation
+ * available beyond the mostly uncommented source code in the Linux
+ * kernel.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/dma.h>
+#include <ipxe/pci.h>
+#include <ipxe/in.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/process.h>
+#include <ipxe/retry.h>
+
+struct gve_nic;
+
+/**
+ * A Google Cloud MAC address
+ *
+ * Google Cloud locally assigned MAC addresses encode the local IPv4
+ * address in the trailing 32 bits, presumably as a performance
+ * optimisation to allow ARP resolution to be skipped by a suitably
+ * aware network stack.
+ */
+struct google_mac {
+	/** Reserved */
+	uint8_t reserved[2];
+	/** Local IPv4 address */
+	struct in_addr in;
+} __attribute__ (( packed ));
+
+/** Page size */
+#define GVE_PAGE_SIZE 0x1000
+
+/**
+ * Address alignment
+ *
+ * All DMA data structure base addresses seem to need to be aligned to
+ * a page boundary.  (This is not documented anywhere, but is inferred
+ * from existing source code and experimentation.)
+ */
+#define GVE_ALIGN GVE_PAGE_SIZE
+
+/**
+ * Length alignment
+ *
+ * All DMA data structure lengths seem to need to be aligned to a
+ * multiple of 64 bytes.  (This is not documented anywhere, but is
+ * inferred from existing source code and experimentation.)
+ */
+#define GVE_LEN_ALIGN 64
+
+/** Maximum number of pages per queue (must be a power of two) */
+#define GVE_QPL_MAX 16
+
+/** Configuration BAR */
+#define GVE_CFG_BAR PCI_BASE_ADDRESS_0
+
+/**
+ * Configuration BAR size
+ *
+ * All registers within the configuration BAR are big-endian.
+ */
+#define GVE_CFG_SIZE 0x1000
+
+/** Device status */
+#define GVE_CFG_DEVSTAT 0x0000
+#define GVE_CFG_DEVSTAT_RESET 0x00000010UL	/**< Device is reset */
+
+/** Driver status */
+#define GVE_CFG_DRVSTAT 0x0004
+#define GVE_CFG_DRVSTAT_RUN 0x00000001UL	/**< Run admin queue */
+
+/** Maximum time to wait for reset */
+#define GVE_RESET_MAX_WAIT_MS 500
+
+/** Admin queue page frame number (for older devices) */
+#define GVE_CFG_ADMIN_PFN 0x0010
+
+/** Admin queue doorbell */
+#define GVE_CFG_ADMIN_DB 0x0014
+
+/** Admin queue event counter */
+#define GVE_CFG_ADMIN_EVT 0x0018
+
+/** Driver version (8-bit register) */
+#define GVE_CFG_VERSION 0x001f
+
+/** Admin queue base address high 32 bits */
+#define GVE_CFG_ADMIN_BASE_HI 0x0020
+
+/** Admin queue base address low 32 bits */
+#define GVE_CFG_ADMIN_BASE_LO 0x0024
+
+/** Admin queue base address length (16-bit register) */
+#define GVE_CFG_ADMIN_LEN 0x0028
+
+/** Doorbell BAR */
+#define GVE_DB_BAR PCI_BASE_ADDRESS_2
+
+/**
+ * Admin queue entry header
+ *
+ * All values within admin queue entries are big-endian.
+ */
+struct gve_admin_header {
+	/** Reserved */
+	uint8_t reserved[3];
+	/** Operation code */
+	uint8_t opcode;
+	/** Status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** Command succeeded */
+#define GVE_ADMIN_STATUS_OK 0x00000001
+
+/** Simple admin command */
+struct gve_admin_simple {
+	/** Header */
+	struct gve_admin_header hdr;
+	/** ID */
+	uint32_t id;
+} __attribute__ (( packed ));
+
+/** Describe device command */
+#define GVE_ADMIN_DESCRIBE 0x0001
+
+/** Describe device command */
+struct gve_admin_describe {
+	/** Header */
+	struct gve_admin_header hdr;
+	/** Descriptor buffer address */
+	uint64_t addr;
+	/** Descriptor version */
+	uint32_t ver;
+	/** Descriptor maximum length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** Device descriptor version */
+#define GVE_ADMIN_DESCRIBE_VER 1
+
+/** Device descriptor */
+struct gve_device_descriptor {
+	/** Reserved */
+	uint8_t reserved_a[10];
+	/** Number of transmit queue entries */
+	uint16_t tx_count;
+	/** Number of receive queue entries */
+	uint16_t rx_count;
+	/** Reserved */
+	uint8_t reserved_b[2];
+	/** Maximum transmit unit */
+	uint16_t mtu;
+	/** Number of event counters */
+	uint16_t counters;
+	/** Reserved */
+	uint8_t reserved_c[4];
+	/** MAC address */
+	struct google_mac mac;
+	/** Reserved */
+	uint8_t reserved_d[10];
+} __attribute__ (( packed ));
+
+/** Configure device resources command */
+#define GVE_ADMIN_CONFIGURE 0x0002
+
+/** Configure device resources command */
+struct gve_admin_configure {
+	/** Header */
+	struct gve_admin_header hdr;
+	/** Event counter array */
+	uint64_t events;
+	/** IRQ doorbell address */
+	uint64_t irqs;
+	/** Number of event counters */
+	uint32_t num_events;
+	/** Number of IRQ doorbells */
+	uint32_t num_irqs;
+	/** IRQ doorbell stride */
+	uint32_t irq_stride;
+} __attribute__ (( packed ));
+
+/** Register page list command */
+#define GVE_ADMIN_REGISTER 0x0003
+
+/** Register page list command */
+struct gve_admin_register {
+	/** Header */
+	struct gve_admin_header hdr;
+	/** Page list ID */
+	uint32_t id;
+	/** Number of pages */
+	uint32_t count;
+	/** Address list address */
+	uint64_t addr;
+	/** Page size */
+	uint64_t size;
+} __attribute__ (( packed ));
+
+/** Page list */
+struct gve_pages {
+	/** Page address */
+	uint64_t addr[GVE_QPL_MAX];
+} __attribute__ (( packed ));
+
+/** Unregister page list command */
+#define GVE_ADMIN_UNREGISTER 0x0004
+
+/** Create transmit queue command */
+#define GVE_ADMIN_CREATE_TX 0x0005
+
+/** Create transmit queue command */
+struct gve_admin_create_tx {
+	/** Header */
+	struct gve_admin_header hdr;
+	/** Queue ID */
+	uint32_t id;
+	/** Reserved */
+	uint8_t reserved_a[4];
+	/** Queue resources address */
+	uint64_t res;
+	/** Descriptor ring address */
+	uint64_t desc;
+	/** Queue page list ID */
+	uint32_t qpl_id;
+	/** Notification channel ID */
+	uint32_t notify_id;
+} __attribute__ (( packed ));
+
+/** Create receive queue command */
+#define GVE_ADMIN_CREATE_RX 0x0006
+
+/** Create receive queue command */
+struct gve_admin_create_rx {
+	/** Header */
+	struct gve_admin_header hdr;
+	/** Queue ID */
+	uint32_t id;
+	/** Index */
+	uint32_t index;
+	/** Reserved */
+	uint8_t reserved_a[4];
+	/** Notification channel ID */
+	uint32_t notify_id;
+	/** Queue resources address */
+	uint64_t res;
+	/** Completion ring address */
+	uint64_t cmplt;
+	/** Descriptor ring address */
+	uint64_t desc;
+	/** Queue page list ID */
+	uint32_t qpl_id;
+	/** Reserved */
+	uint8_t reserved_b[2];
+	/** Packet buffer size */
+	uint16_t bufsz;
+} __attribute__ (( packed ));
+
+/** Destroy transmit queue command */
+#define GVE_ADMIN_DESTROY_TX 0x0007
+
+/** Destroy receive queue command */
+#define GVE_ADMIN_DESTROY_RX 0x0008
+
+/** Deconfigure device resources command */
+#define GVE_ADMIN_DECONFIGURE 0x0009
+
+/** An admin queue command */
+union gve_admin_command {
+	/** Header */
+	struct gve_admin_header hdr;
+	/** Simple command */
+	struct gve_admin_simple simple;
+	/** Describe device */
+	struct gve_admin_describe desc;
+	/** Configure device resources */
+	struct gve_admin_configure conf;
+	/** Register page list */
+	struct gve_admin_register reg;
+	/** Create transmit queue */
+	struct gve_admin_create_tx create_tx;
+	/** Create receive queue */
+	struct gve_admin_create_rx create_rx;
+	/** Padding */
+	uint8_t pad[64];
+};
+
+/**
+ * Number of admin queue commands
+ *
+ * This is theoretically a policy decision.  However, older revisions
+ * of the hardware seem to have only the "admin queue page frame
+ * number" register and no "admin queue length" register, with the
+ * implication that the admin queue must be exactly one page in
+ * length.
+ *
+ * Choose to use a one page (4kB) admin queue for both older and newer
+ * versions of the hardware, to minimise variability.
+ */
+#define GVE_ADMIN_COUNT ( GVE_PAGE_SIZE / sizeof ( union gve_admin_command ) )
+
+/** Admin queue */
+struct gve_admin {
+	/** Commands */
+	union gve_admin_command *cmd;
+	/** Producer counter */
+	uint32_t prod;
+	/** DMA mapping */
+	struct dma_mapping map;
+};
+
+/** Scratch buffer for admin queue commands */
+struct gve_scratch {
+	/** Buffer contents */
+	union {
+		/** Device descriptor */
+		struct gve_device_descriptor desc;
+		/** Page address list */
+		struct gve_pages pages;
+	} *buf;
+	/** DMA mapping */
+	struct dma_mapping map;
+};
+
+/**
+ * An event counter
+ *
+ * Written by the device to indicate completions.  The device chooses
+ * which counter to use for each transmit queue, and stores the index
+ * of the chosen counter in the queue resources.
+ */
+struct gve_event {
+	/** Number of events that have occurred */
+	volatile uint32_t count;
+} __attribute__ (( packed ));
+
+/**
+ * Maximum number of event counters
+ *
+ * We tell the device how many event counters we have provided via the
+ * "configure device resources" admin queue command.  The device will
+ * accept being given only a single counter, but will subsequently
+ * fail to create a receive queue.
+ *
+ * There is, of course, no documentation indicating how may event
+ * counters actually need to be provided.  In the absence of evidence
+ * to the contrary, assume that 16 counters (i.e. the smallest number
+ * we can allocate, given the length alignment constraint on
+ * allocations) will be sufficient.
+ */
+#define GVE_EVENT_MAX ( GVE_LEN_ALIGN / sizeof ( struct gve_event ) )
+
+/** Event counter array */
+struct gve_events {
+	/** Event counters */
+	struct gve_event *event;
+	/** DMA mapping */
+	struct dma_mapping map;
+	/** Actual number of event counters */
+	unsigned int count;
+};
+
+/** An interrupt channel */
+struct gve_irq {
+	/** Interrupt doorbell index (within doorbell BAR) */
+	uint32_t db_idx;
+	/** Reserved */
+	uint8_t reserved[60];
+} __attribute__ (( packed ));
+
+/**
+ * Number of interrupt channels
+ *
+ * We tell the device how many interrupt channels we have provided via
+ * the "configure device resources" admin queue command.  The device
+ * will accept being given zero interrupt channels, but will
+ * subsequently fail to create more than a single queue (either
+ * transmit or receive).
+ *
+ * There is, of course, no documentation indicating how may interrupt
+ * channels actually need to be provided.  In the absence of evidence
+ * to the contrary, assume that two channels (one for transmit, one
+ * for receive) will be sufficient.
+ */
+#define GVE_IRQ_COUNT 2
+
+/** Interrupt channel array */
+struct gve_irqs {
+	/** Interrupt channels */
+	struct gve_irq *irq;
+	/** DMA mapping */
+	struct dma_mapping map;
+	/** Interrupt doorbells */
+	volatile uint32_t *db[GVE_IRQ_COUNT];
+};
+
+/** Disable interrupts */
+#define GVE_IRQ_DISABLE 0x40000000UL
+
+/**
+ * Queue resources
+ *
+ * Written by the device to indicate the indices of the chosen event
+ * counter and descriptor doorbell register.
+ *
+ * This appears to be a largely pointless data structure: the relevant
+ * information is static for the lifetime of the queue and could
+ * trivially have been returned in the response for the "create
+ * transmit/receive queue" command, instead of requiring yet another
+ * page-aligned coherent DMA buffer allocation.
+ */
+struct gve_resources {
+	/** Descriptor doorbell index (within doorbell BAR) */
+	uint32_t db_idx;
+	/** Event counter index (within event counter array) */
+	uint32_t evt_idx;
+	/** Reserved */
+	uint8_t reserved[56];
+} __attribute__ (( packed ));
+
+/**
+ * Queue data buffer size
+ *
+ * In theory, we may specify the size of receive buffers.  However,
+ * the original version of the device seems not to have a parameter
+ * for this, and assumes the use of half-page (2kB) buffers.  Choose
+ * to use this as the buffer size, on the assumption that older
+ * devices will not support any other buffer size.
+ */
+#define GVE_BUF_SIZE ( GVE_PAGE_SIZE / 2 )
+
+/** Number of data buffers per page */
+#define GVE_BUF_PER_PAGE ( GVE_PAGE_SIZE / GVE_BUF_SIZE )
+
+/**
+ * Queue page list
+ *
+ * The device uses preregistered pages for fast-path DMA operations
+ * (i.e. transmit and receive buffers).  A list of device addresses
+ * for each page must be registered before the transmit or receive
+ * queue is created, and cannot subsequently be modified.
+ *
+ * The Linux driver allocates pages as DMA_TO_DEVICE or
+ * DMA_FROM_DEVICE as appropriate, and uses dma_sync_single_for_cpu()
+ * etc to ensure that data is copied to/from bounce buffers as needed.
+ *
+ * Unfortunately there is no such sync operation available within our
+ * DMA API, since we are constrained by the limitations imposed by
+ * EFI_PCI_IO_PROTOCOL.  There is no way to synchronise a buffer
+ * without also [un]mapping it, and no way to force the reuse of the
+ * same device address for a subsequent remapping.  We are therefore
+ * constrained to use only DMA-coherent buffers, since this is the
+ * only way we can repeatedly reuse the same device address.
+ *
+ * Newer versions of the gVNIC device support "raw DMA addressing
+ * (RDA)", which is essentially a prebuilt queue page list covering
+ * the whole of the guest address space.  Unfortunately we cannot rely
+ * on this, since older versions will not support it.
+ *
+ * Experimentation suggests that the device will accept a request to
+ * create a queue page list covering the whole of the guest address
+ * space via two giant "pages" of 2^63 bytes each.  However,
+ * experimentation also suggests that the device will accept any old
+ * garbage value as the "page size".  In the total absence of any
+ * documentation, it is probably unsafe to conclude that the device is
+ * bothering to look at or respect the "page size" parameter: it is
+ * most likely just presuming the use of 4kB pages.
+ */
+struct gve_qpl {
+	/** Page addresses */
+	userptr_t data;
+	/** Page mapping */
+	struct dma_mapping map;
+	/** Number of pages */
+	unsigned int count;
+	/** Queue page list ID */
+	unsigned int id;
+};
+
+/**
+ * Maximum number of transmit buffers
+ *
+ * This is a policy decision.
+ */
+#define GVE_TX_FILL 8
+
+/** Transmit queue page list ID */
+#define GVE_TX_QPL 0x18ae5458
+
+/** Tranmsit queue interrupt channel */
+#define GVE_TX_IRQ 0
+
+/** A transmit or receive buffer descriptor */
+struct gve_buffer {
+	/** Address (within queue page list address space) */
+	uint64_t addr;
+} __attribute__ (( packed ));
+
+/** A transmit packet descriptor */
+struct gve_tx_packet {
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint8_t reserved_a[2];
+	/** Number of descriptors in this packet */
+	uint8_t count;
+	/** Total length of this packet */
+	uint16_t total;
+	/** Length of this descriptor */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** A transmit descriptor */
+struct gve_tx_descriptor {
+	/** Packet descriptor */
+	struct gve_tx_packet pkt;
+	/** Buffer descriptor */
+	struct gve_buffer buf;
+} __attribute__ (( packed ));
+
+/** Start of packet transmit descriptor type */
+#define GVE_TX_TYPE_START 0x00
+
+/** Continuation of packet transmit descriptor type */
+#define GVE_TX_TYPE_CONT 0x20
+
+/**
+ * Maximum number of receive buffers
+ *
+ * This is a policy decision.
+ */
+#define GVE_RX_FILL 16
+
+/** Receive queue page list ID */
+#define GVE_RX_QPL 0x18ae5258
+
+/** Receive queue interrupt channel */
+#define GVE_RX_IRQ 1
+
+/** A receive descriptor */
+struct gve_rx_descriptor {
+	/** Buffer descriptor */
+	struct gve_buffer buf;
+} __attribute__ (( packed ));
+
+/** A receive packet descriptor */
+struct gve_rx_packet {
+	/** Length */
+	uint16_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Sequence number */
+	uint8_t seq;
+} __attribute__ (( packed ));
+
+/** Receive error */
+#define GVE_RXF_ERROR 0x08
+
+/** Receive packet continues into next descriptor */
+#define GVE_RXF_MORE 0x20
+
+/** Receive sequence number mask */
+#define GVE_RX_SEQ_MASK 0x07
+
+/** A receive completion descriptor */
+struct gve_rx_completion {
+	/** Reserved */
+	uint8_t reserved[60];
+	/** Packet descriptor */
+	struct gve_rx_packet pkt;
+} __attribute__ (( packed ));
+
+/** Padding at the start of all received packets */
+#define GVE_RX_PAD 2
+
+/** A descriptor queue */
+struct gve_queue {
+	/** Descriptor ring */
+	userptr_t desc;
+	/** Completion ring */
+	userptr_t cmplt;
+	/** Queue resources */
+	struct gve_resources *res;
+
+	/** Queue type */
+	const struct gve_queue_type *type;
+	/** Number of descriptors (must be a power of two) */
+	unsigned int count;
+	/** Maximum fill level (must be a power of two) */
+	unsigned int fill;
+
+	/** Descriptor mapping */
+	struct dma_mapping desc_map;
+	/** Completion mapping */
+	struct dma_mapping cmplt_map;
+	/** Queue resources mapping */
+	struct dma_mapping res_map;
+
+	/** Doorbell register */
+	volatile uint32_t *db;
+	/** Event counter */
+	struct gve_event *event;
+
+	/** Producer counter */
+	uint32_t prod;
+	/** Consumer counter */
+	uint32_t cons;
+
+	/** Queue page list */
+	struct gve_qpl qpl;
+};
+
+/** A descriptor queue type */
+struct gve_queue_type {
+	/** Name */
+	const char *name;
+	/**
+	 * Populate command parameters to create queue
+	 *
+	 * @v queue		Descriptor queue
+	 * @v cmd		Admin queue command
+	 */
+	void ( * param ) ( struct gve_queue *queue,
+			   union gve_admin_command *cmd );
+	/** Queue page list ID */
+	uint32_t qpl;
+	/** Interrupt channel */
+	uint8_t irq;
+	/** Maximum fill level */
+	uint8_t fill;
+	/** Descriptor size */
+	uint8_t desc_len;
+	/** Completion size */
+	uint8_t cmplt_len;
+	/** Command to create queue */
+	uint8_t create;
+	/** Command to destroy queue */
+	uint8_t destroy;
+};
+
+/** A Google Virtual Ethernet NIC */
+struct gve_nic {
+	/** Configuration registers */
+	void *cfg;
+	/** Doorbell registers */
+	void *db;
+	/** PCI revision */
+	uint8_t revision;
+	/** Network device */
+	struct net_device *netdev;
+	/** DMA device */
+	struct dma_device *dma;
+
+	/** Admin queue */
+	struct gve_admin admin;
+	/** Interrupt channels */
+	struct gve_irqs irqs;
+	/** Event counters */
+	struct gve_events events;
+	/** Scratch buffer */
+	struct gve_scratch scratch;
+
+	/** Transmit queue */
+	struct gve_queue tx;
+	/** Receive queue */
+	struct gve_queue rx;
+	/** Transmit I/O buffers */
+	struct io_buffer *tx_iobuf[GVE_TX_FILL];
+	/** Receive sequence number */
+	unsigned int seq;
+
+	/** Startup process */
+	struct process startup;
+	/** Startup process retry counter */
+	unsigned int retries;
+	/** Reset recovery watchdog timer */
+	struct retry_timer watchdog;
+	/** Reset recovery recorded activity counter */
+	uint32_t activity;
+};
+
+/** Maximum time to wait for admin queue commands */
+#define GVE_ADMIN_MAX_WAIT_MS 500
+
+/** Maximum number of times to reattempt device reset */
+#define GVE_RESET_MAX_RETRY 5
+
+/** Time between reset recovery checks */
+#define GVE_WATCHDOG_TIMEOUT ( 1 * TICKS_PER_SEC )
+
+#endif /* _GVE_H */
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index fcb4f0e68..7615075bc 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -224,6 +224,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_ecam		     ( ERRFILE_DRIVER | 0x00d30000 )
 #define ERRFILE_pcibridge	     ( ERRFILE_DRIVER | 0x00d40000 )
 #define ERRFILE_mnpnet		     ( ERRFILE_DRIVER | 0x00d50000 )
+#define ERRFILE_gve		     ( ERRFILE_DRIVER | 0x00d60000 )
 
 #define ERRFILE_aoe			( ERRFILE_NET | 0x00000000 )
 #define ERRFILE_arp			( ERRFILE_NET | 0x00010000 )

From d2d194bc60f012569fa95ed54693cb6663beb5ce Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 25 Jul 2024 00:10:38 +0100
Subject: [PATCH 155/237] [gve] Increase number of receive buffers to reduce
 packet loss

Experiments suggest that using fewer than 64 receive buffers leads to
excessive packet drop rates on some instance types (observed with a
c3-standard-4 instance in europe-west4-a).

Fix by increasing the number of receive data buffers (and adjusting
the length of the registrable queue page address list to match).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/gve.c |  1 +
 src/drivers/net/gve.h | 17 ++++++++++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/drivers/net/gve.c b/src/drivers/net/gve.c
index 03edc0899..7f86dd219 100644
--- a/src/drivers/net/gve.c
+++ b/src/drivers/net/gve.c
@@ -722,6 +722,7 @@ static int gve_alloc_qpl ( struct gve_nic *gve, struct gve_qpl *qpl,
 	/* Calculate number of pages required */
 	build_assert ( GVE_BUF_SIZE <= GVE_PAGE_SIZE );
 	qpl->count = ( ( buffers + GVE_BUF_PER_PAGE - 1 ) / GVE_BUF_PER_PAGE );
+	assert ( qpl->count <= GVE_QPL_MAX );
 
 	/* Allocate pages (as a single block) */
 	len = ( qpl->count * GVE_PAGE_SIZE );
diff --git a/src/drivers/net/gve.h b/src/drivers/net/gve.h
index 2845699ac..247d6e699 100644
--- a/src/drivers/net/gve.h
+++ b/src/drivers/net/gve.h
@@ -59,9 +59,6 @@ struct google_mac {
  */
 #define GVE_LEN_ALIGN 64
 
-/** Maximum number of pages per queue (must be a power of two) */
-#define GVE_QPL_MAX 16
-
 /** Configuration BAR */
 #define GVE_CFG_BAR PCI_BASE_ADDRESS_0
 
@@ -208,6 +205,14 @@ struct gve_admin_register {
 	uint64_t size;
 } __attribute__ (( packed ));
 
+/**
+ * Maximum number of pages per queue
+ *
+ * This is a policy decision.  Must be sufficient to allow for both
+ * the transmit and receive queue fill levels.
+ */
+#define GVE_QPL_MAX 32
+
 /** Page list */
 struct gve_pages {
 	/** Page address */
@@ -538,9 +543,11 @@ struct gve_tx_descriptor {
 /**
  * Maximum number of receive buffers
  *
- * This is a policy decision.
+ * This is a policy decision.  Experiments suggest that using fewer
+ * than 64 receive buffers leads to excessive packet drop rates on
+ * some instance types.
  */
-#define GVE_RX_FILL 16
+#define GVE_RX_FILL 64
 
 /** Receive queue page list ID */
 #define GVE_RX_QPL 0x18ae5258

From 0dc8933f6769e375a645ed4e9053855191cf8170 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 30 Jul 2024 15:40:11 +0100
Subject: [PATCH 156/237] [cloud] Add utility to read INT13CON partition in
 Google Compute Engine

Following the example of aws-int13con, add a utility that can be used
to read the INT13 console log from a used iPXE boot disk in Google
Compute Engine.

There seems to be no easy way to directly read the contents of either
a disk image or a snapshot in Google Cloud.  Work around this
limitation by creating a snapshot and attaching this snapshot as a
data disk to a temporary Linux instance, which is then used to echo
the INT13 console log to the serial port.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 contrib/cloud/gce-int13con | 146 +++++++++++++++++++++++++++++++++++++
 1 file changed, 146 insertions(+)
 create mode 100755 contrib/cloud/gce-int13con

diff --git a/contrib/cloud/gce-int13con b/contrib/cloud/gce-int13con
new file mode 100755
index 000000000..3b909a44a
--- /dev/null
+++ b/contrib/cloud/gce-int13con
@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+import argparse
+import textwrap
+import time
+from uuid import uuid4
+
+from google.cloud import compute
+
+IPXE_LOG_PREFIX = 'ipxe-log-temp-'
+IPXE_LOG_MAGIC = 'iPXE LOG'
+IPXE_LOG_END = '----- END OF iPXE LOG -----'
+
+def get_log_disk(instances, project, zone, name):
+    """Get log disk source URL"""
+    instance = instances.get(project=project, zone=zone, instance=name)
+    disk = next(x for x in instance.disks if x.boot)
+    return disk.source
+
+def delete_temp_snapshot(snapshots, project, name):
+    """Delete temporary snapshot"""
+    assert name.startswith(IPXE_LOG_PREFIX)
+    snapshots.delete(project=project, snapshot=name)
+
+def delete_temp_snapshots(snapshots, project):
+    """Delete all old temporary snapshots"""
+    filter = "name eq %s.+" % IPXE_LOG_PREFIX
+    request = compute.ListSnapshotsRequest(project=project, filter=filter)
+    for snapshot in snapshots.list(request=request):
+        delete_temp_snapshot(snapshots, project, snapshot.name)
+
+def create_temp_snapshot(snapshots, project, source):
+    """Create temporary snapshot"""
+    name = '%s%s' % (IPXE_LOG_PREFIX, uuid4())
+    snapshot = compute.Snapshot(name=name, source_disk=source)
+    snapshots.insert(project=project, snapshot_resource=snapshot).result()
+    return name
+
+def delete_temp_instance(instances, project, zone, name):
+    """Delete log dumper temporary instance"""
+    assert name.startswith(IPXE_LOG_PREFIX)
+    instances.delete(project=project, zone=zone, instance=name)
+
+def delete_temp_instances(instances, project, zone):
+    """Delete all old log dumper temporary instances"""
+    filter = "name eq %s.+" % IPXE_LOG_PREFIX
+    request = compute.ListInstancesRequest(project=project, zone=zone,
+                                           filter=filter)
+    for instance in instances.list(request=request):
+        delete_temp_instance(instances, project, zone, instance.name)
+
+def create_temp_instance(instances, project, zone, family, image, machine,
+                         snapshot):
+    """Create log dumper temporary instance"""
+    image = "projects/%s/global/images/family/%s" % (family, image)
+    machine_type = "zones/%s/machineTypes/%s" % (zone, machine)
+    logsource = "global/snapshots/%s" % snapshot
+    bootparams = compute.AttachedDiskInitializeParams(source_image=image)
+    bootdisk = compute.AttachedDisk(boot=True, auto_delete=True,
+                                    initialize_params=bootparams)
+    logparams = compute.AttachedDiskInitializeParams(source_snapshot=logsource)
+    logdisk = compute.AttachedDisk(boot=False, auto_delete=True,
+                                   initialize_params=logparams,
+                                   device_name="ipxelog")
+    nic = compute.NetworkInterface()
+    name = '%s%s' % (IPXE_LOG_PREFIX, uuid4())
+    script = textwrap.dedent(f"""
+    #!/bin/sh
+    tr -d '\\000' < /dev/disk/by-id/google-ipxelog-part3 > /dev/ttyS3
+    echo "{IPXE_LOG_END}" > /dev/ttyS3
+    """).strip()
+    items = compute.Items(key="startup-script", value=script)
+    metadata = compute.Metadata(items=[items])
+    instance = compute.Instance(name=name, machine_type=machine_type,
+                                network_interfaces=[nic], metadata=metadata,
+                                disks=[bootdisk, logdisk])
+    instances.insert(project=project, zone=zone,
+                     instance_resource=instance).result()
+    return name
+
+def get_log_output(instances, project, zone, name):
+    """Get iPXE log output"""
+    request = compute.GetSerialPortOutputInstanceRequest(project=project,
+                                                         zone=zone, port=4,
+                                                         instance=name)
+    while True:
+        log = instances.get_serial_port_output(request=request).contents.strip()
+        if log.endswith(IPXE_LOG_END):
+            if log.startswith(IPXE_LOG_MAGIC):
+                return log[len(IPXE_LOG_MAGIC):-len(IPXE_LOG_END)]
+            else:
+                return log[:-len(IPXE_LOG_END)]
+        time.sleep(1)
+
+# Parse command-line arguments
+#
+parser = argparse.ArgumentParser(description="Import Google Cloud image")
+parser.add_argument('--project', '-j', default="ipxe-images",
+                    help="Google Cloud project")
+parser.add_argument('--zone', '-z', required=True,
+                    help="Google Cloud zone")
+parser.add_argument('--family', '-f', default="debian-cloud",
+                    help="Helper OS image family")
+parser.add_argument('--image', '-i', default="debian-12",
+                    help="Helper OS image")
+parser.add_argument('--machine', '-m', default="e2-micro",
+                    help="Helper machine type")
+parser.add_argument('instance', help="Instance name")
+args = parser.parse_args()
+
+# Construct client objects
+#
+instances = compute.InstancesClient()
+snapshots = compute.SnapshotsClient()
+
+# Clean up old temporary objects
+#
+delete_temp_instances(instances, project=args.project, zone=args.zone)
+delete_temp_snapshots(snapshots, project=args.project)
+
+# Create log disk snapshot
+#
+logdisk = get_log_disk(instances, project=args.project, zone=args.zone,
+                       name=args.instance)
+logsnap = create_temp_snapshot(snapshots, project=args.project, source=logdisk)
+
+# Create log dumper instance
+#
+dumper = create_temp_instance(instances, project=args.project, zone=args.zone,
+                              family=args.family, image=args.image,
+                              machine=args.machine, snapshot=logsnap)
+
+# Wait for log output
+#
+output = get_log_output(instances, project=args.project, zone=args.zone,
+                        name=dumper)
+
+# Print log output
+#
+print(output)
+
+# Clean up
+#
+delete_temp_instance(instances, project=args.project, zone=args.zone,
+                     name=dumper)
+delete_temp_snapshot(snapshots, project=args.project, name=logsnap)

From 60d682409ed77cadf4ac3942c256c0fad00d2103 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 31 Jul 2024 16:20:37 +0100
Subject: [PATCH 157/237] [smbios] Avoid reading beyond end of constructed
 SMBIOS setting

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/smbios/smbios_settings.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/interface/smbios/smbios_settings.c b/src/interface/smbios/smbios_settings.c
index ec31b43f2..ca3f2fe2f 100644
--- a/src/interface/smbios/smbios_settings.c
+++ b/src/interface/smbios/smbios_settings.c
@@ -130,6 +130,13 @@ static int smbios_fetch ( struct settings *settings __unused,
 			return rc;
 		}
 
+		/* Limit length */
+		if ( tag_offset > sizeof ( buf ) ) {
+			tag_len = 0;
+		} else if ( ( tag_offset + tag_len ) > sizeof ( buf ) ) {
+			tag_len = ( sizeof ( buf ) - tag_offset );
+		}
+
 		/* Mangle UUIDs if necessary.  iPXE treats UUIDs as
 		 * being in network byte order (big-endian).  SMBIOS
 		 * specification version 2.6 states that UUIDs are

From c117e6a4811efa057dc70426d58a8dab75245862 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 31 Jul 2024 16:26:48 +0100
Subject: [PATCH 158/237] [smbios] Allow reading an entire SMBIOS data
 structure as a setting

The general syntax for SMBIOS settings:

  smbios/<instance>.<type>.<offset>.<length>

is currently extended such that a <length> of zero indicates that the
byte at <offset> contains a string index, and an <offset> of zero
indicates that the <length> contains a literal string index.

Since the byte at offset zero can never contain a string index, and a
literal string index can never have a zero value, the combination of
both <length> and <offset> being zero is currently invalid and will
always return "not found".

Extend the syntax such that the combination of both <length> and
<offset> being zero may be used to read the entire data structure.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/smbios/smbios_settings.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/interface/smbios/smbios_settings.c b/src/interface/smbios/smbios_settings.c
index ca3f2fe2f..095c35d37 100644
--- a/src/interface/smbios/smbios_settings.c
+++ b/src/interface/smbios/smbios_settings.c
@@ -117,8 +117,16 @@ static int smbios_fetch ( struct settings *settings __unused,
 		 * <offset> contains a string index.  An <offset> of
 		 * zero indicates that the <length> contains a literal
 		 * string index.
+		 *
+		 * Since the byte at offset zero can never contain a
+		 * string index, and a literal string index can never
+		 * be zero, the combination of both <length> and
+		 * <offset> being zero indicates that the entire
+		 * structure is to be read.
 		 */
-		if ( ( tag_len == 0 ) || ( tag_offset == 0 ) ) {
+		if ( ( tag_len == 0 ) && ( tag_offset == 0 ) ) {
+			tag_len = sizeof ( buf );
+		} else if ( ( tag_len == 0 ) || ( tag_offset == 0 ) ) {
 			index = ( ( tag_offset == 0 ) ?
 				  tag_len : buf[tag_offset] );
 			if ( ( rc = read_smbios_string ( &structure, index,

From 121d96b903b221e2c63fa0fc5a8901dc24645a47 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 31 Jul 2024 16:35:31 +0100
Subject: [PATCH 159/237] [cpuid] Allow reading hypervisor CPUID leaves

Hypervisors typically intercept CPUID leaves in the range 0x40000000
to 0x400000ff, with leaf 0x40000000 returning the maximum supported
function within this range in register %eax.

iPXE currently masks off bit 30 from the requested CPUID leaf when
checking to see if a function is supported, which causes this check to
read from leaf 0x00000000 instead of 0x40000000.

Fix by including bit 30 within the mask.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/core/cpuid.c         | 4 ++--
 src/arch/x86/include/ipxe/cpuid.h | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/arch/x86/core/cpuid.c b/src/arch/x86/core/cpuid.c
index 1a7c93e83..b7d9fb6c6 100644
--- a/src/arch/x86/core/cpuid.c
+++ b/src/arch/x86/core/cpuid.c
@@ -84,8 +84,8 @@ int cpuid_supported ( uint32_t function ) {
 		return rc;
 
 	/* Find highest supported function number within this family */
-	cpuid ( ( function & CPUID_EXTENDED ), 0, &max_function, &discard_b,
-		&discard_c, &discard_d );
+	cpuid ( ( function & ( CPUID_EXTENDED | CPUID_HYPERVISOR ) ), 0,
+		  &max_function, &discard_b, &discard_c, &discard_d );
 
 	/* Fail if maximum function number is meaningless (e.g. if we
 	 * are attempting to call an extended function on a CPU which
diff --git a/src/arch/x86/include/ipxe/cpuid.h b/src/arch/x86/include/ipxe/cpuid.h
index 90d1bf01d..99b91c5c8 100644
--- a/src/arch/x86/include/ipxe/cpuid.h
+++ b/src/arch/x86/include/ipxe/cpuid.h
@@ -33,6 +33,9 @@ struct x86_features {
 /** CPUID extended function */
 #define CPUID_EXTENDED 0x80000000UL
 
+/** CPUID hypervisor function */
+#define CPUID_HYPERVISOR 0x40000000UL
+
 /** Get vendor ID and largest standard function */
 #define CPUID_VENDOR_ID 0x00000000UL
 

From a064d397688930dcb341f62587db4e8e786777a0 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 31 Jul 2024 16:43:27 +0100
Subject: [PATCH 160/237] [cpuid] Allow hypervisor CPUID leaves to be accessed
 as settings

Redefine bit 30 of an SMBIOS numerical setting to be part of the
function number, in order to allow access to hypervisor CPUID leaves.

This technically breaks backwards compatibility with scripts
attempting to read more than 64 consecutive functions.  Since there is
no meaningful block of 64 consecutive related functions, it is
vanishingly unlikely that this capability has ever been used.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/core/cpuid_settings.c | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/arch/x86/core/cpuid_settings.c b/src/arch/x86/core/cpuid_settings.c
index 0b67ee91d..9bc69f477 100644
--- a/src/arch/x86/core/cpuid_settings.c
+++ b/src/arch/x86/core/cpuid_settings.c
@@ -38,7 +38,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  *  Bit  31	Extended function
  *  Bits 30-24	(bit 22 = 1) Subfunction number
- *		(bit 22 = 0) Number of consecutive functions to call, minus one
+ *  Bit  30	(bit 22 = 0) Hypervisor function
+ *  Bits 29-24	(bit 22 = 0) Number of consecutive functions to call, minus one
  *  Bit  23	Return result as little-endian (used for strings)
  *  Bit  22	Interpret bits 30-24 as a subfunction number
  *  Bits 21-18	Unused
@@ -98,7 +99,7 @@ enum cpuid_flags {
  * @v tag		Setting tag
  * @ret function	Starting function number
  */
-#define CPUID_FUNCTION( tag ) ( (tag) & 0x800000ffUL )
+#define CPUID_FUNCTION( tag ) ( (tag) & 0xc00000ffUL )
 
 /**
  * Extract subfunction number from CPUID setting tag
@@ -108,6 +109,14 @@ enum cpuid_flags {
  */
 #define CPUID_SUBFUNCTION( tag ) ( ( (tag) >> 24 ) & 0x7f )
 
+/**
+ * Extract number of consecutive functions from CPUID setting tag
+ *
+ * @v tag		Setting tag
+ * @ret num_functions	Number of consecutive functions
+ */
+#define CPUID_NUM_FUNCTIONS( tag ) ( ( ( (tag) >> 24 ) & 0x3f ) + 1 )
+
 /**
  * Extract register array from CPUID setting tag
  *
@@ -165,12 +174,13 @@ static int cpuid_settings_fetch ( struct settings *settings,
 
 	/* Call each function in turn */
 	function = CPUID_FUNCTION ( setting->tag );
-	subfunction = CPUID_SUBFUNCTION ( setting->tag );
 	if ( setting->tag & CPUID_USE_SUBFUNCTION ) {
+		function &= ~CPUID_HYPERVISOR;
+		subfunction = CPUID_SUBFUNCTION ( setting->tag );
 		num_functions = 1;
 	} else {
-		num_functions = ( subfunction + 1 );
 		subfunction = 0;
+		num_functions = CPUID_NUM_FUNCTIONS ( setting->tag );
 	}
 	for ( ; num_functions-- ; function++ ) {
 

From 309ac8fd21be744486ee8be2b64a446cbc754c99 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 6 Aug 2024 13:56:16 +0100
Subject: [PATCH 161/237] [crypto] Do not return an error when skipping the
 final ASN.1 object

Successfully reaching the end of a well-formed ASN.1 object list is
arguably not an error, but the current code (dating back to the
original ASN.1 commit in 2007) will explicitly check for and report
this as an error condition.

Remove the explicit check for reaching the end of a well-formed ASN.1
object list, and instead return success along with a zero-length (and
hence implicitly invalidated) cursor.

Almost every existing caller of asn1_skip() or asn1_skip_if_exists()
currently ignores the return value anyway.  Skipped objects are (by
definition) not of interest to the caller, and the invalidation
behaviour of asn1_skip() ensures that any errors will be safely caught
on a subsequent attempt to actually use the ASN.1 object content.
Since these existing callers ignore the return value, they cannot be
affected by this change.

There is one existing caller of asn1_skip_if_exists() that does check
the return value: in asn1_skip() itself, an error returned from
asn1_skip_if_exists() will cause the cursor to be invalidated.  In the
case of an error indicating only that the cursor length is already
zero, invalidation is a no-op, and so this change affects only the
return value propagated from asn1_skip().

This leaves only a single call site within ocsp_request() where the
return value from asn1_skip() is currently checked.  The return status
here is moot since there is no way for the code in question to fail
(absent a bug in the ASN.1 construction or parsing code).

There are therefore no callers of asn1_skip() or asn1_skip_if_exists()
that rely on an error being returned for successfully reaching the end
of a well-formed ASN.1 object list.  Simplify the code by redefining
this as a successful outcome.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/asn1.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/crypto/asn1.c b/src/crypto/asn1.c
index dc9d1c54d..ec4712f6b 100644
--- a/src/crypto/asn1.c
+++ b/src/crypto/asn1.c
@@ -196,11 +196,6 @@ int asn1_skip_if_exists ( struct asn1_cursor *cursor, unsigned int type ) {
 	DBGC ( cursor, "ASN1 %p skipped object type %02x (len %x)\n",
 	       cursor, type, len );
 
-	if ( ! cursor->len ) {
-		DBGC ( cursor, "ASN1 %p reached end of object\n", cursor );
-		return -ENOENT;
-	}
-
 	return 0;
 }
 

From 0e73b48f77589f69a5418c211b77b95ed26520c8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 7 Aug 2024 13:36:35 +0100
Subject: [PATCH 162/237] [crypto] Clarify ASN.1 cursor invalidation behaviour

Calling asn1_skip_if_exists() on a malformed ASN.1 object may
currently leave the cursor in a partially-updated state, where the tag
byte and one of the length bytes have been stripped.  The cursor is
left with a valid data pointer and length and so no out-of-bounds
access can arise, but the cursor no longer points to the start of an
ASN.1 object.

Ensure that each ASN.1 cursor manipulation code path leads to the
cursor being either fully updated, left unmodified, or invalidated,
and update the function descriptions to reflect this.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/asn1.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/src/crypto/asn1.c b/src/crypto/asn1.c
index ec4712f6b..90619dc0e 100644
--- a/src/crypto/asn1.c
+++ b/src/crypto/asn1.c
@@ -94,6 +94,10 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * object body (i.e. the first byte following the length byte(s)), and
  * the length of the object body (i.e. the number of bytes until the
  * following object tag, if any) is returned.
+ *
+ * If the expected type is not found, the object cursor will not be
+ * modified.  If any other error occurs, the object cursor will be
+ * invalidated.
  */
 int asn1_start ( struct asn1_cursor *cursor, unsigned int type, size_t extra ) {
 	unsigned int len_len;
@@ -103,6 +107,7 @@ int asn1_start ( struct asn1_cursor *cursor, unsigned int type, size_t extra ) {
 	if ( cursor->len < 2 /* Tag byte and first length byte */ ) {
 		if ( cursor->len )
 			DBGC ( cursor, "ASN1 %p too short\n", cursor );
+		asn1_invalidate_cursor ( cursor );
 		return -EINVAL_ASN1_EMPTY;
 	}
 
@@ -127,6 +132,7 @@ int asn1_start ( struct asn1_cursor *cursor, unsigned int type, size_t extra ) {
 	if ( cursor->len < len_len ) {
 		DBGC ( cursor, "ASN1 %p bad length field length %d (max "
 		       "%zd)\n", cursor, len_len, cursor->len );
+		asn1_invalidate_cursor ( cursor );
 		return -EINVAL_ASN1_LEN_LEN;
 	}
 
@@ -140,6 +146,7 @@ int asn1_start ( struct asn1_cursor *cursor, unsigned int type, size_t extra ) {
 	if ( ( cursor->len + extra ) < len ) {
 		DBGC ( cursor, "ASN1 %p bad length %d (max %zd)\n",
 		       cursor, len, ( cursor->len + extra ) );
+		asn1_invalidate_cursor ( cursor );
 		return -EINVAL_ASN1_LEN;
 	}
 
@@ -154,8 +161,9 @@ int asn1_start ( struct asn1_cursor *cursor, unsigned int type, size_t extra ) {
  * @ret rc		Return status code
  *
  * The object cursor will be updated to point to the body of the
- * current ASN.1 object.  If any error occurs, the object cursor will
- * be invalidated.
+ * current ASN.1 object.
+ *
+ * If any error occurs, the object cursor will be invalidated.
  */
 int asn1_enter ( struct asn1_cursor *cursor, unsigned int type ) {
 	int len;
@@ -181,8 +189,11 @@ int asn1_enter ( struct asn1_cursor *cursor, unsigned int type ) {
  * @ret rc		Return status code
  *
  * The object cursor will be updated to point to the next ASN.1
- * object.  If any error occurs, the object cursor will not be
- * modified.
+ * object.
+ *
+ * If the expected type is not found, the object cursor will not be
+ * modified.  If any other error occurs, the object cursor will be
+ * invalidated.
  */
 int asn1_skip_if_exists ( struct asn1_cursor *cursor, unsigned int type ) {
 	int len;
@@ -207,8 +218,9 @@ int asn1_skip_if_exists ( struct asn1_cursor *cursor, unsigned int type ) {
  * @ret rc		Return status code
  *
  * The object cursor will be updated to point to the next ASN.1
- * object.  If any error occurs, the object cursor will be
- * invalidated.
+ * object.
+ *
+ * If any error occurs, the object cursor will be invalidated.
  */
 int asn1_skip ( struct asn1_cursor *cursor, unsigned int type ) {
 	int rc;
@@ -229,8 +241,9 @@ int asn1_skip ( struct asn1_cursor *cursor, unsigned int type ) {
  * @ret rc		Return status code
  *
  * The object cursor will be shrunk to contain only the current ASN.1
- * object.  If any error occurs, the object cursor will be
- * invalidated.
+ * object.
+ *
+ * If any error occurs, the object cursor will be invalidated.
  */
 int asn1_shrink ( struct asn1_cursor *cursor, unsigned int type ) {
 	struct asn1_cursor temp;

From 59e2b03e6ac842d0e69bc4f757bf6da452fca074 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 7 Aug 2024 13:18:47 +0100
Subject: [PATCH 163/237] [crypto] Extend asn1_enter() to handle partial object
 cursors

Handling large ASN.1 objects such as encrypted CMS files will require
the ability to use the asn1_enter() and asn1_skip() family of
functions on partial object cursors, where a defined additional length
is known to exist after the end of the data buffer pointed to by the
ASN.1 object cursor.

We already have support for partial object cursors in the underlying
asn1_start() operation used by both asn1_enter() and asn1_skip(), and
this is used by the DER image probe routine to check that the
potential DER file comprises a single ASN.1 SEQUENCE object.

Add asn1_enter_partial() to formalise the process of entering an ASN.1
partial object, and refactor the DER image probe routine to use this
instead of open-coding calls to the underlying asn1_start() operation.

There is no need for an equivalent asn1_skip_partial() function, since
only objects that are wholly contained within the partial cursor may
be successfully skipped.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/asn1.c       | 56 ++++++++++++++++++++++++++++++-----------
 src/image/der.c         | 15 +++--------
 src/include/ipxe/asn1.h |  4 +--
 3 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/src/crypto/asn1.c b/src/crypto/asn1.c
index 90619dc0e..e66da45b9 100644
--- a/src/crypto/asn1.c
+++ b/src/crypto/asn1.c
@@ -99,7 +99,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * modified.  If any other error occurs, the object cursor will be
  * invalidated.
  */
-int asn1_start ( struct asn1_cursor *cursor, unsigned int type, size_t extra ) {
+static int asn1_start ( struct asn1_cursor *cursor, unsigned int type,
+			size_t extra ) {
 	unsigned int len_len;
 	unsigned int len;
 
@@ -153,6 +154,41 @@ int asn1_start ( struct asn1_cursor *cursor, unsigned int type, size_t extra ) {
 	return len;
 }
 
+/**
+ * Enter ASN.1 partial object
+ *
+ * @v cursor		ASN.1 object cursor
+ * @v type		Expected type, or ASN1_ANY
+ * @v extra		Additional length beyond partial object
+ * @ret rc		Return status code
+ *
+ * The object cursor and additional length will be updated to point to
+ * the body of the current ASN.1 object.
+ *
+ * If any error occurs, the object cursor will be invalidated.
+ */
+int asn1_enter_partial ( struct asn1_cursor *cursor, unsigned int type,
+			 size_t *extra ) {
+	int len;
+
+	/* Parse current object */
+	len = asn1_start ( cursor, type, *extra );
+	if ( len < 0 ) {
+		asn1_invalidate_cursor ( cursor );
+		return len;
+	}
+
+	/* Update cursor and additional length */
+	if ( ( ( size_t ) len ) <= cursor->len )
+		cursor->len = len;
+	assert ( ( len - cursor->len ) <= *extra );
+	*extra = ( len - cursor->len );
+
+	DBGC ( cursor, "ASN1 %p entered object type %02x (len %x)\n",
+	       cursor, type, len );
+	return 0;
+}
+
 /**
  * Enter ASN.1 object
  *
@@ -166,19 +202,9 @@ int asn1_start ( struct asn1_cursor *cursor, unsigned int type, size_t extra ) {
  * If any error occurs, the object cursor will be invalidated.
  */
 int asn1_enter ( struct asn1_cursor *cursor, unsigned int type ) {
-	int len;
+	static size_t no_extra = 0;
 
-	len = asn1_start ( cursor, type, 0 );
-	if ( len < 0 ) {
-		asn1_invalidate_cursor ( cursor );
-		return len;
-	}
-
-	cursor->len = len;
-	DBGC ( cursor, "ASN1 %p entered object type %02x (len %x)\n",
-	       cursor, type, len );
-
-	return 0;
+	return asn1_enter_partial ( cursor, type, &no_extra );
 }
 
 /**
@@ -198,15 +224,17 @@ int asn1_enter ( struct asn1_cursor *cursor, unsigned int type ) {
 int asn1_skip_if_exists ( struct asn1_cursor *cursor, unsigned int type ) {
 	int len;
 
+	/* Parse current object */
 	len = asn1_start ( cursor, type, 0 );
 	if ( len < 0 )
 		return len;
 
+	/* Update cursor */
 	cursor->data += len;
 	cursor->len -= len;
+
 	DBGC ( cursor, "ASN1 %p skipped object type %02x (len %x)\n",
 	       cursor, type, len );
-
 	return 0;
 }
 
diff --git a/src/image/der.c b/src/image/der.c
index fa17e5659..9d31c253b 100644
--- a/src/image/der.c
+++ b/src/image/der.c
@@ -76,8 +76,6 @@ static int der_probe ( struct image *image ) {
 	struct asn1_cursor cursor;
 	uint8_t buf[8];
 	size_t extra;
-	size_t total;
-	int len;
 	int rc;
 
 	/* Sanity check: no realistic DER image can be smaller than this */
@@ -90,21 +88,16 @@ static int der_probe ( struct image *image ) {
 	copy_from_user ( buf, image->data, 0, sizeof ( buf ) );
 	extra = ( image->len - sizeof ( buf ) );
 
-	/* Get length of ASN.1 sequence */
-	len = asn1_start ( &cursor, ASN1_SEQUENCE, extra );
-	if ( len < 0 ) {
-		rc = len;
+	/* Check that image begins with an ASN.1 sequence object */
+	if ( ( rc = asn1_enter_partial ( &cursor, ASN1_SEQUENCE,
+					 &extra ) ) != 0 ) {
 		DBGC ( image, "DER %s is not valid ASN.1: %s\n",
 		       image->name, strerror ( rc ) );
 		return rc;
 	}
 
-	/* Add length of tag and length bytes consumed by asn1_start() */
-	total = ( len + ( cursor.data - ( ( void * ) buf ) ) );
-	assert ( total <= image->len );
-
 	/* Check that image comprises a single well-formed ASN.1 object */
-	if ( total != image->len ) {
+	if ( extra != ( image->len - sizeof ( buf ) ) ) {
 		DBGC ( image, "DER %s is not single ASN.1\n", image->name );
 		return -ENOEXEC;
 	}
diff --git a/src/include/ipxe/asn1.h b/src/include/ipxe/asn1.h
index ac7ea5604..1580c8baf 100644
--- a/src/include/ipxe/asn1.h
+++ b/src/include/ipxe/asn1.h
@@ -404,8 +404,8 @@ asn1_built ( struct asn1_builder *builder ) {
 	return &u->cursor;
 }
 
-extern int asn1_start ( struct asn1_cursor *cursor, unsigned int type,
-			size_t extra );
+extern int asn1_enter_partial ( struct asn1_cursor *cursor, unsigned int type,
+				size_t *extra );
 extern int asn1_enter ( struct asn1_cursor *cursor, unsigned int type );
 extern int asn1_skip_if_exists ( struct asn1_cursor *cursor,
 				 unsigned int type );

From d85590b6584499569c19f7ee4a1e0c10d5132f70 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 12 Aug 2024 12:26:52 +0100
Subject: [PATCH 164/237] [crypto] Centralise mechanisms for identifying X.509
 certificates

Centralise all current mechanisms for identifying an X.509 certificate
(by raw content, by subject, by issuer and serial number, and by
matching public key), and remove the certstore-specific and
CMS-specific variants of these functions.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/certstore.c       |  74 ++++++-------------------
 src/crypto/cms.c             |  30 +---------
 src/crypto/x509.c            | 103 ++++++++++++++++++++++++++++++++++-
 src/include/ipxe/certstore.h |   4 --
 src/include/ipxe/x509.h      |  19 +++++++
 src/net/tls.c                |   2 +-
 6 files changed, 139 insertions(+), 93 deletions(-)

diff --git a/src/crypto/certstore.c b/src/crypto/certstore.c
index 2676c7e1e..f8ddbd3d7 100644
--- a/src/crypto/certstore.c
+++ b/src/crypto/certstore.c
@@ -69,67 +69,29 @@ static struct asn1_cursor certstore_raw[] = {
 static struct x509_certificate certstore_certs[ sizeof ( certstore_raw ) /
 						sizeof ( certstore_raw[0] ) ];
 
+/**
+ * Mark stored certificate as most recently used
+ *
+ * @v certs		X.509 certificate list
+ * @v cert		X.509 certificate
+ */
+static void certstore_found ( struct x509_chain *certs,
+			      struct x509_certificate *cert ) {
+
+	/* Mark as most recently used */
+	list_del ( &cert->store.list );
+	list_add ( &cert->store.list, &certs->links );
+	DBGC2 ( certs, "CERTSTORE found certificate %s\n",
+		x509_name ( cert ) );
+}
+
 /** Certificate store */
 struct x509_chain certstore = {
 	.refcnt = REF_INIT ( ref_no_free ),
 	.links = LIST_HEAD_INIT ( certstore.links ),
+	.found = certstore_found,
 };
 
-/**
- * Mark stored certificate as most recently used
- *
- * @v cert		X.509 certificate
- * @ret cert		X.509 certificate
- */
-static struct x509_certificate *
-certstore_found ( struct x509_certificate *cert ) {
-
-	/* Mark as most recently used */
-	list_del ( &cert->store.list );
-	list_add ( &cert->store.list, &certstore.links );
-	DBGC2 ( &certstore, "CERTSTORE found certificate %s\n",
-		x509_name ( cert ) );
-
-	return cert;
-}
-
-/**
- * Find certificate in store
- *
- * @v raw		Raw certificate data
- * @ret cert		X.509 certificate, or NULL if not found
- */
-struct x509_certificate * certstore_find ( struct asn1_cursor *raw ) {
-	struct x509_certificate *cert;
-
-	/* Search for certificate within store */
-	list_for_each_entry ( cert, &certstore.links, store.list ) {
-		if ( asn1_compare ( raw, &cert->raw ) == 0 )
-			return certstore_found ( cert );
-	}
-	return NULL;
-}
-
-/**
- * Find certificate in store corresponding to a private key
- *
- * @v key		Private key
- * @ret cert		X.509 certificate, or NULL if not found
- */
-struct x509_certificate * certstore_find_key ( struct private_key *key ) {
-	struct x509_certificate *cert;
-
-	/* Search for certificate within store */
-	list_for_each_entry ( cert, &certstore.links, store.list ) {
-		if ( pubkey_match ( cert->signature_algorithm->pubkey,
-				    key->builder.data, key->builder.len,
-				    cert->subject.public_key.raw.data,
-				    cert->subject.public_key.raw.len ) == 0 )
-			return certstore_found ( cert );
-	}
-	return NULL;
-}
-
 /**
  * Add certificate to store
  *
@@ -219,7 +181,7 @@ static void certstore_init ( void ) {
 
 		/* Skip if certificate already present in store */
 		raw = &certstore_raw[i];
-		if ( ( cert = certstore_find ( raw ) ) != NULL ) {
+		if ( ( cert = x509_find ( &certstore, raw ) ) != NULL ) {
 			DBGC ( &certstore, "CERTSTORE permanent certificate %d "
 			       "is a duplicate of %s\n", i, x509_name ( cert ));
 			continue;
diff --git a/src/crypto/cms.c b/src/crypto/cms.c
index 9511cec8a..19a0ce7ad 100644
--- a/src/crypto/cms.c
+++ b/src/crypto/cms.c
@@ -143,34 +143,6 @@ static int cms_parse_certificates ( struct cms_signature *sig,
 	return 0;
 }
 
-/**
- * Identify CMS signature certificate by issuer and serial number
- *
- * @v sig		CMS signature
- * @v issuer		Issuer
- * @v serial		Serial number
- * @ret cert		X.509 certificate, or NULL if not found
- */
-static struct x509_certificate *
-cms_find_issuer_serial ( struct cms_signature *sig,
-			 const struct asn1_cursor *issuer,
-			 const struct asn1_cursor *serial ) {
-	struct x509_link *link;
-	struct x509_certificate *cert;
-
-	/* Scan through certificate list */
-	list_for_each_entry ( link, &sig->certificates->links, list ) {
-
-		/* Check issuer and serial number */
-		cert = link->cert;
-		if ( ( asn1_compare ( issuer, &cert->issuer.raw ) == 0 ) &&
-		     ( asn1_compare ( serial, &cert->serial.raw ) == 0 ) )
-			return cert;
-	}
-
-	return NULL;
-}
-
 /**
  * Parse CMS signature signer identifier
  *
@@ -216,7 +188,7 @@ static int cms_parse_signer_identifier ( struct cms_signature *sig,
 	DBGC_HDA ( sig, 0, serial.data, serial.len );
 
 	/* Identify certificate */
-	cert = cms_find_issuer_serial ( sig, &issuer, &serial );
+	cert = x509_find_issuer_serial ( sig->certificates, &issuer, &serial );
 	if ( ! cert ) {
 		DBGC ( sig, "CMS %p/%p could not identify signer's "
 		       "certificate\n", sig, info );
diff --git a/src/crypto/x509.c b/src/crypto/x509.c
index 92318093e..341b91449 100644
--- a/src/crypto/x509.c
+++ b/src/crypto/x509.c
@@ -38,6 +38,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/rsa.h>
 #include <ipxe/rootcert.h>
 #include <ipxe/certstore.h>
+#include <ipxe/privkey.h>
 #include <ipxe/socket.h>
 #include <ipxe/in.h>
 #include <ipxe/image.h>
@@ -1078,7 +1079,7 @@ int x509_certificate ( const void *data, size_t len,
 	asn1_shrink_any ( &cursor );
 
 	/* Return stored certificate, if present */
-	if ( ( *cert = certstore_find ( &cursor ) ) != NULL ) {
+	if ( ( *cert = x509_find ( &certstore, &cursor ) ) != NULL ) {
 
 		/* Add caller's reference */
 		x509_get ( *cert );
@@ -1710,6 +1711,47 @@ void x509_truncate ( struct x509_chain *chain, struct x509_link *link ) {
 	}
 }
 
+/**
+ * Mark X.509 certificate as found
+ *
+ * @v certs		X.509 certificate list
+ * @v cert		X.509 certificate
+ * @ret cert		X.509 certificate
+ */
+static struct x509_certificate * x509_found ( struct x509_chain *certs,
+					      struct x509_certificate *cert ) {
+
+	/* Mark as found, if applicable */
+	if ( certs->found )
+		certs->found ( certs, cert );
+
+	return cert;
+}
+
+/**
+ * Identify X.509 certificate by raw certificate data
+ *
+ * @v certs		X.509 certificate list
+ * @v raw		Raw certificate data
+ * @ret cert		X.509 certificate, or NULL if not found
+ */
+struct x509_certificate * x509_find ( struct x509_chain *certs,
+				      const struct asn1_cursor *raw ) {
+	struct x509_link *link;
+	struct x509_certificate *cert;
+
+	/* Search for certificate within store */
+	list_for_each_entry ( link, &certs->links, list ) {
+
+		/* Check raw certificate data */
+		cert = link->cert;
+		if ( asn1_compare ( raw, &cert->raw ) == 0 )
+			return x509_found ( certs, cert );
+	}
+
+	return NULL;
+}
+
 /**
  * Identify X.509 certificate by subject
  *
@@ -1717,7 +1759,7 @@ void x509_truncate ( struct x509_chain *chain, struct x509_link *link ) {
  * @v subject		Subject
  * @ret cert		X.509 certificate, or NULL if not found
  */
-static struct x509_certificate *
+struct x509_certificate *
 x509_find_subject ( struct x509_chain *certs,
 		    const struct asn1_cursor *subject ) {
 	struct x509_link *link;
@@ -1729,7 +1771,62 @@ x509_find_subject ( struct x509_chain *certs,
 		/* Check subject */
 		cert = link->cert;
 		if ( asn1_compare ( subject, &cert->subject.raw ) == 0 )
-			return cert;
+			return x509_found ( certs, cert );
+	}
+
+	return NULL;
+}
+
+/**
+ * Identify X.509 certificate by issuer and serial number
+ *
+ * @v certs		X.509 certificate list
+ * @v issuer		Issuer
+ * @v serial		Serial number
+ * @ret cert		X.509 certificate, or NULL if not found
+ */
+struct x509_certificate *
+x509_find_issuer_serial ( struct x509_chain *certs,
+			  const struct asn1_cursor *issuer,
+			  const struct asn1_cursor *serial ) {
+	struct x509_link *link;
+	struct x509_certificate *cert;
+
+	/* Scan through certificate list */
+	list_for_each_entry ( link, &certs->links, list ) {
+
+		/* Check issuer and serial number */
+		cert = link->cert;
+		if ( ( asn1_compare ( issuer, &cert->issuer.raw ) == 0 ) &&
+		     ( asn1_compare ( serial, &cert->serial.raw ) == 0 ) )
+			return x509_found ( certs, cert );
+	}
+
+	return NULL;
+}
+
+/**
+ * Identify X.509 certificate by corresponding public key
+ *
+ * @v certs		X.509 certificate list
+ * @v key		Private key
+ * @ret cert		X.509 certificate, or NULL if not found
+ */
+struct x509_certificate * x509_find_key ( struct x509_chain *certs,
+					  struct private_key *key ) {
+	struct x509_link *link;
+	struct x509_certificate *cert;
+
+	/* Scan through certificate list */
+	list_for_each_entry ( link, &certs->links, list ) {
+
+		/* Check public key */
+		cert = link->cert;
+		if ( pubkey_match ( cert->signature_algorithm->pubkey,
+				    key->builder.data, key->builder.len,
+				    cert->subject.public_key.raw.data,
+				    cert->subject.public_key.raw.len ) == 0 )
+			return x509_found ( certs, cert );
 	}
 
 	return NULL;
diff --git a/src/include/ipxe/certstore.h b/src/include/ipxe/certstore.h
index ce96666cf..e276d6792 100644
--- a/src/include/ipxe/certstore.h
+++ b/src/include/ipxe/certstore.h
@@ -9,14 +9,10 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#include <ipxe/asn1.h>
 #include <ipxe/x509.h>
-#include <ipxe/privkey.h>
 
 extern struct x509_chain certstore;
 
-extern struct x509_certificate * certstore_find ( struct asn1_cursor *raw );
-extern struct x509_certificate * certstore_find_key ( struct private_key *key );
 extern void certstore_add ( struct x509_certificate *cert );
 extern void certstore_del ( struct x509_certificate *cert );
 
diff --git a/src/include/ipxe/x509.h b/src/include/ipxe/x509.h
index 87323cec0..612743a77 100644
--- a/src/include/ipxe/x509.h
+++ b/src/include/ipxe/x509.h
@@ -17,6 +17,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/list.h>
 
 struct image;
+struct private_key;
 
 /** An X.509 serial number */
 struct x509_serial {
@@ -201,6 +202,13 @@ struct x509_chain {
 	struct refcnt refcnt;
 	/** List of links */
 	struct list_head links;
+	/** Mark certificate as found
+	 *
+	 * @v certs		X.509 certificate list
+	 * @v cert		X.509 certificate
+	 */
+	void ( * found ) ( struct x509_chain *certs,
+			   struct x509_certificate *cert );
 };
 
 /** An X.509 certificate */
@@ -424,6 +432,17 @@ extern int x509_append ( struct x509_chain *chain,
 extern int x509_append_raw ( struct x509_chain *chain, const void *data,
 			     size_t len );
 extern void x509_truncate ( struct x509_chain *chain, struct x509_link *link );
+extern struct x509_certificate * x509_find ( struct x509_chain *certs,
+					     const struct asn1_cursor *raw );
+extern struct x509_certificate *
+x509_find_subject ( struct x509_chain *certs,
+		    const struct asn1_cursor *subject );
+extern struct x509_certificate *
+x509_find_issuer_serial ( struct x509_chain *certs,
+			  const struct asn1_cursor *issuer,
+			  const struct asn1_cursor *serial );
+extern struct x509_certificate * x509_find_key ( struct x509_chain *certs,
+						 struct private_key *key );
 extern int x509_auto_append ( struct x509_chain *chain,
 			      struct x509_chain *certs );
 extern int x509_validate_chain ( struct x509_chain *chain, time_t time,
diff --git a/src/net/tls.c b/src/net/tls.c
index 5f89be452..98414e2b1 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -2467,7 +2467,7 @@ static int tls_new_certificate_request ( struct tls_connection *tls,
 	tls->certs = NULL;
 
 	/* Determine client certificate to be sent */
-	cert = certstore_find_key ( tls->key );
+	cert = x509_find_key ( &certstore, tls->key );
 	if ( ! cert ) {
 		DBGC ( tls, "TLS %p could not find certificate corresponding "
 		       "to private key\n", tls );

From 96fb7a0a9395cec423a58069e1b49535e8ceceef Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 13 Aug 2024 12:25:25 +0100
Subject: [PATCH 165/237] [crypto] Allow passing a NULL certificate store to
 x509_find() et al

Allow passing a NULL value for the certificate list to all functions
used for identifying an X.509 certificate from an existing set of
certificates, and rename function parameters to indicate that this
certificate list represents an unordered certificate store (rather
than an ordered certificate chain).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/certstore.c  |  8 ++---
 src/crypto/x509.c       | 71 +++++++++++++++++++++++++----------------
 src/include/ipxe/x509.h | 14 ++++----
 src/net/tls.c           |  2 +-
 4 files changed, 55 insertions(+), 40 deletions(-)

diff --git a/src/crypto/certstore.c b/src/crypto/certstore.c
index f8ddbd3d7..31797c4cd 100644
--- a/src/crypto/certstore.c
+++ b/src/crypto/certstore.c
@@ -72,16 +72,16 @@ static struct x509_certificate certstore_certs[ sizeof ( certstore_raw ) /
 /**
  * Mark stored certificate as most recently used
  *
- * @v certs		X.509 certificate list
+ * @v store		Certificate store
  * @v cert		X.509 certificate
  */
-static void certstore_found ( struct x509_chain *certs,
+static void certstore_found ( struct x509_chain *store,
 			      struct x509_certificate *cert ) {
 
 	/* Mark as most recently used */
 	list_del ( &cert->store.list );
-	list_add ( &cert->store.list, &certs->links );
-	DBGC2 ( certs, "CERTSTORE found certificate %s\n",
+	list_add ( &cert->store.list, &store->links );
+	DBGC2 ( store, "CERTSTORE found certificate %s\n",
 		x509_name ( cert ) );
 }
 
diff --git a/src/crypto/x509.c b/src/crypto/x509.c
index 341b91449..acb85620f 100644
--- a/src/crypto/x509.c
+++ b/src/crypto/x509.c
@@ -1079,7 +1079,7 @@ int x509_certificate ( const void *data, size_t len,
 	asn1_shrink_any ( &cursor );
 
 	/* Return stored certificate, if present */
-	if ( ( *cert = x509_find ( &certstore, &cursor ) ) != NULL ) {
+	if ( ( *cert = x509_find ( NULL, &cursor ) ) != NULL ) {
 
 		/* Add caller's reference */
 		x509_get ( *cert );
@@ -1714,16 +1714,19 @@ void x509_truncate ( struct x509_chain *chain, struct x509_link *link ) {
 /**
  * Mark X.509 certificate as found
  *
- * @v certs		X.509 certificate list
+ * @v store		Certificate store
  * @v cert		X.509 certificate
  * @ret cert		X.509 certificate
  */
-static struct x509_certificate * x509_found ( struct x509_chain *certs,
+static struct x509_certificate * x509_found ( struct x509_chain *store,
 					      struct x509_certificate *cert ) {
 
+	/* Sanity check */
+	assert ( store != NULL );
+
 	/* Mark as found, if applicable */
-	if ( certs->found )
-		certs->found ( certs, cert );
+	if ( store->found )
+		store->found ( store, cert );
 
 	return cert;
 }
@@ -1731,22 +1734,26 @@ static struct x509_certificate * x509_found ( struct x509_chain *certs,
 /**
  * Identify X.509 certificate by raw certificate data
  *
- * @v certs		X.509 certificate list
+ * @v store		Certificate store, or NULL to use default
  * @v raw		Raw certificate data
  * @ret cert		X.509 certificate, or NULL if not found
  */
-struct x509_certificate * x509_find ( struct x509_chain *certs,
+struct x509_certificate * x509_find ( struct x509_chain *store,
 				      const struct asn1_cursor *raw ) {
 	struct x509_link *link;
 	struct x509_certificate *cert;
 
+	/* Use default certificate store if none specified */
+	if ( ! store )
+		store = &certstore;
+
 	/* Search for certificate within store */
-	list_for_each_entry ( link, &certs->links, list ) {
+	list_for_each_entry ( link, &store->links, list ) {
 
 		/* Check raw certificate data */
 		cert = link->cert;
 		if ( asn1_compare ( raw, &cert->raw ) == 0 )
-			return x509_found ( certs, cert );
+			return x509_found ( store, cert );
 	}
 
 	return NULL;
@@ -1755,23 +1762,27 @@ struct x509_certificate * x509_find ( struct x509_chain *certs,
 /**
  * Identify X.509 certificate by subject
  *
- * @v certs		X.509 certificate list
+ * @v store		Certificate store, or NULL to use default
  * @v subject		Subject
  * @ret cert		X.509 certificate, or NULL if not found
  */
 struct x509_certificate *
-x509_find_subject ( struct x509_chain *certs,
+x509_find_subject ( struct x509_chain *store,
 		    const struct asn1_cursor *subject ) {
 	struct x509_link *link;
 	struct x509_certificate *cert;
 
+	/* Use default certificate store if none specified */
+	if ( ! store )
+		store = &certstore;
+
 	/* Scan through certificate list */
-	list_for_each_entry ( link, &certs->links, list ) {
+	list_for_each_entry ( link, &store->links, list ) {
 
 		/* Check subject */
 		cert = link->cert;
 		if ( asn1_compare ( subject, &cert->subject.raw ) == 0 )
-			return x509_found ( certs, cert );
+			return x509_found ( store, cert );
 	}
 
 	return NULL;
@@ -1780,26 +1791,30 @@ x509_find_subject ( struct x509_chain *certs,
 /**
  * Identify X.509 certificate by issuer and serial number
  *
- * @v certs		X.509 certificate list
+ * @v store		Certificate store, or NULL to use default
  * @v issuer		Issuer
  * @v serial		Serial number
  * @ret cert		X.509 certificate, or NULL if not found
  */
 struct x509_certificate *
-x509_find_issuer_serial ( struct x509_chain *certs,
+x509_find_issuer_serial ( struct x509_chain *store,
 			  const struct asn1_cursor *issuer,
 			  const struct asn1_cursor *serial ) {
 	struct x509_link *link;
 	struct x509_certificate *cert;
 
+	/* Use default certificate store if none specified */
+	if ( ! store )
+		store = &certstore;
+
 	/* Scan through certificate list */
-	list_for_each_entry ( link, &certs->links, list ) {
+	list_for_each_entry ( link, &store->links, list ) {
 
 		/* Check issuer and serial number */
 		cert = link->cert;
 		if ( ( asn1_compare ( issuer, &cert->issuer.raw ) == 0 ) &&
 		     ( asn1_compare ( serial, &cert->serial.raw ) == 0 ) )
-			return x509_found ( certs, cert );
+			return x509_found ( store, cert );
 	}
 
 	return NULL;
@@ -1808,17 +1823,21 @@ x509_find_issuer_serial ( struct x509_chain *certs,
 /**
  * Identify X.509 certificate by corresponding public key
  *
- * @v certs		X.509 certificate list
+ * @v store		Certificate store, or NULL to use default
  * @v key		Private key
  * @ret cert		X.509 certificate, or NULL if not found
  */
-struct x509_certificate * x509_find_key ( struct x509_chain *certs,
+struct x509_certificate * x509_find_key ( struct x509_chain *store,
 					  struct private_key *key ) {
 	struct x509_link *link;
 	struct x509_certificate *cert;
 
+	/* Use default certificate store if none specified */
+	if ( ! store )
+		store = &certstore;
+
 	/* Scan through certificate list */
-	list_for_each_entry ( link, &certs->links, list ) {
+	list_for_each_entry ( link, &store->links, list ) {
 
 		/* Check public key */
 		cert = link->cert;
@@ -1826,7 +1845,7 @@ struct x509_certificate * x509_find_key ( struct x509_chain *certs,
 				    key->builder.data, key->builder.len,
 				    cert->subject.public_key.raw.data,
 				    cert->subject.public_key.raw.len ) == 0 )
-			return x509_found ( certs, cert );
+			return x509_found ( store, cert );
 	}
 
 	return NULL;
@@ -1836,13 +1855,13 @@ struct x509_certificate * x509_find_key ( struct x509_chain *certs,
  * Append X.509 certificates to X.509 certificate chain
  *
  * @v chain		X.509 certificate chain
- * @v certs		X.509 certificate list
+ * @v store		Certificate store, or NULL to use default
  * @ret rc		Return status code
  *
  * Certificates will be automatically appended to the chain based upon
  * the subject and issuer names.
  */
-int x509_auto_append ( struct x509_chain *chain, struct x509_chain *certs ) {
+int x509_auto_append ( struct x509_chain *chain, struct x509_chain *store ) {
 	struct x509_certificate *cert;
 	struct x509_certificate *previous;
 	int rc;
@@ -1859,7 +1878,7 @@ int x509_auto_append ( struct x509_chain *chain, struct x509_chain *certs ) {
 
 		/* Find issuing certificate */
 		previous = cert;
-		cert = x509_find_subject ( certs, &cert->issuer.raw );
+		cert = x509_find_subject ( store, &cert->issuer.raw );
 		if ( ! cert )
 			break;
 		if ( cert == previous )
@@ -1888,10 +1907,6 @@ int x509_validate_chain ( struct x509_chain *chain, time_t time,
 	struct x509_link *link;
 	int rc;
 
-	/* Use default certificate store if none specified */
-	if ( ! store )
-		store = &certstore;
-
 	/* Append any applicable certificates from the certificate store */
 	if ( ( rc = x509_auto_append ( chain, store ) ) != 0 )
 		return rc;
diff --git a/src/include/ipxe/x509.h b/src/include/ipxe/x509.h
index 612743a77..e71cee8a3 100644
--- a/src/include/ipxe/x509.h
+++ b/src/include/ipxe/x509.h
@@ -204,10 +204,10 @@ struct x509_chain {
 	struct list_head links;
 	/** Mark certificate as found
 	 *
-	 * @v certs		X.509 certificate list
+	 * @v store		Certificate store
 	 * @v cert		X.509 certificate
 	 */
-	void ( * found ) ( struct x509_chain *certs,
+	void ( * found ) ( struct x509_chain *store,
 			   struct x509_certificate *cert );
 };
 
@@ -432,19 +432,19 @@ extern int x509_append ( struct x509_chain *chain,
 extern int x509_append_raw ( struct x509_chain *chain, const void *data,
 			     size_t len );
 extern void x509_truncate ( struct x509_chain *chain, struct x509_link *link );
-extern struct x509_certificate * x509_find ( struct x509_chain *certs,
+extern struct x509_certificate * x509_find ( struct x509_chain *store,
 					     const struct asn1_cursor *raw );
 extern struct x509_certificate *
-x509_find_subject ( struct x509_chain *certs,
+x509_find_subject ( struct x509_chain *store,
 		    const struct asn1_cursor *subject );
 extern struct x509_certificate *
-x509_find_issuer_serial ( struct x509_chain *certs,
+x509_find_issuer_serial ( struct x509_chain *store,
 			  const struct asn1_cursor *issuer,
 			  const struct asn1_cursor *serial );
-extern struct x509_certificate * x509_find_key ( struct x509_chain *certs,
+extern struct x509_certificate * x509_find_key ( struct x509_chain *store,
 						 struct private_key *key );
 extern int x509_auto_append ( struct x509_chain *chain,
-			      struct x509_chain *certs );
+			      struct x509_chain *store );
 extern int x509_validate_chain ( struct x509_chain *chain, time_t time,
 				 struct x509_chain *store,
 				 struct x509_root *root );
diff --git a/src/net/tls.c b/src/net/tls.c
index 98414e2b1..c08057103 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -2467,7 +2467,7 @@ static int tls_new_certificate_request ( struct tls_connection *tls,
 	tls->certs = NULL;
 
 	/* Determine client certificate to be sent */
-	cert = x509_find_key ( &certstore, tls->key );
+	cert = x509_find_key ( NULL, tls->key );
 	if ( ! cert ) {
 		DBGC ( tls, "TLS %p could not find certificate corresponding "
 		       "to private key\n", tls );

From 3b4d0cb555a01df8b56f422d9d17522ae60e17be Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 9 Aug 2024 16:33:51 +0100
Subject: [PATCH 166/237] [crypto] Pass image as parameter to CMS functions

The cms_signature() and cms_verify() functions currently accept raw
data pointers.  This will not be possible for cms_decrypt(), which
will need the ability to extract fragments of ASN.1 data from a
potentially large image.

Change cms_signature() and cms_verify() to accept an image as an input
parameter, and move the responsibility for setting the image trust
flag within cms_verify() since that now becomes a more natural fit.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/cms.c       | 48 +++++++++++++++--------
 src/include/ipxe/cms.h |  6 ++-
 src/tests/cms_test.c   | 86 ++++++++++++++++++++++++++++++------------
 src/usr/imgtrust.c     | 26 ++-----------
 4 files changed, 101 insertions(+), 65 deletions(-)

diff --git a/src/crypto/cms.c b/src/crypto/cms.c
index 19a0ce7ad..cbc0736bb 100644
--- a/src/crypto/cms.c
+++ b/src/crypto/cms.c
@@ -37,6 +37,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <errno.h>
 #include <ipxe/asn1.h>
 #include <ipxe/x509.h>
+#include <ipxe/image.h>
 #include <ipxe/malloc.h>
 #include <ipxe/uaccess.h>
 #include <ipxe/cms.h>
@@ -372,7 +373,6 @@ static int cms_parse ( struct cms_signature *sig,
 	asn1_enter ( &cursor, ASN1_SEQUENCE );
 
 	/* Parse contentType */
-
 	if ( ( rc = cms_parse_content_type ( sig, &cursor ) ) != 0 )
 		return rc;
 	asn1_skip_any ( &cursor );
@@ -453,16 +453,16 @@ static void cms_free ( struct refcnt *refcnt ) {
 /**
  * Create CMS signature
  *
- * @v data		Raw signature data
- * @v len		Length of raw data
+ * @v image		Image
  * @ret sig		CMS signature
  * @ret rc		Return status code
  *
  * On success, the caller holds a reference to the CMS signature, and
  * is responsible for ultimately calling cms_put().
  */
-int cms_signature ( const void *data, size_t len, struct cms_signature **sig ) {
-	struct asn1_cursor cursor;
+int cms_signature ( struct image *image, struct cms_signature **sig ) {
+	struct asn1_cursor *raw;
+	int next;
 	int rc;
 
 	/* Allocate and initialise signature */
@@ -481,18 +481,30 @@ int cms_signature ( const void *data, size_t len, struct cms_signature **sig ) {
 		goto err_alloc_chain;
 	}
 
-	/* Initialise cursor */
-	cursor.data = data;
-	cursor.len = len;
-	asn1_shrink_any ( &cursor );
+	/* Get raw signature data */
+	next = image_asn1 ( image, 0, &raw );
+	if ( next < 0 ) {
+		rc = next;
+		DBGC ( *sig, "CMS %p could not get raw ASN.1 data: %s\n",
+		       *sig, strerror ( rc ) );
+		goto err_asn1;
+	}
+
+	/* Use only first signature in image */
+	asn1_shrink_any ( raw );
 
 	/* Parse signature */
-	if ( ( rc = cms_parse ( *sig, &cursor ) ) != 0 )
+	if ( ( rc = cms_parse ( *sig, raw ) ) != 0 )
 		goto err_parse;
 
+	/* Free raw signature data */
+	free ( raw );
+
 	return 0;
 
  err_parse:
+	free ( raw );
+ err_asn1:
  err_alloc_chain:
 	cms_put ( *sig );
  err_alloc:
@@ -642,15 +654,14 @@ static int cms_verify_signer_info ( struct cms_signature *sig,
  * Verify CMS signature
  *
  * @v sig		CMS signature
- * @v data		Signed data
- * @v len		Length of signed data
+ * @v image		Signed image
  * @v name		Required common name, or NULL to check all signatures
  * @v time		Time at which to validate certificates
  * @v store		Certificate store, or NULL to use default
  * @v root		Root certificate list, or NULL to use default
  * @ret rc		Return status code
  */
-int cms_verify ( struct cms_signature *sig, userptr_t data, size_t len,
+int cms_verify ( struct cms_signature *sig, struct image *image,
 		 const char *name, time_t time, struct x509_chain *store,
 		 struct x509_root *root ) {
 	struct cms_signer_info *info;
@@ -658,13 +669,17 @@ int cms_verify ( struct cms_signature *sig, userptr_t data, size_t len,
 	int count = 0;
 	int rc;
 
+	/* Mark image as untrusted */
+	image_untrust ( image );
+
 	/* Verify using all signerInfos */
 	list_for_each_entry ( info, &sig->info, list ) {
 		cert = x509_first ( info->chain );
 		if ( name && ( x509_check_name ( cert, name ) != 0 ) )
 			continue;
-		if ( ( rc = cms_verify_signer_info ( sig, info, data, len, time,
-						     store, root ) ) != 0 )
+		if ( ( rc = cms_verify_signer_info ( sig, info, image->data,
+						     image->len, time, store,
+						     root ) ) != 0 )
 			return rc;
 		count++;
 	}
@@ -681,5 +696,8 @@ int cms_verify ( struct cms_signature *sig, userptr_t data, size_t len,
 		}
 	}
 
+	/* Mark image as trusted */
+	image_trust ( image );
+
 	return 0;
 }
diff --git a/src/include/ipxe/cms.h b/src/include/ipxe/cms.h
index 7adf724b2..cca7779c5 100644
--- a/src/include/ipxe/cms.h
+++ b/src/include/ipxe/cms.h
@@ -16,6 +16,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/refcnt.h>
 #include <ipxe/uaccess.h>
 
+struct image;
+
 /** CMS signer information */
 struct cms_signer_info {
 	/** List of signer information blocks */
@@ -67,9 +69,9 @@ cms_put ( struct cms_signature *sig ) {
 	ref_put ( &sig->refcnt );
 }
 
-extern int cms_signature ( const void *data, size_t len,
+extern int cms_signature ( struct image *image,
 			   struct cms_signature **sig );
-extern int cms_verify ( struct cms_signature *sig, userptr_t data, size_t len,
+extern int cms_verify ( struct cms_signature *sig, struct image *image,
 			const char *name, time_t time, struct x509_chain *store,
 			struct x509_root *root );
 
diff --git a/src/tests/cms_test.c b/src/tests/cms_test.c
index f35fa206d..d98b2c3e5 100644
--- a/src/tests/cms_test.c
+++ b/src/tests/cms_test.c
@@ -36,7 +36,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <string.h>
 #include <ipxe/sha256.h>
 #include <ipxe/x509.h>
+#include <ipxe/image.h>
 #include <ipxe/uaccess.h>
+#include <ipxe/der.h>
 #include <ipxe/cms.h>
 #include <ipxe/test.h>
 
@@ -45,19 +47,14 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** CMS test code blob */
 struct cms_test_code {
-	/** Data */
-	const void *data;
-	/** Length of data */
-	size_t len;
+	/** Code image */
+	struct image image;
 };
 
 /** CMS test signature */
 struct cms_test_signature {
-	/** Data */
-	const void *data;
-	/** Length of data */
-	size_t len;
-
+	/** Signature image */
+	struct image image;
 	/** Parsed signature */
 	struct cms_signature *sig;
 };
@@ -69,19 +66,29 @@ struct cms_test_signature {
 #define FINGERPRINT(...) { __VA_ARGS__ }
 
 /** Define a test code blob */
-#define SIGNED_CODE( name, DATA )					\
-	static const uint8_t name ## _data[] = DATA;			\
-	static struct cms_test_code name = {				\
-		.data = name ## _data,					\
-		.len = sizeof ( name ## _data ),			\
+#define SIGNED_CODE( NAME, DATA )					\
+	static const uint8_t NAME ## _data[] = DATA;			\
+	static struct cms_test_code NAME = {				\
+		.image = {						\
+			.refcnt = REF_INIT ( ref_no_free ),		\
+			.name = #NAME,					\
+			.type = &der_image_type,			\
+			.data = ( userptr_t ) ( NAME ## _data ),	\
+			.len = sizeof ( NAME ## _data ),		\
+		},							\
 	}
 
 /** Define a test signature */
-#define SIGNATURE( name, DATA )						\
-	static const uint8_t name ## _data[] = DATA;			\
-	static struct cms_test_signature name = {			\
-		.data = name ## _data,					\
-		.len = sizeof ( name ## _data ),			\
+#define SIGNATURE( NAME, DATA )						\
+	static const uint8_t NAME ## _data[] = DATA;			\
+	static struct cms_test_signature NAME = {			\
+		.image = {						\
+			.refcnt = REF_INIT ( ref_no_free ),		\
+			.name = #NAME,					\
+			.type = &der_image_type,			\
+			.data = ( userptr_t ) ( NAME ## _data ),	\
+			.len = sizeof ( NAME ## _data ),		\
+		},							\
 	}
 
 /** Code that has been signed */
@@ -1353,9 +1360,16 @@ static time_t test_expired = 1375573111ULL; /* Sat Aug  3 23:38:31 2013 */
  */
 static void cms_signature_okx ( struct cms_test_signature *sgn,
 				const char *file, unsigned int line ) {
+	const void *data = ( ( void * ) sgn->image.data );
 
-	okx ( cms_signature ( sgn->data, sgn->len, &sgn->sig ) == 0,
-	      file, line );
+	/* Fix up image data pointer */
+	sgn->image.data = virt_to_user ( data );
+
+	/* Check ability to parse signature */
+	okx ( cms_signature ( &sgn->image, &sgn->sig ) == 0, file, line );
+
+	/* Reset image data pointer */
+	sgn->image.data = ( ( userptr_t ) data );
 }
 #define cms_signature_ok( sgn ) \
 	cms_signature_okx ( sgn, __FILE__, __LINE__ )
@@ -1377,10 +1391,21 @@ static void cms_verify_okx ( struct cms_test_signature *sgn,
 			     time_t time, struct x509_chain *store,
 			     struct x509_root *root, const char *file,
 			     unsigned int line ) {
+	const void *data = ( ( void * ) code->image.data );
 
+	/* Fix up image data pointer */
+	code->image.data = virt_to_user ( data );
+
+	/* Invalidate any certificates from previous tests */
 	x509_invalidate_chain ( sgn->sig->certificates );
-	okx ( cms_verify ( sgn->sig, virt_to_user ( code->data ), code->len,
-			   name, time, store, root ) == 0, file, line );
+
+	/* Check ability to verify signature */
+	okx ( cms_verify ( sgn->sig, &code->image, name, time, store,
+			   root ) == 0, file, line );
+	okx ( code->image.flags & IMAGE_TRUSTED, file, line );
+
+	/* Reset image data pointer */
+	code->image.data = ( ( userptr_t ) data );
 }
 #define cms_verify_ok( sgn, code, name, time, store, root )		\
 	cms_verify_okx ( sgn, code, name, time, store, root,		\
@@ -1403,10 +1428,21 @@ static void cms_verify_fail_okx ( struct cms_test_signature *sgn,
 				  time_t time, struct x509_chain *store,
 				  struct x509_root *root, const char *file,
 				  unsigned int line ) {
+	const void *data = ( ( void * ) code->image.data );
 
+	/* Fix up image data pointer */
+	code->image.data = virt_to_user ( data );
+
+	/* Invalidate any certificates from previous tests */
 	x509_invalidate_chain ( sgn->sig->certificates );
-	okx ( cms_verify ( sgn->sig, virt_to_user ( code->data ), code->len,
-			   name, time, store, root ) != 0, file, line );
+
+	/* Check inability to verify signature */
+	okx ( cms_verify ( sgn->sig, &code->image, name, time, store,
+			   root ) != 0, file, line );
+	okx ( ! ( code->image.flags & IMAGE_TRUSTED ), file, line );
+
+	/* Reset image data pointer */
+	code->image.data = ( ( userptr_t ) data );
 }
 #define cms_verify_fail_ok( sgn, code, name, time, store, root )	\
 	cms_verify_fail_okx ( sgn, code, name, time, store, root,	\
diff --git a/src/usr/imgtrust.c b/src/usr/imgtrust.c
index e7c2067a0..54ea3378f 100644
--- a/src/usr/imgtrust.c
+++ b/src/usr/imgtrust.c
@@ -50,31 +50,15 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 int imgverify ( struct image *image, struct image *signature,
 		const char *name ) {
-	struct asn1_cursor *data;
 	struct cms_signature *sig;
 	struct cms_signer_info *info;
 	time_t now;
-	int next;
 	int rc;
 
-	/* Mark image as untrusted */
-	image_untrust ( image );
-
-	/* Get raw signature data */
-	next = image_asn1 ( signature, 0, &data );
-	if ( next < 0 ) {
-		rc = next;
-		goto err_asn1;
-	}
-
 	/* Parse signature */
-	if ( ( rc = cms_signature ( data->data, data->len, &sig ) ) != 0 )
+	if ( ( rc = cms_signature ( signature, &sig ) ) != 0 )
 		goto err_parse;
 
-	/* Free raw signature data */
-	free ( data );
-	data = NULL;
-
 	/* Complete all certificate chains */
 	list_for_each_entry ( info, &sig->info, list ) {
 		if ( ( rc = create_validator ( &monojob, info->chain,
@@ -86,16 +70,14 @@ int imgverify ( struct image *image, struct image *signature,
 
 	/* Use signature to verify image */
 	now = time ( NULL );
-	if ( ( rc = cms_verify ( sig, image->data, image->len,
-				 name, now, NULL, NULL ) ) != 0 )
+	if ( ( rc = cms_verify ( sig, image, name, now, NULL, NULL ) ) != 0 )
 		goto err_verify;
 
 	/* Drop reference to signature */
 	cms_put ( sig );
 	sig = NULL;
 
-	/* Mark image as trusted */
-	image_trust ( image );
+	/* Record signature verification */
 	syslog ( LOG_NOTICE, "Image \"%s\" signature OK\n", image->name );
 
 	return 0;
@@ -105,8 +87,6 @@ int imgverify ( struct image *image, struct image *signature,
  err_create_validator:
 	cms_put ( sig );
  err_parse:
-	free ( data );
- err_asn1:
 	syslog ( LOG_ERR, "Image \"%s\" signature bad: %s\n",
 		 image->name, strerror ( rc ) );
 	return rc;

From 998edc6ec515a6c9b0635d728b1cc51253e7dd7f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 14 Aug 2024 13:02:22 +0100
Subject: [PATCH 167/237] [crypto] Add OID-identified algorithms for AES
 ciphers

Extend the definition of an ASN.1 OID-identified algorithm to include
a potential cipher suite, and add identifiers for AES-CBC and AES-GCM.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config_crypto.c        | 10 ++++++
 src/crypto/asn1.c                 | 26 ++++++++++++++
 src/crypto/mishmash/oid_aes_cbc.c | 57 +++++++++++++++++++++++++++++++
 src/crypto/mishmash/oid_aes_gcm.c | 57 +++++++++++++++++++++++++++++++
 src/include/ipxe/asn1.h           | 46 +++++++++++++++++++++++++
 5 files changed, 196 insertions(+)
 create mode 100644 src/crypto/mishmash/oid_aes_cbc.c
 create mode 100644 src/crypto/mishmash/oid_aes_gcm.c

diff --git a/src/config/config_crypto.c b/src/config/config_crypto.c
index 5211224ab..f118a9709 100644
--- a/src/config/config_crypto.c
+++ b/src/config/config_crypto.c
@@ -88,6 +88,16 @@ REQUIRE_OBJECT ( oid_sha512_256 );
 REQUIRE_OBJECT ( oid_x25519 );
 #endif
 
+/* AES-CBC */
+#if defined ( CRYPTO_CIPHER_AES_CBC )
+REQUIRE_OBJECT ( oid_aes_cbc );
+#endif
+
+/* AES-GCM */
+#if defined ( CRYPTO_CIPHER_AES_GCM )
+REQUIRE_OBJECT ( oid_aes_gcm );
+#endif
+
 /* RSA and MD5 */
 #if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_MD5 )
 REQUIRE_OBJECT ( rsa_md5 );
diff --git a/src/crypto/asn1.c b/src/crypto/asn1.c
index e66da45b9..0e5b1a9dd 100644
--- a/src/crypto/asn1.c
+++ b/src/crypto/asn1.c
@@ -591,6 +591,32 @@ int asn1_digest_algorithm ( const struct asn1_cursor *cursor,
 	return 0;
 }
 
+/**
+ * Parse ASN.1 OID-identified cipher algorithm
+ *
+ * @v cursor		ASN.1 object cursor
+ * @ret algorithm	Algorithm
+ * @ret rc		Return status code
+ */
+int asn1_cipher_algorithm ( const struct asn1_cursor *cursor,
+			    struct asn1_algorithm **algorithm ) {
+	int rc;
+
+	/* Parse algorithm */
+	if ( ( rc = asn1_algorithm ( cursor, algorithm ) ) != 0 )
+		return rc;
+
+	/* Check algorithm has a cipher */
+	if ( ! (*algorithm)->cipher ) {
+		DBGC ( cursor, "ASN1 %p algorithm %s is not a cipher "
+		       "algorithm:\n", cursor, (*algorithm)->name );
+		DBGC_HDA ( cursor, 0, cursor->data, cursor->len );
+		return -ENOTTY_ALGORITHM;
+	}
+
+	return 0;
+}
+
 /**
  * Parse ASN.1 OID-identified signature algorithm
  *
diff --git a/src/crypto/mishmash/oid_aes_cbc.c b/src/crypto/mishmash/oid_aes_cbc.c
new file mode 100644
index 000000000..0a7951c63
--- /dev/null
+++ b/src/crypto/mishmash/oid_aes_cbc.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/aes.h>
+#include <ipxe/asn1.h>
+
+/** "aes128-cbc" object identifier */
+static uint8_t oid_aes_128_cbc[] = { ASN1_OID_AES128_CBC };
+
+/** "aes192-cbc" object identifier */
+static uint8_t oid_aes_192_cbc[] = { ASN1_OID_AES192_CBC };
+
+/** "aes256-cbc" object identifier */
+static uint8_t oid_aes_256_cbc[] = { ASN1_OID_AES256_CBC };
+
+/** "aes128-cbc" OID-identified algorithm */
+struct asn1_algorithm aes_128_cbc_algorithm __asn1_algorithm = {
+	.name = "aes128-cbc",
+	.cipher = &aes_cbc_algorithm,
+	.oid = ASN1_CURSOR ( oid_aes_128_cbc ),
+};
+
+/** "aes192-cbc" OID-identified algorithm */
+struct asn1_algorithm aes_192_cbc_algorithm __asn1_algorithm = {
+	.name = "aes192-cbc",
+	.cipher = &aes_cbc_algorithm,
+	.oid = ASN1_CURSOR ( oid_aes_192_cbc ),
+};
+
+/** "aes256-cbc" OID-identified algorithm */
+struct asn1_algorithm aes_256_cbc_algorithm __asn1_algorithm = {
+	.name = "aes256-cbc",
+	.cipher = &aes_cbc_algorithm,
+	.oid = ASN1_CURSOR ( oid_aes_256_cbc ),
+};
diff --git a/src/crypto/mishmash/oid_aes_gcm.c b/src/crypto/mishmash/oid_aes_gcm.c
new file mode 100644
index 000000000..0f94d5323
--- /dev/null
+++ b/src/crypto/mishmash/oid_aes_gcm.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/aes.h>
+#include <ipxe/asn1.h>
+
+/** "aes128-gcm" object identifier */
+static uint8_t oid_aes_128_gcm[] = { ASN1_OID_AES128_GCM };
+
+/** "aes192-gcm" object identifier */
+static uint8_t oid_aes_192_gcm[] = { ASN1_OID_AES192_GCM };
+
+/** "aes256-gcm" object identifier */
+static uint8_t oid_aes_256_gcm[] = { ASN1_OID_AES256_GCM };
+
+/** "aes128-gcm" OID-identified algorithm */
+struct asn1_algorithm aes_128_gcm_algorithm __asn1_algorithm = {
+	.name = "aes128-gcm",
+	.cipher = &aes_gcm_algorithm,
+	.oid = ASN1_CURSOR ( oid_aes_128_gcm ),
+};
+
+/** "aes192-gcm" OID-identified algorithm */
+struct asn1_algorithm aes_192_gcm_algorithm __asn1_algorithm = {
+	.name = "aes192-gcm",
+	.cipher = &aes_gcm_algorithm,
+	.oid = ASN1_CURSOR ( oid_aes_192_gcm ),
+};
+
+/** "aes256-gcm" OID-identified algorithm */
+struct asn1_algorithm aes_256_gcm_algorithm __asn1_algorithm = {
+	.name = "aes256-gcm",
+	.cipher = &aes_gcm_algorithm,
+	.oid = ASN1_CURSOR ( oid_aes_256_gcm ),
+};
diff --git a/src/include/ipxe/asn1.h b/src/include/ipxe/asn1.h
index 1580c8baf..fd7244570 100644
--- a/src/include/ipxe/asn1.h
+++ b/src/include/ipxe/asn1.h
@@ -192,6 +192,48 @@ struct asn1_builder_header {
 	ASN1_OID_INITIAL ( 1, 3 ), ASN1_OID_SINGLE ( 101 ),	\
 	ASN1_OID_SINGLE ( 110 )
 
+/** ASN.1 OID for id-aes128-cbc (2.16.840.1.101.3.4.1.2) */
+#define ASN1_OID_AES128_CBC					\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 2 )
+
+/** ASN.1 OID for id-aes128-gcm (2.16.840.1.101.3.4.1.6) */
+#define ASN1_OID_AES128_GCM					\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 6 )
+
+/** ASN.1 OID for id-aes192-cbc (2.16.840.1.101.3.4.1.22) */
+#define ASN1_OID_AES192_CBC					\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 22 )
+
+/** ASN.1 OID for id-aes192-gcm (2.16.840.1.101.3.4.1.26) */
+#define ASN1_OID_AES192_GCM					\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 26 )
+
+/** ASN.1 OID for id-aes256-cbc (2.16.840.1.101.3.4.1.42) */
+#define ASN1_OID_AES256_CBC					\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 42 )
+
+/** ASN.1 OID for id-aes256-gcm (2.16.840.1.101.3.4.1.46) */
+#define ASN1_OID_AES256_GCM					\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 46 )
+
 /** ASN.1 OID for id-sha256 (2.16.840.1.101.3.4.2.1) */
 #define ASN1_OID_SHA256						\
 	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
@@ -317,6 +359,8 @@ struct asn1_algorithm {
 	struct pubkey_algorithm *pubkey;
 	/** Digest algorithm (if applicable) */
 	struct digest_algorithm *digest;
+	/** Cipher algorithm (if applicable) */
+	struct cipher_algorithm *cipher;
 	/** Elliptic curve (if applicable) */
 	struct elliptic_curve *curve;
 };
@@ -428,6 +472,8 @@ extern int asn1_pubkey_algorithm ( const struct asn1_cursor *cursor,
 				   struct asn1_algorithm **algorithm );
 extern int asn1_digest_algorithm ( const struct asn1_cursor *cursor,
 				   struct asn1_algorithm **algorithm );
+extern int asn1_cipher_algorithm ( const struct asn1_cursor *cursor,
+				   struct asn1_algorithm **algorithm );
 extern int asn1_signature_algorithm ( const struct asn1_cursor *cursor,
 				      struct asn1_algorithm **algorithm );
 extern int asn1_check_algorithm ( const struct asn1_cursor *cursor,

From 97635eb71b5ad7e81e79f32fef5f4394bcee0722 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 12 Aug 2024 12:36:41 +0100
Subject: [PATCH 168/237] [crypto] Generalise cms_signature to cms_message

There is some exploitable similarity between the data structures used
for representing CMS signatures and CMS encryption keys.  In both
cases, the CMS message fundamentally encodes a list of participants
(either message signers or message recipients), where each participant
has an associated certificate and an opaque octet string representing
the signature or encrypted cipher key.  The ASN.1 structures are not
identical, but are sufficiently similar to be worth exploiting: for
example, the SignerIdentifier and RecipientIdentifier data structures
are defined identically.

Rename data structures and functions, and add the concept of a CMS
message type.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/cms.c        | 519 ++++++++++++++++++++++------------------
 src/include/ipxe/asn1.h |   2 +-
 src/include/ipxe/cms.h  |  87 ++++---
 src/tests/cms_test.c    |  24 +-
 src/usr/imgtrust.c      |  20 +-
 5 files changed, 366 insertions(+), 286 deletions(-)

diff --git a/src/crypto/cms.c b/src/crypto/cms.c
index cbc0736bb..1f33613f4 100644
--- a/src/crypto/cms.c
+++ b/src/crypto/cms.c
@@ -59,60 +59,68 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	__einfo_error ( EINFO_EACCES_NO_SIGNATURES )
 #define EINFO_EACCES_NO_SIGNATURES \
 	__einfo_uniqify ( EINFO_EACCES, 0x05, "No signatures present" )
-#define EINVAL_DIGEST \
-	__einfo_error ( EINFO_EINVAL_DIGEST )
-#define EINFO_EINVAL_DIGEST \
-	__einfo_uniqify ( EINFO_EINVAL, 0x01, "Not a digest algorithm" )
-#define EINVAL_PUBKEY \
-	__einfo_error ( EINFO_EINVAL_PUBKEY )
-#define EINFO_EINVAL_PUBKEY \
-	__einfo_uniqify ( EINFO_EINVAL, 0x02, "Not a public-key algorithm" )
-#define ENOTSUP_SIGNEDDATA \
-	__einfo_error ( EINFO_ENOTSUP_SIGNEDDATA )
-#define EINFO_ENOTSUP_SIGNEDDATA \
-	__einfo_uniqify ( EINFO_ENOTSUP, 0x01, "Not a digital signature" )
+#define ENOTSUP_TYPE \
+	__einfo_error ( EINFO_ENOTSUP_TYPE )
+#define EINFO_ENOTSUP_TYPE \
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01, "Unrecognised message type" )
 
-/** "pkcs7-signedData" object identifier */
+static int cms_parse_signed ( struct cms_message *cms,
+			      const struct asn1_cursor *raw );
+
+/** "id-signedData" object identifier */
 static uint8_t oid_signeddata[] = { ASN1_OID_SIGNEDDATA };
 
-/** "pkcs7-signedData" object identifier cursor */
-static struct asn1_cursor oid_signeddata_cursor =
-	ASN1_CURSOR ( oid_signeddata );
+/** CMS message types */
+static struct cms_type cms_types[] = {
+	{
+		.name = "signed",
+		.oid = ASN1_CURSOR ( oid_signeddata ),
+		.parse = cms_parse_signed,
+	},
+};
 
 /**
- * Parse CMS signature content type
+ * Parse CMS message content type
  *
- * @v sig		CMS signature
+ * @v cms		CMS message
  * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse_content_type ( struct cms_signature *sig,
+static int cms_parse_content_type ( struct cms_message *cms,
 				    const struct asn1_cursor *raw ) {
 	struct asn1_cursor cursor;
+	struct cms_type *type;
+	unsigned int i;
 
 	/* Enter contentType */
 	memcpy ( &cursor, raw, sizeof ( cursor ) );
 	asn1_enter ( &cursor, ASN1_OID );
 
-	/* Check OID is pkcs7-signedData */
-	if ( asn1_compare ( &cursor, &oid_signeddata_cursor ) != 0 ) {
-		DBGC ( sig, "CMS %p does not contain signedData:\n", sig );
-		DBGC_HDA ( sig, 0, raw->data, raw->len );
-		return -ENOTSUP_SIGNEDDATA;
+	/* Check for a recognised OID */
+	for ( i = 0 ; i < ( sizeof ( cms_types ) /
+			    sizeof ( cms_types[0] ) ) ; i++ ) {
+		type = &cms_types[i];
+		if ( asn1_compare ( &cursor, &type->oid ) == 0 ) {
+			cms->type = type;
+			DBGC ( cms, "CMS %p contains %sData\n",
+			       cms, type->name );
+			return 0;
+		}
 	}
 
-	DBGC ( sig, "CMS %p contains signedData\n", sig );
-	return 0;
+	DBGC ( cms, "CMS %p is not a recognised message type:\n", cms );
+	DBGC_HDA ( cms, 0, raw->data, raw->len );
+	return -ENOTSUP_TYPE;
 }
 
 /**
- * Parse CMS signature certificate list
+ * Parse CMS message certificate list
  *
- * @v sig		CMS signature
+ * @v cms		CMS message
  * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse_certificates ( struct cms_signature *sig,
+static int cms_parse_certificates ( struct cms_message *cms,
 				    const struct asn1_cursor *raw ) {
 	struct asn1_cursor cursor;
 	struct x509_certificate *cert;
@@ -126,16 +134,16 @@ static int cms_parse_certificates ( struct cms_signature *sig,
 	while ( cursor.len ) {
 
 		/* Add certificate to chain */
-		if ( ( rc = x509_append_raw ( sig->certificates, cursor.data,
+		if ( ( rc = x509_append_raw ( cms->certificates, cursor.data,
 					      cursor.len ) ) != 0 ) {
-			DBGC ( sig, "CMS %p could not append certificate: %s\n",
-			       sig, strerror ( rc) );
-			DBGC_HDA ( sig, 0, cursor.data, cursor.len );
+			DBGC ( cms, "CMS %p could not append certificate: %s\n",
+			       cms, strerror ( rc) );
+			DBGC_HDA ( cms, 0, cursor.data, cursor.len );
 			return rc;
 		}
-		cert = x509_last ( sig->certificates );
-		DBGC ( sig, "CMS %p found certificate %s\n",
-		       sig, x509_name ( cert ) );
+		cert = x509_last ( cms->certificates );
+		DBGC ( cms, "CMS %p found certificate %s\n",
+		       cms, x509_name ( cert ) );
 
 		/* Move to next certificate */
 		asn1_skip_any ( &cursor );
@@ -145,16 +153,16 @@ static int cms_parse_certificates ( struct cms_signature *sig,
 }
 
 /**
- * Parse CMS signature signer identifier
+ * Parse CMS message participant identifier
  *
- * @v sig		CMS signature
- * @v info		Signer information to fill in
+ * @v cms		CMS message
+ * @v part		Participant information to fill in
  * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse_signer_identifier ( struct cms_signature *sig,
-					 struct cms_signer_info *info,
-					 const struct asn1_cursor *raw ) {
+static int cms_parse_identifier ( struct cms_message *cms,
+				  struct cms_participant *part,
+				  const struct asn1_cursor *raw ) {
 	struct asn1_cursor cursor;
 	struct asn1_cursor serial;
 	struct asn1_cursor issuer;
@@ -168,46 +176,46 @@ static int cms_parse_signer_identifier ( struct cms_signature *sig,
 	/* Identify issuer */
 	memcpy ( &issuer, &cursor, sizeof ( issuer ) );
 	if ( ( rc = asn1_shrink ( &issuer, ASN1_SEQUENCE ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not locate issuer: %s\n",
-		       sig, info, strerror ( rc ) );
-		DBGC_HDA ( sig, 0, raw->data, raw->len );
+		DBGC ( cms, "CMS %p/%p could not locate issuer: %s\n",
+		       cms, part, strerror ( rc ) );
+		DBGC_HDA ( cms, 0, raw->data, raw->len );
 		return rc;
 	}
-	DBGC ( sig, "CMS %p/%p issuer is:\n", sig, info );
-	DBGC_HDA ( sig, 0, issuer.data, issuer.len );
+	DBGC ( cms, "CMS %p/%p issuer is:\n", cms, part );
+	DBGC_HDA ( cms, 0, issuer.data, issuer.len );
 	asn1_skip_any ( &cursor );
 
 	/* Identify serialNumber */
 	memcpy ( &serial, &cursor, sizeof ( serial ) );
 	if ( ( rc = asn1_shrink ( &serial, ASN1_INTEGER ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not locate serialNumber: %s\n",
-		       sig, info, strerror ( rc ) );
-		DBGC_HDA ( sig, 0, raw->data, raw->len );
+		DBGC ( cms, "CMS %p/%p could not locate serialNumber: %s\n",
+		       cms, part, strerror ( rc ) );
+		DBGC_HDA ( cms, 0, raw->data, raw->len );
 		return rc;
 	}
-	DBGC ( sig, "CMS %p/%p serial number is:\n", sig, info );
-	DBGC_HDA ( sig, 0, serial.data, serial.len );
+	DBGC ( cms, "CMS %p/%p serial number is:\n", cms, part );
+	DBGC_HDA ( cms, 0, serial.data, serial.len );
 
 	/* Identify certificate */
-	cert = x509_find_issuer_serial ( sig->certificates, &issuer, &serial );
+	cert = x509_find_issuer_serial ( cms->certificates, &issuer, &serial );
 	if ( ! cert ) {
-		DBGC ( sig, "CMS %p/%p could not identify signer's "
-		       "certificate\n", sig, info );
+		DBGC ( cms, "CMS %p/%p could not identify certificate\n",
+		       cms, part );
 		return -ENOENT;
 	}
 
 	/* Append certificate to chain */
-	if ( ( rc = x509_append ( info->chain, cert ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not append certificate: %s\n",
-		       sig, info, strerror ( rc ) );
+	if ( ( rc = x509_append ( part->chain, cert ) ) != 0 ) {
+		DBGC ( cms, "CMS %p/%p could not append certificate: %s\n",
+		       cms, part, strerror ( rc ) );
 		return rc;
 	}
 
 	/* Append remaining certificates to chain */
-	if ( ( rc = x509_auto_append ( info->chain,
-				       sig->certificates ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not append certificates: %s\n",
-		       sig, info, strerror ( rc ) );
+	if ( ( rc = x509_auto_append ( part->chain,
+				       cms->certificates ) ) != 0 ) {
+		DBGC ( cms, "CMS %p/%p could not append certificates: %s\n",
+		       cms, part, strerror ( rc ) );
 		return rc;
 	}
 
@@ -215,110 +223,110 @@ static int cms_parse_signer_identifier ( struct cms_signature *sig,
 }
 
 /**
- * Parse CMS signature digest algorithm
+ * Parse CMS message digest algorithm
  *
- * @v sig		CMS signature
- * @v info		Signer information to fill in
+ * @v cms		CMS message
+ * @v part		Participant information to fill in
  * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse_digest_algorithm ( struct cms_signature *sig,
-					struct cms_signer_info *info,
+static int cms_parse_digest_algorithm ( struct cms_message *cms,
+					struct cms_participant *part,
 					const struct asn1_cursor *raw ) {
 	struct asn1_algorithm *algorithm;
 	int rc;
 
 	/* Identify algorithm */
 	if ( ( rc = asn1_digest_algorithm ( raw, &algorithm ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not identify digest algorithm: "
-		       "%s\n", sig, info, strerror ( rc ) );
-		DBGC_HDA ( sig, 0, raw->data, raw->len );
+		DBGC ( cms, "CMS %p/%p could not identify digest algorithm: "
+		       "%s\n", cms, part, strerror ( rc ) );
+		DBGC_HDA ( cms, 0, raw->data, raw->len );
 		return rc;
 	}
 
 	/* Record digest algorithm */
-	info->digest = algorithm->digest;
-	DBGC ( sig, "CMS %p/%p digest algorithm is %s\n",
-	       sig, info, algorithm->name );
+	part->digest = algorithm->digest;
+	DBGC ( cms, "CMS %p/%p digest algorithm is %s\n",
+	       cms, part, algorithm->name );
 
 	return 0;
 }
 
 /**
- * Parse CMS signature algorithm
+ * Parse CMS message public-key algorithm
  *
- * @v sig		CMS signature
- * @v info		Signer information to fill in
+ * @v cms		CMS message
+ * @v part		Participant information to fill in
  * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse_signature_algorithm ( struct cms_signature *sig,
-					   struct cms_signer_info *info,
-					   const struct asn1_cursor *raw ) {
+static int cms_parse_pubkey_algorithm ( struct cms_message *cms,
+					struct cms_participant *part,
+					const struct asn1_cursor *raw ) {
 	struct asn1_algorithm *algorithm;
 	int rc;
 
 	/* Identify algorithm */
 	if ( ( rc = asn1_pubkey_algorithm ( raw, &algorithm ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not identify public-key "
-		       "algorithm: %s\n", sig, info, strerror ( rc ) );
-		DBGC_HDA ( sig, 0, raw->data, raw->len );
+		DBGC ( cms, "CMS %p/%p could not identify public-key "
+		       "algorithm: %s\n", cms, part, strerror ( rc ) );
+		DBGC_HDA ( cms, 0, raw->data, raw->len );
 		return rc;
 	}
 
-	/* Record signature algorithm */
-	info->pubkey = algorithm->pubkey;
-	DBGC ( sig, "CMS %p/%p public-key algorithm is %s\n",
-	       sig, info, algorithm->name );
+	/* Record public-key algorithm */
+	part->pubkey = algorithm->pubkey;
+	DBGC ( cms, "CMS %p/%p public-key algorithm is %s\n",
+	       cms, part, algorithm->name );
 
 	return 0;
 }
 
 /**
- * Parse CMS signature value
+ * Parse CMS message signature or key value
  *
- * @v sig		CMS signature
- * @v info		Signer information to fill in
+ * @v cms		CMS message
+ * @v part		Participant information to fill in
  * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse_signature_value ( struct cms_signature *sig,
-				       struct cms_signer_info *info,
-				       const struct asn1_cursor *raw ) {
+static int cms_parse_value ( struct cms_message *cms,
+			     struct cms_participant *part,
+			     const struct asn1_cursor *raw ) {
 	struct asn1_cursor cursor;
 	int rc;
 
 	/* Enter signature */
 	memcpy ( &cursor, raw, sizeof ( cursor ) );
 	if ( ( rc = asn1_enter ( &cursor, ASN1_OCTET_STRING ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not locate signature:\n",
-		       sig, info );
-		DBGC_HDA ( sig, 0, raw->data, raw->len );
+		DBGC ( cms, "CMS %p/%p could not locate value:\n",
+		       cms, part );
+		DBGC_HDA ( cms, 0, raw->data, raw->len );
 		return rc;
 	}
 
 	/* Record signature */
-	info->signature_len = cursor.len;
-	info->signature = malloc ( info->signature_len );
-	if ( ! info->signature )
+	part->len = cursor.len;
+	part->value = malloc ( part->len );
+	if ( ! part->value )
 		return -ENOMEM;
-	memcpy ( info->signature, cursor.data, info->signature_len );
-	DBGC ( sig, "CMS %p/%p signature value is:\n", sig, info );
-	DBGC_HDA ( sig, 0, info->signature, info->signature_len );
+	memcpy ( part->value, cursor.data, part->len );
+	DBGC ( cms, "CMS %p/%p value is:\n", cms, part );
+	DBGC_HDA ( cms, 0, part->value, part->len );
 
 	return 0;
 }
 
 /**
- * Parse CMS signature signer information
+ * Parse CMS message participant information
  *
- * @v sig		CMS signature
- * @v info		Signer information to fill in
+ * @v cms		CMS message
+ * @v part		Participant information to fill in
  * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse_signer_info ( struct cms_signature *sig,
-				   struct cms_signer_info *info,
+static int cms_parse_participant ( struct cms_message *cms,
+				   struct cms_participant *part,
 				   const struct asn1_cursor *raw ) {
 	struct asn1_cursor cursor;
 	int rc;
@@ -331,12 +339,13 @@ static int cms_parse_signer_info ( struct cms_signature *sig,
 	asn1_skip ( &cursor, ASN1_INTEGER );
 
 	/* Parse sid */
-	if ( ( rc = cms_parse_signer_identifier ( sig, info, &cursor ) ) != 0 )
+	if ( ( rc = cms_parse_identifier ( cms, part, &cursor ) ) != 0 )
 		return rc;
 	asn1_skip_any ( &cursor );
 
 	/* Parse digestAlgorithm */
-	if ( ( rc = cms_parse_digest_algorithm ( sig, info, &cursor ) ) != 0 )
+	if ( ( rc = cms_parse_digest_algorithm ( cms, part,
+						 &cursor ) ) != 0 )
 		return rc;
 	asn1_skip_any ( &cursor );
 
@@ -344,43 +353,79 @@ static int cms_parse_signer_info ( struct cms_signature *sig,
 	asn1_skip_if_exists ( &cursor, ASN1_EXPLICIT_TAG ( 0 ) );
 
 	/* Parse signatureAlgorithm */
-	if ( ( rc = cms_parse_signature_algorithm ( sig, info, &cursor ) ) != 0)
+	if ( ( rc = cms_parse_pubkey_algorithm ( cms, part, &cursor ) ) != 0 )
 		return rc;
 	asn1_skip_any ( &cursor );
 
 	/* Parse signature */
-	if ( ( rc = cms_parse_signature_value ( sig, info, &cursor ) ) != 0 )
+	if ( ( rc = cms_parse_value ( cms, part, &cursor ) ) != 0 )
 		return rc;
 
 	return 0;
 }
 
 /**
- * Parse CMS signature from ASN.1 data
+ * Parse CMS message participants information
  *
- * @v sig		CMS signature
+ * @v cms		CMS message
  * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse ( struct cms_signature *sig,
-		       const struct asn1_cursor *raw ) {
+static int cms_parse_participants ( struct cms_message *cms,
+				    const struct asn1_cursor *raw ) {
 	struct asn1_cursor cursor;
-	struct cms_signer_info *info;
+	struct cms_participant *part;
 	int rc;
 
-	/* Enter contentInfo */
+	/* Enter signerInfos */
 	memcpy ( &cursor, raw, sizeof ( cursor ) );
-	asn1_enter ( &cursor, ASN1_SEQUENCE );
+	asn1_enter ( &cursor, ASN1_SET );
 
-	/* Parse contentType */
-	if ( ( rc = cms_parse_content_type ( sig, &cursor ) ) != 0 )
-		return rc;
-	asn1_skip_any ( &cursor );
+	/* Add each signerInfo.  Errors are handled by ensuring that
+	 * cms_put() will always be able to free any allocated memory.
+	 */
+	while ( cursor.len ) {
 
-	/* Enter content */
-	asn1_enter ( &cursor, ASN1_EXPLICIT_TAG ( 0 ) );
+		/* Allocate participant information block */
+		part = zalloc ( sizeof ( *part ) );
+		if ( ! part )
+			return -ENOMEM;
+		list_add ( &part->list, &cms->participants );
+
+		/* Allocate certificate chain */
+		part->chain = x509_alloc_chain();
+		if ( ! part->chain )
+			return -ENOMEM;
+
+		/* Parse signerInfo */
+		if ( ( rc = cms_parse_participant ( cms, part,
+						    &cursor ) ) != 0 )
+			return rc;
+		asn1_skip_any ( &cursor );
+	}
+
+	return 0;
+}
+
+/**
+ * Parse CMS signed data
+ *
+ * @v cms		CMS message
+ * @v raw		ASN.1 cursor
+ * @ret rc		Return status code
+ */
+static int cms_parse_signed ( struct cms_message *cms,
+			      const struct asn1_cursor *raw ) {
+	struct asn1_cursor cursor;
+	int rc;
+
+	/* Allocate certificate list */
+	cms->certificates = x509_alloc_chain();
+	if ( ! cms->certificates )
+		return -ENOMEM;
 
 	/* Enter signedData */
+	memcpy ( &cursor, raw, sizeof ( cursor ) );
 	asn1_enter ( &cursor, ASN1_SEQUENCE );
 
 	/* Skip version */
@@ -393,111 +438,113 @@ static int cms_parse ( struct cms_signature *sig,
 	asn1_skip ( &cursor, ASN1_SEQUENCE );
 
 	/* Parse certificates */
-	if ( ( rc = cms_parse_certificates ( sig, &cursor ) ) != 0 )
+	if ( ( rc = cms_parse_certificates ( cms, &cursor ) ) != 0 )
 		return rc;
 	asn1_skip_any ( &cursor );
 
 	/* Skip crls, if present */
 	asn1_skip_if_exists ( &cursor, ASN1_EXPLICIT_TAG ( 1 ) );
 
-	/* Enter signerInfos */
-	asn1_enter ( &cursor, ASN1_SET );
-
-	/* Add each signerInfo.  Errors are handled by ensuring that
-	 * cms_put() will always be able to free any allocated memory.
-	 */
-	while ( cursor.len ) {
-
-		/* Allocate signer information block */
-		info = zalloc ( sizeof ( *info ) );
-		if ( ! info )
-			return -ENOMEM;
-		list_add ( &info->list, &sig->info );
-
-		/* Allocate certificate chain */
-		info->chain = x509_alloc_chain();
-		if ( ! info->chain )
-			return -ENOMEM;
-
-		/* Parse signerInfo */
-		if ( ( rc = cms_parse_signer_info ( sig, info,
-						    &cursor ) ) != 0 )
-			return rc;
-		asn1_skip_any ( &cursor );
-	}
+	/* Parse signerInfos */
+	if ( ( rc = cms_parse_participants ( cms, &cursor ) ) != 0 )
+		return rc;
 
 	return 0;
 }
 
 /**
- * Free CMS signature
+ * Parse CMS message from ASN.1 data
+ *
+ * @v cms		CMS message
+ * @v raw		ASN.1 cursor
+ * @ret rc		Return status code
+ */
+static int cms_parse ( struct cms_message *cms,
+		       const struct asn1_cursor *raw ) {
+	struct asn1_cursor cursor;
+	int rc;
+
+	/* Enter contentInfo */
+	memcpy ( &cursor, raw, sizeof ( cursor ) );
+	asn1_enter ( &cursor, ASN1_SEQUENCE );
+
+	/* Parse contentType */
+	if ( ( rc = cms_parse_content_type ( cms, &cursor ) ) != 0 )
+		return rc;
+	asn1_skip_any ( &cursor );
+
+	/* Enter content */
+	asn1_enter ( &cursor, ASN1_EXPLICIT_TAG ( 0 ) );
+
+	/* Parse type-specific content */
+	if ( ( rc = cms->type->parse ( cms, &cursor ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Free CMS message
  *
  * @v refcnt		Reference count
  */
 static void cms_free ( struct refcnt *refcnt ) {
-	struct cms_signature *sig =
-		container_of ( refcnt, struct cms_signature, refcnt );
-	struct cms_signer_info *info;
-	struct cms_signer_info *tmp;
+	struct cms_message *cms =
+		container_of ( refcnt, struct cms_message, refcnt );
+	struct cms_participant *part;
+	struct cms_participant *tmp;
 
-	list_for_each_entry_safe ( info, tmp, &sig->info, list ) {
-		list_del ( &info->list );
-		x509_chain_put ( info->chain );
-		free ( info->signature );
-		free ( info );
+	list_for_each_entry_safe ( part, tmp, &cms->participants, list ) {
+		list_del ( &part->list );
+		x509_chain_put ( part->chain );
+		free ( part->value );
+		free ( part );
 	}
-	x509_chain_put ( sig->certificates );
-	free ( sig );
+	x509_chain_put ( cms->certificates );
+	free ( cms );
 }
 
 /**
- * Create CMS signature
+ * Create CMS message
  *
  * @v image		Image
- * @ret sig		CMS signature
+ * @ret sig		CMS message
  * @ret rc		Return status code
  *
- * On success, the caller holds a reference to the CMS signature, and
+ * On success, the caller holds a reference to the CMS message, and
  * is responsible for ultimately calling cms_put().
  */
-int cms_signature ( struct image *image, struct cms_signature **sig ) {
+int cms_message ( struct image *image, struct cms_message **cms ) {
 	struct asn1_cursor *raw;
 	int next;
 	int rc;
 
-	/* Allocate and initialise signature */
-	*sig = zalloc ( sizeof ( **sig ) );
-	if ( ! *sig ) {
+	/* Allocate and initialise message */
+	*cms = zalloc ( sizeof ( **cms ) );
+	if ( ! *cms ) {
 		rc = -ENOMEM;
 		goto err_alloc;
 	}
-	ref_init ( &(*sig)->refcnt, cms_free );
-	INIT_LIST_HEAD ( &(*sig)->info );
+	ref_init ( &(*cms)->refcnt, cms_free );
+	INIT_LIST_HEAD ( &(*cms)->participants );
 
-	/* Allocate certificate list */
-	(*sig)->certificates = x509_alloc_chain();
-	if ( ! (*sig)->certificates ) {
-		rc = -ENOMEM;
-		goto err_alloc_chain;
-	}
-
-	/* Get raw signature data */
+	/* Get raw message data */
 	next = image_asn1 ( image, 0, &raw );
 	if ( next < 0 ) {
 		rc = next;
-		DBGC ( *sig, "CMS %p could not get raw ASN.1 data: %s\n",
-		       *sig, strerror ( rc ) );
+		DBGC ( *cms, "CMS %p could not get raw ASN.1 data: %s\n",
+		       *cms, strerror ( rc ) );
 		goto err_asn1;
 	}
 
-	/* Use only first signature in image */
+	/* Use only first message in image */
 	asn1_shrink_any ( raw );
 
-	/* Parse signature */
-	if ( ( rc = cms_parse ( *sig, raw ) ) != 0 )
+	/* Parse message */
+	if ( ( rc = cms_parse ( *cms, raw ) ) != 0 )
 		goto err_parse;
 
-	/* Free raw signature data */
+	/* Free raw message data */
 	free ( raw );
 
 	return 0;
@@ -505,8 +552,7 @@ int cms_signature ( struct image *image, struct cms_signature **sig ) {
  err_parse:
 	free ( raw );
  err_asn1:
- err_alloc_chain:
-	cms_put ( *sig );
+	cms_put ( *cms );
  err_alloc:
 	return rc;
 }
@@ -514,16 +560,16 @@ int cms_signature ( struct image *image, struct cms_signature **sig ) {
 /**
  * Calculate digest of CMS-signed data
  *
- * @v sig		CMS signature
- * @v info		Signer information
+ * @v cms		CMS message
+ * @v part		Participant information
  * @v data		Signed data
  * @v len		Length of signed data
  * @v out		Digest output
  */
-static void cms_digest ( struct cms_signature *sig,
-			 struct cms_signer_info *info,
+static void cms_digest ( struct cms_message *cms,
+			 struct cms_participant *part,
 			 userptr_t data, size_t len, void *out ) {
-	struct digest_algorithm *digest = info->digest;
+	struct digest_algorithm *digest = part->digest;
 	uint8_t ctx[ digest->ctxsize ];
 	uint8_t block[ digest->blocksize ];
 	size_t offset = 0;
@@ -546,48 +592,47 @@ static void cms_digest ( struct cms_signature *sig,
 	/* Finalise digest */
 	digest_final ( digest, ctx, out );
 
-	DBGC ( sig, "CMS %p/%p digest value:\n", sig, info );
-	DBGC_HDA ( sig, 0, out, digest->digestsize );
+	DBGC ( cms, "CMS %p/%p digest value:\n", cms, part );
+	DBGC_HDA ( cms, 0, out, digest->digestsize );
 }
 
 /**
  * Verify digest of CMS-signed data
  *
- * @v sig		CMS signature
- * @v info		Signer information
+ * @v cms		CMS message
+ * @v part		Participant information
  * @v cert		Corresponding certificate
  * @v data		Signed data
  * @v len		Length of signed data
  * @ret rc		Return status code
  */
-static int cms_verify_digest ( struct cms_signature *sig,
-			       struct cms_signer_info *info,
+static int cms_verify_digest ( struct cms_message *cms,
+			       struct cms_participant *part,
 			       struct x509_certificate *cert,
 			       userptr_t data, size_t len ) {
-	struct digest_algorithm *digest = info->digest;
-	struct pubkey_algorithm *pubkey = info->pubkey;
+	struct digest_algorithm *digest = part->digest;
+	struct pubkey_algorithm *pubkey = part->pubkey;
 	struct x509_public_key *public_key = &cert->subject.public_key;
 	uint8_t digest_out[ digest->digestsize ];
 	uint8_t ctx[ pubkey->ctxsize ];
 	int rc;
 
 	/* Generate digest */
-	cms_digest ( sig, info, data, len, digest_out );
+	cms_digest ( cms, part, data, len, digest_out );
 
 	/* Initialise public-key algorithm */
 	if ( ( rc = pubkey_init ( pubkey, ctx, public_key->raw.data,
 				  public_key->raw.len ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not initialise public key: %s\n",
-		       sig, info, strerror ( rc ) );
+		DBGC ( cms, "CMS %p/%p could not initialise public key: %s\n",
+		       cms, part, strerror ( rc ) );
 		goto err_init;
 	}
 
 	/* Verify digest */
 	if ( ( rc = pubkey_verify ( pubkey, ctx, digest, digest_out,
-				    info->signature,
-				    info->signature_len ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p signature verification failed: %s\n",
-		       sig, info, strerror ( rc ) );
+				    part->value, part->len ) ) != 0 ) {
+		DBGC ( cms, "CMS %p/%p signature verification failed: %s\n",
+		       cms, part, strerror ( rc ) );
 		goto err_verify;
 	}
 
@@ -598,10 +643,10 @@ static int cms_verify_digest ( struct cms_signature *sig,
 }
 
 /**
- * Verify CMS signature signer information
+ * Verify CMS message signer
  *
- * @v sig		CMS signature
- * @v info		Signer information
+ * @v cms		CMS message
+ * @v part		Participant information
  * @v data		Signed data
  * @v len		Length of signed data
  * @v time		Time at which to validate certificates
@@ -609,42 +654,42 @@ static int cms_verify_digest ( struct cms_signature *sig,
  * @v root		Root certificate list, or NULL to use default
  * @ret rc		Return status code
  */
-static int cms_verify_signer_info ( struct cms_signature *sig,
-				    struct cms_signer_info *info,
-				    userptr_t data, size_t len,
-				    time_t time, struct x509_chain *store,
-				    struct x509_root *root ) {
+static int cms_verify_signer ( struct cms_message *cms,
+			       struct cms_participant *part,
+			       userptr_t data, size_t len,
+			       time_t time, struct x509_chain *store,
+			       struct x509_root *root ) {
 	struct x509_certificate *cert;
 	int rc;
 
 	/* Validate certificate chain */
-	if ( ( rc = x509_validate_chain ( info->chain, time, store,
+	if ( ( rc = x509_validate_chain ( part->chain, time, store,
 					  root ) ) != 0 ) {
-		DBGC ( sig, "CMS %p/%p could not validate chain: %s\n",
-		       sig, info, strerror ( rc ) );
+		DBGC ( cms, "CMS %p/%p could not validate chain: %s\n",
+		       cms, part, strerror ( rc ) );
 		return rc;
 	}
 
 	/* Extract code-signing certificate */
-	cert = x509_first ( info->chain );
+	cert = x509_first ( part->chain );
 	assert ( cert != NULL );
 
 	/* Check that certificate can create digital signatures */
 	if ( ! ( cert->extensions.usage.bits & X509_DIGITAL_SIGNATURE ) ) {
-		DBGC ( sig, "CMS %p/%p certificate cannot create signatures\n",
-		       sig, info );
+		DBGC ( cms, "CMS %p/%p certificate cannot create signatures\n",
+		       cms, part );
 		return -EACCES_NON_SIGNING;
 	}
 
 	/* Check that certificate can sign code */
 	if ( ! ( cert->extensions.ext_usage.bits & X509_CODE_SIGNING ) ) {
-		DBGC ( sig, "CMS %p/%p certificate is not code-signing\n",
-		       sig, info );
+		DBGC ( cms, "CMS %p/%p certificate is not code-signing\n",
+		       cms, part );
 		return -EACCES_NON_CODE_SIGNING;
 	}
 
 	/* Verify digest */
-	if ( ( rc = cms_verify_digest ( sig, info, cert, data, len ) ) != 0 )
+	if ( ( rc = cms_verify_digest ( cms, part, cert, data, len ) ) != 0 )
 		return rc;
 
 	return 0;
@@ -653,7 +698,7 @@ static int cms_verify_signer_info ( struct cms_signature *sig,
 /**
  * Verify CMS signature
  *
- * @v sig		CMS signature
+ * @v cms		CMS message
  * @v image		Signed image
  * @v name		Required common name, or NULL to check all signatures
  * @v time		Time at which to validate certificates
@@ -661,10 +706,10 @@ static int cms_verify_signer_info ( struct cms_signature *sig,
  * @v root		Root certificate list, or NULL to use default
  * @ret rc		Return status code
  */
-int cms_verify ( struct cms_signature *sig, struct image *image,
+int cms_verify ( struct cms_message *cms, struct image *image,
 		 const char *name, time_t time, struct x509_chain *store,
 		 struct x509_root *root ) {
-	struct cms_signer_info *info;
+	struct cms_participant *part;
 	struct x509_certificate *cert;
 	int count = 0;
 	int rc;
@@ -672,14 +717,18 @@ int cms_verify ( struct cms_signature *sig, struct image *image,
 	/* Mark image as untrusted */
 	image_untrust ( image );
 
-	/* Verify using all signerInfos */
-	list_for_each_entry ( info, &sig->info, list ) {
-		cert = x509_first ( info->chain );
+	/* Sanity check */
+	if ( ! cms_is_signature ( cms ) )
+		return -ENOTTY;
+
+	/* Verify using all signers */
+	list_for_each_entry ( part, &cms->participants, list ) {
+		cert = x509_first ( part->chain );
 		if ( name && ( x509_check_name ( cert, name ) != 0 ) )
 			continue;
-		if ( ( rc = cms_verify_signer_info ( sig, info, image->data,
-						     image->len, time, store,
-						     root ) ) != 0 )
+		if ( ( rc = cms_verify_signer ( cms, part, image->data,
+						image->len, time, store,
+						root ) ) != 0 )
 			return rc;
 		count++;
 	}
@@ -687,11 +736,11 @@ int cms_verify ( struct cms_signature *sig, struct image *image,
 	/* Check that we have verified at least one signature */
 	if ( count == 0 ) {
 		if ( name ) {
-			DBGC ( sig, "CMS %p had no signatures matching name "
-			       "%s\n", sig, name );
+			DBGC ( cms, "CMS %p had no signatures matching name "
+			       "%s\n", cms, name );
 			return -EACCES_WRONG_NAME;
 		} else {
-			DBGC ( sig, "CMS %p had no signatures\n", sig );
+			DBGC ( cms, "CMS %p had no signatures\n", cms );
 			return -EACCES_NO_SIGNATURES;
 		}
 	}
diff --git a/src/include/ipxe/asn1.h b/src/include/ipxe/asn1.h
index fd7244570..26dc47992 100644
--- a/src/include/ipxe/asn1.h
+++ b/src/include/ipxe/asn1.h
@@ -303,7 +303,7 @@ struct asn1_builder_header {
 	ASN1_OID_SINGLE ( 5 ), ASN1_OID_SINGLE ( 7 ),		\
 	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 3 )
 
-/** ASN.1 OID for pkcs-signedData (1.2.840.113549.1.7.2) */
+/** ASN.1 OID for id-signedData (1.2.840.113549.1.7.2) */
 #define ASN1_OID_SIGNEDDATA					\
 	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
 	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
diff --git a/src/include/ipxe/cms.h b/src/include/ipxe/cms.h
index cca7779c5..1c8a0c587 100644
--- a/src/include/ipxe/cms.h
+++ b/src/include/ipxe/cms.h
@@ -17,61 +17,92 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/uaccess.h>
 
 struct image;
+struct cms_message;
 
-/** CMS signer information */
-struct cms_signer_info {
-	/** List of signer information blocks */
+/** A CMS message type */
+struct cms_type {
+	/** Name */
+	const char *name;
+	/** Object identifier */
+	struct asn1_cursor oid;
+	/** Parse content
+	 *
+	 * @v cms		CMS message
+	 * @v raw		ASN.1 cursor
+	 * @ret rc		Return status code
+	 */
+	int ( * parse ) ( struct cms_message *cms,
+			  const struct asn1_cursor *raw );
+};
+
+/** CMS participant information */
+struct cms_participant {
+	/** List of participant information blocks */
 	struct list_head list;
-
 	/** Certificate chain */
 	struct x509_chain *chain;
 
-	/** Digest algorithm */
+	/** Digest algorithm (for signature messages) */
 	struct digest_algorithm *digest;
 	/** Public-key algorithm */
 	struct pubkey_algorithm *pubkey;
 
-	/** Signature */
-	void *signature;
-	/** Length of signature */
-	size_t signature_len;
+	/** Signature or key value */
+	void *value;
+	/** Length of signature or key value */
+	size_t len;
 };
 
-/** A CMS signature */
-struct cms_signature {
+/** A CMS message */
+struct cms_message {
 	/** Reference count */
 	struct refcnt refcnt;
-	/** List of all certificates */
+	/** Message type */
+	struct cms_type *type;
+
+	/** List of all certificates (for signature messages) */
 	struct x509_chain *certificates;
-	/** List of signer information blocks */
-	struct list_head info;
+	/** List of participant information blocks */
+	struct list_head participants;
 };
 
 /**
- * Get reference to CMS signature
+ * Get reference to CMS message
  *
- * @v sig		CMS signature
- * @ret sig		CMS signature
+ * @v cms		CMS message
+ * @ret cms		CMS message
  */
-static inline __attribute__ (( always_inline )) struct cms_signature *
-cms_get ( struct cms_signature *sig ) {
-	ref_get ( &sig->refcnt );
-	return sig;
+static inline __attribute__ (( always_inline )) struct cms_message *
+cms_get ( struct cms_message *cms ) {
+	ref_get ( &cms->refcnt );
+	return cms;
 }
 
 /**
- * Drop reference to CMS signature
+ * Drop reference to CMS message
  *
- * @v sig		CMS signature
+ * @v cms		CMS message
  */
 static inline __attribute__ (( always_inline )) void
-cms_put ( struct cms_signature *sig ) {
-	ref_put ( &sig->refcnt );
+cms_put ( struct cms_message *cms ) {
+	ref_put ( &cms->refcnt );
 }
 
-extern int cms_signature ( struct image *image,
-			   struct cms_signature **sig );
-extern int cms_verify ( struct cms_signature *sig, struct image *image,
+/**
+ * Check if CMS message is a signature message
+ *
+ * @v cms		CMS message
+ * @ret is_signature	Message is a signature message
+ */
+static inline __attribute__ (( always_inline )) int
+cms_is_signature ( struct cms_message *cms ) {
+
+	/* CMS signatures include an optional CertificateSet */
+	return ( cms->certificates != NULL );
+}
+
+extern int cms_message ( struct image *image, struct cms_message **cms );
+extern int cms_verify ( struct cms_message *cms, struct image *image,
 			const char *name, time_t time, struct x509_chain *store,
 			struct x509_root *root );
 
diff --git a/src/tests/cms_test.c b/src/tests/cms_test.c
index d98b2c3e5..86f9bb98f 100644
--- a/src/tests/cms_test.c
+++ b/src/tests/cms_test.c
@@ -55,8 +55,8 @@ struct cms_test_code {
 struct cms_test_signature {
 	/** Signature image */
 	struct image image;
-	/** Parsed signature */
-	struct cms_signature *sig;
+	/** Parsed message */
+	struct cms_message *cms;
 };
 
 /** Define inline data */
@@ -1366,7 +1366,7 @@ static void cms_signature_okx ( struct cms_test_signature *sgn,
 	sgn->image.data = virt_to_user ( data );
 
 	/* Check ability to parse signature */
-	okx ( cms_signature ( &sgn->image, &sgn->sig ) == 0, file, line );
+	okx ( cms_message ( &sgn->image, &sgn->cms ) == 0, file, line );
 
 	/* Reset image data pointer */
 	sgn->image.data = ( ( userptr_t ) data );
@@ -1397,10 +1397,10 @@ static void cms_verify_okx ( struct cms_test_signature *sgn,
 	code->image.data = virt_to_user ( data );
 
 	/* Invalidate any certificates from previous tests */
-	x509_invalidate_chain ( sgn->sig->certificates );
+	x509_invalidate_chain ( sgn->cms->certificates );
 
 	/* Check ability to verify signature */
-	okx ( cms_verify ( sgn->sig, &code->image, name, time, store,
+	okx ( cms_verify ( sgn->cms, &code->image, name, time, store,
 			   root ) == 0, file, line );
 	okx ( code->image.flags & IMAGE_TRUSTED, file, line );
 
@@ -1434,10 +1434,10 @@ static void cms_verify_fail_okx ( struct cms_test_signature *sgn,
 	code->image.data = virt_to_user ( data );
 
 	/* Invalidate any certificates from previous tests */
-	x509_invalidate_chain ( sgn->sig->certificates );
+	x509_invalidate_chain ( sgn->cms->certificates );
 
 	/* Check inability to verify signature */
-	okx ( cms_verify ( sgn->sig, &code->image, name, time, store,
+	okx ( cms_verify ( sgn->cms, &code->image, name, time, store,
 			   root ) != 0, file, line );
 	okx ( ! ( code->image.flags & IMAGE_TRUSTED ), file, line );
 
@@ -1498,11 +1498,11 @@ static void cms_test_exec ( void ) {
 	/* Sanity check */
 	assert ( list_empty ( &empty_store.links ) );
 
-	/* Drop signature references */
-	cms_put ( nonsigned_sig.sig );
-	cms_put ( genericsigned_sig.sig );
-	cms_put ( brokenchain_sig.sig );
-	cms_put ( codesigned_sig.sig );
+	/* Drop message references */
+	cms_put ( nonsigned_sig.cms );
+	cms_put ( genericsigned_sig.cms );
+	cms_put ( brokenchain_sig.cms );
+	cms_put ( codesigned_sig.cms );
 }
 
 /** CMS self-test */
diff --git a/src/usr/imgtrust.c b/src/usr/imgtrust.c
index 54ea3378f..7f7e7ed14 100644
--- a/src/usr/imgtrust.c
+++ b/src/usr/imgtrust.c
@@ -50,18 +50,18 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 int imgverify ( struct image *image, struct image *signature,
 		const char *name ) {
-	struct cms_signature *sig;
-	struct cms_signer_info *info;
+	struct cms_message *cms;
+	struct cms_participant *part;
 	time_t now;
 	int rc;
 
 	/* Parse signature */
-	if ( ( rc = cms_signature ( signature, &sig ) ) != 0 )
+	if ( ( rc = cms_message ( signature, &cms ) ) != 0 )
 		goto err_parse;
 
 	/* Complete all certificate chains */
-	list_for_each_entry ( info, &sig->info, list ) {
-		if ( ( rc = create_validator ( &monojob, info->chain,
+	list_for_each_entry ( part, &cms->participants, list ) {
+		if ( ( rc = create_validator ( &monojob, part->chain,
 					       NULL ) ) != 0 )
 			goto err_create_validator;
 		if ( ( rc = monojob_wait ( NULL, 0 ) ) != 0 )
@@ -70,12 +70,12 @@ int imgverify ( struct image *image, struct image *signature,
 
 	/* Use signature to verify image */
 	now = time ( NULL );
-	if ( ( rc = cms_verify ( sig, image, name, now, NULL, NULL ) ) != 0 )
+	if ( ( rc = cms_verify ( cms, image, name, now, NULL, NULL ) ) != 0 )
 		goto err_verify;
 
-	/* Drop reference to signature */
-	cms_put ( sig );
-	sig = NULL;
+	/* Drop reference to message */
+	cms_put ( cms );
+	cms = NULL;
 
 	/* Record signature verification */
 	syslog ( LOG_NOTICE, "Image \"%s\" signature OK\n", image->name );
@@ -85,7 +85,7 @@ int imgverify ( struct image *image, struct image *signature,
  err_verify:
  err_validator_wait:
  err_create_validator:
-	cms_put ( sig );
+	cms_put ( cms );
  err_parse:
 	syslog ( LOG_ERR, "Image \"%s\" signature bad: %s\n",
 		 image->name, strerror ( rc ) );

From 9d9465b140cf59750f97995c501d054d2543c29a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 14 Aug 2024 14:00:48 +0100
Subject: [PATCH 169/237] [crypto] Fix debug name for empty certificate chain
 validators

An attempt to use a validator for an empty certificate chain will
correctly fail the overall validation with the "empty certificate
chain" error propagated from x509_auto_append().

In a debug build, the call to validator_name() will attempt to call
x509_name() on a non-existent certificate, resulting in garbage in the
debug message.

Fix by checking for the special case of an empty certificate chain.

This issue does not affect non-debug builds, since validator_name() is
(as per its description) called only for debug messages.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/net/validator.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/net/validator.c b/src/net/validator.c
index 69c0df333..e1371d2e6 100644
--- a/src/net/validator.c
+++ b/src/net/validator.c
@@ -135,9 +135,11 @@ struct validator {
  * @ret name		Validator name
  */
 static const char * validator_name ( struct validator *validator ) {
+	struct x509_certificate *cert;
 
-	/* Use name of first certificate in chain */
-	return x509_name ( x509_first ( validator->chain ) );
+	/* Use name of first certificate in chain, if present */
+	cert = x509_first ( validator->chain );
+	return ( cert ? x509_name ( cert ) : "<empty>" );
 }
 
 /**

From 7c82ff0b6b12437bfc25d01d52308fc6fe2e1311 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 15 Aug 2024 08:46:41 +0100
Subject: [PATCH 170/237] [pci] Separate permission to probe buses from
 bus:dev.fn range discovery

The UEFI device model requires us to not probe the PCI bus directly,
but instead to wait to be offered the opportunity to drive devices via
our driver service binding handle.

We currently inhibit PCI bus probing by having pci_discover() return
an empty range when using the EFI PCI I/O API.  This has the unwanted
side effect that scanning the bus manually using the "pciscan" command
will also fail to discover any devices.

Separate out the concept of being allowed to probe PCI buses from the
mechanism for discovering PCI bus:dev.fn address ranges, so that this
limitation may be removed.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/x86/core/pcidirect.c            |  1 +
 src/arch/x86/include/ipxe/pcibios.h      | 10 ++++++++++
 src/arch/x86/include/ipxe/pcicloud.h     | 10 ++++++++++
 src/arch/x86/include/ipxe/pcidirect.h    | 10 ++++++++++
 src/arch/x86/interface/pcbios/pcibios.c  |  1 +
 src/arch/x86/interface/pcbios/pcicloud.c |  1 +
 src/drivers/bus/ecam.c                   |  1 +
 src/drivers/bus/pci.c                    |  4 ++++
 src/include/ipxe/ecam.h                  | 10 ++++++++++
 src/include/ipxe/efi/efi_pci_api.h       | 10 ++++++++++
 src/include/ipxe/linux/linux_pci.h       | 10 ++++++++++
 src/include/ipxe/pci_io.h                |  7 +++++++
 src/interface/efi/efi_pci.c              |  1 +
 src/interface/linux/linux_pci.c          |  1 +
 14 files changed, 77 insertions(+)

diff --git a/src/arch/x86/core/pcidirect.c b/src/arch/x86/core/pcidirect.c
index f4659a1ac..90d4623b3 100644
--- a/src/arch/x86/core/pcidirect.c
+++ b/src/arch/x86/core/pcidirect.c
@@ -45,6 +45,7 @@ void pcidirect_prepare ( struct pci_device *pci, int where ) {
 	       PCIDIRECT_CONFIG_ADDRESS );
 }
 
+PROVIDE_PCIAPI_INLINE ( direct, pci_can_probe );
 PROVIDE_PCIAPI_INLINE ( direct, pci_discover );
 PROVIDE_PCIAPI_INLINE ( direct, pci_read_config_byte );
 PROVIDE_PCIAPI_INLINE ( direct, pci_read_config_word );
diff --git a/src/arch/x86/include/ipxe/pcibios.h b/src/arch/x86/include/ipxe/pcibios.h
index 3caea1cfe..ef9585654 100644
--- a/src/arch/x86/include/ipxe/pcibios.h
+++ b/src/arch/x86/include/ipxe/pcibios.h
@@ -32,6 +32,16 @@ extern int pcibios_read ( struct pci_device *pci, uint32_t command,
 extern int pcibios_write ( struct pci_device *pci, uint32_t command,
 			   uint32_t value );
 
+/**
+ * Check if PCI bus probing is allowed
+ *
+ * @ret ok		Bus probing is allowed
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( pcbios, pci_can_probe ) ( void ) {
+	return 1;
+}
+
 /**
  * Read byte from PCI configuration space via PCI BIOS
  *
diff --git a/src/arch/x86/include/ipxe/pcicloud.h b/src/arch/x86/include/ipxe/pcicloud.h
index 52268908c..1feef56cb 100644
--- a/src/arch/x86/include/ipxe/pcicloud.h
+++ b/src/arch/x86/include/ipxe/pcicloud.h
@@ -15,4 +15,14 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define PCIAPI_PREFIX_cloud __cloud_
 #endif
 
+/**
+ * Check if PCI bus probing is allowed
+ *
+ * @ret ok		Bus probing is allowed
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( cloud, pci_can_probe ) ( void ) {
+	return 1;
+}
+
 #endif /* _IPXE_PCICLOUD_H */
diff --git a/src/arch/x86/include/ipxe/pcidirect.h b/src/arch/x86/include/ipxe/pcidirect.h
index 98c6a2bbb..999b52763 100644
--- a/src/arch/x86/include/ipxe/pcidirect.h
+++ b/src/arch/x86/include/ipxe/pcidirect.h
@@ -25,6 +25,16 @@ struct pci_device;
 
 extern void pcidirect_prepare ( struct pci_device *pci, int where );
 
+/**
+ * Check if PCI bus probing is allowed
+ *
+ * @ret ok		Bus probing is allowed
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( direct, pci_can_probe ) ( void ) {
+	return 1;
+}
+
 /**
  * Find next PCI bus:dev.fn address range in system
  *
diff --git a/src/arch/x86/interface/pcbios/pcibios.c b/src/arch/x86/interface/pcbios/pcibios.c
index 7b7a769e3..ebe40ba7d 100644
--- a/src/arch/x86/interface/pcbios/pcibios.c
+++ b/src/arch/x86/interface/pcbios/pcibios.c
@@ -120,6 +120,7 @@ int pcibios_write ( struct pci_device *pci, uint32_t command, uint32_t value ){
 	return ( status >> 8 );
 }
 
+PROVIDE_PCIAPI_INLINE ( pcbios, pci_can_probe );
 PROVIDE_PCIAPI ( pcbios, pci_discover, pcibios_discover );
 PROVIDE_PCIAPI_INLINE ( pcbios, pci_read_config_byte );
 PROVIDE_PCIAPI_INLINE ( pcbios, pci_read_config_word );
diff --git a/src/arch/x86/interface/pcbios/pcicloud.c b/src/arch/x86/interface/pcbios/pcicloud.c
index 98ba38b31..f7d4a2da1 100644
--- a/src/arch/x86/interface/pcbios/pcicloud.c
+++ b/src/arch/x86/interface/pcbios/pcicloud.c
@@ -148,6 +148,7 @@ static void * pcicloud_ioremap ( struct pci_device *pci,
 	return pcicloud->pci_ioremap ( pci, bus_addr, len );
 }
 
+PROVIDE_PCIAPI_INLINE ( cloud, pci_can_probe );
 PROVIDE_PCIAPI ( cloud, pci_discover, pcicloud_discover );
 PROVIDE_PCIAPI ( cloud, pci_read_config_byte, pcicloud_read_config_byte );
 PROVIDE_PCIAPI ( cloud, pci_read_config_word, pcicloud_read_config_word );
diff --git a/src/drivers/bus/ecam.c b/src/drivers/bus/ecam.c
index 5e3debddd..cde5952b8 100644
--- a/src/drivers/bus/ecam.c
+++ b/src/drivers/bus/ecam.c
@@ -276,6 +276,7 @@ int ecam_write ( struct pci_device *pci, unsigned int location,
 	return 0;
 }
 
+PROVIDE_PCIAPI_INLINE ( ecam, pci_can_probe );
 PROVIDE_PCIAPI ( ecam, pci_discover, ecam_discover );
 PROVIDE_PCIAPI_INLINE ( ecam, pci_read_config_byte );
 PROVIDE_PCIAPI_INLINE ( ecam, pci_read_config_word );
diff --git a/src/drivers/bus/pci.c b/src/drivers/bus/pci.c
index 92b389641..05c9a5c26 100644
--- a/src/drivers/bus/pci.c
+++ b/src/drivers/bus/pci.c
@@ -361,6 +361,10 @@ static int pcibus_probe ( struct root_device *rootdev ) {
 	uint32_t busdevfn = 0;
 	int rc;
 
+	/* Skip automatic probing if prohibited */
+	if ( ! pci_can_probe() )
+		return 0;
+
 	do {
 		/* Allocate struct pci_device */
 		if ( ! pci )
diff --git a/src/include/ipxe/ecam.h b/src/include/ipxe/ecam.h
index ff08aee5a..f656083f7 100644
--- a/src/include/ipxe/ecam.h
+++ b/src/include/ipxe/ecam.h
@@ -54,6 +54,16 @@ struct ecam_mapping {
 	int rc;
 };
 
+/**
+ * Check if PCI bus probing is allowed
+ *
+ * @ret ok		Bus probing is allowed
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( ecam, pci_can_probe ) ( void ) {
+	return 1;
+}
+
 extern struct pci_api ecam_api;
 
 #endif /* _IPXE_ECAM_H */
diff --git a/src/include/ipxe/efi/efi_pci_api.h b/src/include/ipxe/efi/efi_pci_api.h
index cf5e1d020..0c4c1b72c 100644
--- a/src/include/ipxe/efi/efi_pci_api.h
+++ b/src/include/ipxe/efi/efi_pci_api.h
@@ -32,6 +32,16 @@ extern int efipci_read ( struct pci_device *pci, unsigned long location,
 extern int efipci_write ( struct pci_device *pci, unsigned long location,
 			  unsigned long value );
 
+/**
+ * Check if PCI bus probing is allowed
+ *
+ * @ret ok		Bus probing is allowed
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( efi, pci_can_probe ) ( void ) {
+	return 0;
+}
+
 /**
  * Find next PCI bus:dev.fn address range in system
  *
diff --git a/src/include/ipxe/linux/linux_pci.h b/src/include/ipxe/linux/linux_pci.h
index ec6ff8b1c..2b19e13c3 100644
--- a/src/include/ipxe/linux/linux_pci.h
+++ b/src/include/ipxe/linux/linux_pci.h
@@ -22,6 +22,16 @@ extern int linux_pci_read ( struct pci_device *pci, unsigned long where,
 extern int linux_pci_write ( struct pci_device *pci, unsigned long where,
 			     unsigned long value, size_t len );
 
+/**
+ * Check if PCI bus probing is allowed
+ *
+ * @ret ok		Bus probing is allowed
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( linux, pci_can_probe ) ( void ) {
+	return 1;
+}
+
 /**
  * Find next PCI bus:dev.fn address range in system
  *
diff --git a/src/include/ipxe/pci_io.h b/src/include/ipxe/pci_io.h
index 4c035b18b..322fdbb24 100644
--- a/src/include/ipxe/pci_io.h
+++ b/src/include/ipxe/pci_io.h
@@ -66,6 +66,13 @@ struct pci_range {
 /* Include all architecture-dependent I/O API headers */
 #include <bits/pci_io.h>
 
+/**
+ * Check if PCI bus probing is allowed
+ *
+ * @ret ok		Bus probing is allowed
+ */
+int pci_can_probe ( void );
+
 /**
  * Find next PCI bus:dev.fn address range in system
  *
diff --git a/src/interface/efi/efi_pci.c b/src/interface/efi/efi_pci.c
index e2eeeb344..61071d8a4 100644
--- a/src/interface/efi/efi_pci.c
+++ b/src/interface/efi/efi_pci.c
@@ -362,6 +362,7 @@ void * efipci_ioremap ( struct pci_device *pci, unsigned long bus_addr,
 	return ioremap ( bus_addr, len );
 }
 
+PROVIDE_PCIAPI_INLINE ( efi, pci_can_probe );
 PROVIDE_PCIAPI_INLINE ( efi, pci_discover );
 PROVIDE_PCIAPI_INLINE ( efi, pci_read_config_byte );
 PROVIDE_PCIAPI_INLINE ( efi, pci_read_config_word );
diff --git a/src/interface/linux/linux_pci.c b/src/interface/linux/linux_pci.c
index 300844737..a3a0828c1 100644
--- a/src/interface/linux/linux_pci.c
+++ b/src/interface/linux/linux_pci.c
@@ -188,6 +188,7 @@ int linux_pci_write ( struct pci_device *pci, unsigned long where,
 	return rc;
 }
 
+PROVIDE_PCIAPI_INLINE ( linux, pci_can_probe );
 PROVIDE_PCIAPI_INLINE ( linux, pci_discover );
 PROVIDE_PCIAPI_INLINE ( linux, pci_read_config_byte );
 PROVIDE_PCIAPI_INLINE ( linux, pci_read_config_word );

From 950f6b5861d8d6b247b37e4e1401d26d8f908ee8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 14 Aug 2024 23:40:50 +0100
Subject: [PATCH 171/237] [efi] Allow discovery of PCI bus:dev.fn address
 ranges

Generalise the logic for identifying the matching PCI root bridge I/O
protocol to allow for identifying the closest matching PCI bus:dev.fn
address range, and use this to provide PCI address range discovery
(while continuing to inhibit automatic PCI bus probing).

This allows the "pciscan" command to work as expected under UEFI.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/efi/efi_pci_api.h |  14 --
 src/interface/efi/efi_pci.c        | 250 +++++++++++++++++++++--------
 2 files changed, 181 insertions(+), 83 deletions(-)

diff --git a/src/include/ipxe/efi/efi_pci_api.h b/src/include/ipxe/efi/efi_pci_api.h
index 0c4c1b72c..9aca02f65 100644
--- a/src/include/ipxe/efi/efi_pci_api.h
+++ b/src/include/ipxe/efi/efi_pci_api.h
@@ -42,20 +42,6 @@ PCIAPI_INLINE ( efi, pci_can_probe ) ( void ) {
 	return 0;
 }
 
-/**
- * Find next PCI bus:dev.fn address range in system
- *
- * @v busdevfn		Starting PCI bus:dev.fn address
- * @v range		PCI bus:dev.fn address range to fill in
- */
-static inline __always_inline void
-PCIAPI_INLINE ( efi, pci_discover ) ( uint32_t busdevfn __unused,
-				      struct pci_range *range ) {
-
-	/* EFI does not want us to scan the PCI bus ourselves */
-	range->count = 0;
-}
-
 /**
  * Read byte from PCI configuration space via EFI
  *
diff --git a/src/interface/efi/efi_pci.c b/src/interface/efi/efi_pci.c
index 61071d8a4..8d4e08567 100644
--- a/src/interface/efi/efi_pci.c
+++ b/src/interface/efi/efi_pci.c
@@ -63,69 +63,202 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 
 /**
- * Check for a matching PCI root bridge I/O protocol
+ * Find closest bus:dev.fn address range within a root bridge
  *
- * @v pci		PCI device
+ * @v pci		Starting PCI device
  * @v handle		EFI PCI root bridge handle
- * @v root		EFI PCI root bridge I/O protocol
+ * @v range		PCI bus:dev.fn address range to fill in
  * @ret rc		Return status code
  */
-static int efipci_root_match ( struct pci_device *pci, EFI_HANDLE handle,
-			       EFI_PCI_ROOT_BRIDGE_IO_PROTOCOL *root ) {
+static int efipci_discover_one ( struct pci_device *pci, EFI_HANDLE handle,
+				 struct pci_range *range ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	union {
+		void *interface;
+		EFI_PCI_ROOT_BRIDGE_IO_PROTOCOL *root;
+	} root;
 	union {
 		union acpi_resource *res;
 		void *raw;
-	} u;
-	unsigned int segment = PCI_SEG ( pci->busdevfn );
-	unsigned int bus = PCI_BUS ( pci->busdevfn );
-	unsigned int start;
-	unsigned int end;
+	} acpi;
+	uint32_t best = 0;
+	uint32_t start;
+	uint32_t count;
+	uint32_t index;
 	unsigned int tag;
 	EFI_STATUS efirc;
 	int rc;
 
-	/* Check segment number */
-	if ( root->SegmentNumber != segment )
-		return -ENOENT;
+	/* Return empty range on error */
+	range->start = 0;
+	range->count = 0;
+
+	/* Open root bridge I/O protocol */
+	if ( ( efirc = bs->OpenProtocol ( handle,
+			&efi_pci_root_bridge_io_protocol_guid,
+			&root.interface, efi_image_handle, handle,
+			EFI_OPEN_PROTOCOL_GET_PROTOCOL ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( pci, "EFIPCI " PCI_FMT " cannot open %s: %s\n",
+		       PCI_ARGS ( pci ), efi_handle_name ( handle ),
+		       strerror ( rc ) );
+		goto err_open;
+	}
 
 	/* Get ACPI resource descriptors */
-	if ( ( efirc = root->Configuration ( root, &u.raw ) ) != 0 ) {
+	if ( ( efirc = root.root->Configuration ( root.root,
+						  &acpi.raw ) ) != 0 ) {
 		rc = -EEFI ( efirc );
 		DBGC ( pci, "EFIPCI " PCI_FMT " cannot get configuration for "
 		       "%s: %s\n", PCI_ARGS ( pci ),
 		       efi_handle_name ( handle ), strerror ( rc ) );
-		return rc;
+		goto err_config;
 	}
 
-	/* Assume success if no bus number range descriptors are found */
-	rc = 0;
-
 	/* Parse resource descriptors */
-	for ( ; ( ( tag = acpi_resource_tag ( u.res ) ) != ACPI_END_RESOURCE ) ;
-	      u.res = acpi_resource_next ( u.res ) ) {
+	for ( ; ( ( tag = acpi_resource_tag ( acpi.res ) ) !=
+		  ACPI_END_RESOURCE ) ;
+	      acpi.res = acpi_resource_next ( acpi.res ) ) {
 
 		/* Ignore anything other than a bus number range descriptor */
 		if ( tag != ACPI_QWORD_ADDRESS_SPACE_RESOURCE )
 			continue;
-		if ( u.res->qword.type != ACPI_ADDRESS_TYPE_BUS )
+		if ( acpi.res->qword.type != ACPI_ADDRESS_TYPE_BUS )
 			continue;
 
-		/* Check for a matching bus number */
-		start = le64_to_cpu ( u.res->qword.min );
-		end = ( start + le64_to_cpu ( u.res->qword.len ) );
-		if ( ( bus >= start ) && ( bus < end ) )
-			return 0;
+		/* Get range for this descriptor */
+		start = PCI_BUSDEVFN ( root.root->SegmentNumber,
+				       le64_to_cpu ( acpi.res->qword.min ),
+				       0, 0 );
+		count = PCI_BUSDEVFN ( 0, le64_to_cpu ( acpi.res->qword.len ),
+				       0, 0 );
+		DBGC2 ( pci, "EFIPCI " PCI_FMT " found %04x:[%02x-%02x] via "
+			"%s\n", PCI_ARGS ( pci ), root.root->SegmentNumber,
+			PCI_BUS ( start ), PCI_BUS ( start + count - 1 ),
+			efi_handle_name ( handle ) );
 
-		/* We have seen at least one non-matching range
-		 * descriptor, so assume failure unless we find a
-		 * subsequent match.
-		 */
-		rc = -ENOENT;
+		/* Check for a matching or new closest range */
+		index = ( pci->busdevfn - start );
+		if ( ( index < count ) || ( index > best ) ) {
+			range->start = start;
+			range->count = count;
+			best = index;
+		}
+
+		/* Stop if this range contains the target bus:dev.fn address */
+		if ( index < count )
+			break;
 	}
 
+	/* If no range descriptors were seen, assume that the root
+	 * bridge has a single bus.
+	 */
+	if ( ! range->count ) {
+		range->start = PCI_BUSDEVFN ( root.root->SegmentNumber,
+					      0, 0, 0 );
+		range->count = PCI_BUSDEVFN ( 0, 1, 0, 0 );
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_config:
+	bs->CloseProtocol ( handle, &efi_pci_root_bridge_io_protocol_guid,
+			    efi_image_handle, handle );
+ err_open:
 	return rc;
 }
 
+/**
+ * Find closest bus:dev.fn address range within any root bridge
+ *
+ * @v pci		Starting PCI device
+ * @v range		PCI bus:dev.fn address range to fill in
+ * @v handle		PCI root bridge I/O handle to fill in
+ * @ret rc		Return status code
+ */
+static int efipci_discover_any ( struct pci_device *pci,
+				 struct pci_range *range,
+				 EFI_HANDLE *handle ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	uint32_t best = 0;
+	uint32_t index;
+	struct pci_range tmp;
+	EFI_HANDLE *handles;
+	UINTN num_handles;
+	UINTN i;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Return an empty range and no handle on error */
+	range->start = 0;
+	range->count = 0;
+	*handle = NULL;
+
+	/* Enumerate all root bridge I/O protocol handles */
+	if ( ( efirc = bs->LocateHandleBuffer ( ByProtocol,
+			&efi_pci_root_bridge_io_protocol_guid,
+			NULL, &num_handles, &handles ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( pci, "EFIPCI " PCI_FMT " cannot locate root bridges: "
+		       "%s\n", PCI_ARGS ( pci ), strerror ( rc ) );
+		goto err_locate;
+	}
+
+	/* Iterate over all root bridge I/O protocols */
+	for ( i = 0 ; i < num_handles ; i++ ) {
+
+		/* Get matching or closest range for this root bridge */
+		if ( ( rc = efipci_discover_one ( pci, handles[i],
+						  &tmp ) ) != 0 )
+			continue;
+
+		/* Check for a matching or new closest range */
+		index = ( pci->busdevfn - tmp.start );
+		if ( ( index < tmp.count ) || ( index > best ) ) {
+			range->start = tmp.start;
+			range->count = tmp.count;
+			best = index;
+		}
+
+		/* Stop if this range contains the target bus:dev.fn address */
+		if ( index < tmp.count ) {
+			*handle = handles[i];
+			break;
+		}
+	}
+
+	/* Check for a range containing the target bus:dev.fn address */
+	if ( ! *handle ) {
+		rc = -ENOENT;
+		goto err_range;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_range:
+	bs->FreePool ( handles );
+ err_locate:
+	return rc;
+}
+
+/**
+ * Find next PCI bus:dev.fn address range in system
+ *
+ * @v busdevfn		Starting PCI bus:dev.fn address
+ * @v range		PCI bus:dev.fn address range to fill in
+ */
+static void efipci_discover ( uint32_t busdevfn, struct pci_range *range ) {
+	struct pci_device pci;
+	EFI_HANDLE handle;
+
+	/* Find range */
+	memset ( &pci, 0, sizeof ( pci ) );
+	pci_init ( &pci, busdevfn );
+	efipci_discover_any ( &pci, range, &handle );
+}
+
 /**
  * Open EFI PCI root bridge I/O protocol
  *
@@ -137,55 +270,34 @@ static int efipci_root_match ( struct pci_device *pci, EFI_HANDLE handle,
 static int efipci_root_open ( struct pci_device *pci, EFI_HANDLE *handle,
 			      EFI_PCI_ROOT_BRIDGE_IO_PROTOCOL **root ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	EFI_HANDLE *handles;
-	UINTN num_handles;
+	struct pci_range tmp;
 	union {
 		void *interface;
 		EFI_PCI_ROOT_BRIDGE_IO_PROTOCOL *root;
 	} u;
 	EFI_STATUS efirc;
-	UINTN i;
 	int rc;
 
-	/* Enumerate all handles */
-	if ( ( efirc = bs->LocateHandleBuffer ( ByProtocol,
+	/* Find matching root bridge I/O protocol handle */
+	if ( ( rc = efipci_discover_any ( pci, &tmp, handle ) ) != 0 )
+		return rc;
+
+	/* (Re)open PCI root bridge I/O protocol */
+	if ( ( efirc = bs->OpenProtocol ( *handle,
 			&efi_pci_root_bridge_io_protocol_guid,
-			NULL, &num_handles, &handles ) ) != 0 ) {
+			&u.interface, efi_image_handle, *handle,
+			EFI_OPEN_PROTOCOL_GET_PROTOCOL ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( pci, "EFIPCI " PCI_FMT " cannot locate root bridges: "
-		       "%s\n", PCI_ARGS ( pci ), strerror ( rc ) );
-		goto err_locate;
+		DBGC ( pci, "EFIPCI " PCI_FMT " cannot open %s: %s\n",
+		       PCI_ARGS ( pci ), efi_handle_name ( *handle ),
+		       strerror ( rc ) );
+		return rc;
 	}
 
-	/* Look for matching root bridge I/O protocol */
-	for ( i = 0 ; i < num_handles ; i++ ) {
-		*handle = handles[i];
-		if ( ( efirc = bs->OpenProtocol ( *handle,
-				&efi_pci_root_bridge_io_protocol_guid,
-				&u.interface, efi_image_handle, *handle,
-				EFI_OPEN_PROTOCOL_GET_PROTOCOL ) ) != 0 ) {
-			rc = -EEFI ( efirc );
-			DBGC ( pci, "EFIPCI " PCI_FMT " cannot open %s: %s\n",
-			       PCI_ARGS ( pci ), efi_handle_name ( *handle ),
-			       strerror ( rc ) );
-			continue;
-		}
-		if ( efipci_root_match ( pci, *handle, u.root ) == 0 ) {
-			*root = u.root;
-			bs->FreePool ( handles );
-			return 0;
-		}
-		bs->CloseProtocol ( *handle,
-				    &efi_pci_root_bridge_io_protocol_guid,
-				    efi_image_handle, *handle );
-	}
-	DBGC ( pci, "EFIPCI " PCI_FMT " found no root bridge\n",
-	       PCI_ARGS ( pci ) );
-	rc = -ENOENT;
+	/* Return opened protocol */
+	*root = u.root;
 
-	bs->FreePool ( handles );
- err_locate:
-	return rc;
+	return 0;
 }
 
 /**
@@ -363,7 +475,7 @@ void * efipci_ioremap ( struct pci_device *pci, unsigned long bus_addr,
 }
 
 PROVIDE_PCIAPI_INLINE ( efi, pci_can_probe );
-PROVIDE_PCIAPI_INLINE ( efi, pci_discover );
+PROVIDE_PCIAPI ( efi, pci_discover, efipci_discover );
 PROVIDE_PCIAPI_INLINE ( efi, pci_read_config_byte );
 PROVIDE_PCIAPI_INLINE ( efi, pci_read_config_word );
 PROVIDE_PCIAPI_INLINE ( efi, pci_read_config_dword );

From 53f089b723e16eecb4fd2e2a59b74b3932431b30 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 18 Aug 2024 10:43:52 +0100
Subject: [PATCH 172/237] [crypto] Pass asymmetric keys as ASN.1 cursors

Asymmetric keys are invariably encountered within ASN.1 structures
such as X.509 certificates, and the various large integers within an
RSA key are themselves encoded using ASN.1.

Simplify all code handling asymmetric keys by passing keys as a single
ASN.1 cursor, rather than separate data and length pointers.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/cms.c          |  3 +-
 src/crypto/crypto_null.c  |  4 +--
 src/crypto/ocsp.c         |  4 +--
 src/crypto/rsa.c          | 30 ++++-------------
 src/crypto/x509.c         |  9 +++---
 src/drivers/net/iphone.c  |  3 +-
 src/include/ipxe/crypto.h | 23 ++++++-------
 src/net/tls.c             |  5 ++-
 src/tests/pubkey_test.h   | 37 ++++++++-------------
 src/tests/rsa_test.c      | 68 ++++++++++++++++++---------------------
 10 files changed, 74 insertions(+), 112 deletions(-)

diff --git a/src/crypto/cms.c b/src/crypto/cms.c
index 1f33613f4..0b772f1cf 100644
--- a/src/crypto/cms.c
+++ b/src/crypto/cms.c
@@ -621,8 +621,7 @@ static int cms_verify_digest ( struct cms_message *cms,
 	cms_digest ( cms, part, data, len, digest_out );
 
 	/* Initialise public-key algorithm */
-	if ( ( rc = pubkey_init ( pubkey, ctx, public_key->raw.data,
-				  public_key->raw.len ) ) != 0 ) {
+	if ( ( rc = pubkey_init ( pubkey, ctx, &public_key->raw ) ) != 0 ) {
 		DBGC ( cms, "CMS %p/%p could not initialise public key: %s\n",
 		       cms, part, strerror ( rc ) );
 		goto err_init;
diff --git a/src/crypto/crypto_null.c b/src/crypto/crypto_null.c
index 0ad463c3e..b4169382b 100644
--- a/src/crypto/crypto_null.c
+++ b/src/crypto/crypto_null.c
@@ -93,8 +93,8 @@ struct cipher_algorithm cipher_null = {
 	.auth = cipher_null_auth,
 };
 
-int pubkey_null_init ( void *ctx __unused, const void *key __unused,
-		       size_t key_len __unused ) {
+int pubkey_null_init ( void *ctx __unused,
+		       const struct asn1_cursor *key __unused ) {
 	return 0;
 }
 
diff --git a/src/crypto/ocsp.c b/src/crypto/ocsp.c
index cc957b40c..f35593454 100644
--- a/src/crypto/ocsp.c
+++ b/src/crypto/ocsp.c
@@ -857,8 +857,8 @@ static int ocsp_check_signature ( struct ocsp_check *ocsp,
 	digest_final ( digest, digest_ctx, digest_out );
 
 	/* Initialise public-key algorithm */
-	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx, public_key->raw.data,
-				  public_key->raw.len ) ) != 0 ) {
+	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx,
+				  &public_key->raw ) ) != 0 ) {
 		DBGC ( ocsp, "OCSP %p \"%s\" could not initialise public key: "
 		       "%s\n", ocsp, x509_name ( ocsp->cert ), strerror ( rc ));
 		goto err_init;
diff --git a/src/crypto/rsa.c b/src/crypto/rsa.c
index 16c67d822..2d288a953 100644
--- a/src/crypto/rsa.c
+++ b/src/crypto/rsa.c
@@ -233,27 +233,21 @@ static int rsa_parse_mod_exp ( struct asn1_cursor *modulus,
  *
  * @v ctx		RSA context
  * @v key		Key
- * @v key_len		Length of key
  * @ret rc		Return status code
  */
-static int rsa_init ( void *ctx, const void *key, size_t key_len ) {
+static int rsa_init ( void *ctx, const struct asn1_cursor *key ) {
 	struct rsa_context *context = ctx;
 	struct asn1_cursor modulus;
 	struct asn1_cursor exponent;
-	struct asn1_cursor cursor;
 	int rc;
 
 	/* Initialise context */
 	memset ( context, 0, sizeof ( *context ) );
 
-	/* Initialise cursor */
-	cursor.data = key;
-	cursor.len = key_len;
-
 	/* Parse modulus and exponent */
-	if ( ( rc = rsa_parse_mod_exp ( &modulus, &exponent, &cursor ) ) != 0 ){
+	if ( ( rc = rsa_parse_mod_exp ( &modulus, &exponent, key ) ) != 0 ){
 		DBGC ( context, "RSA %p invalid modulus/exponent:\n", context );
-		DBGC_HDA ( context, 0, cursor.data, cursor.len );
+		DBGC_HDA ( context, 0, key->data, key->len );
 		goto err_parse;
 	}
 
@@ -592,33 +586,23 @@ static void rsa_final ( void *ctx ) {
  * Check for matching RSA public/private key pair
  *
  * @v private_key	Private key
- * @v private_key_len	Private key length
  * @v public_key	Public key
- * @v public_key_len	Public key length
  * @ret rc		Return status code
  */
-static int rsa_match ( const void *private_key, size_t private_key_len,
-		       const void *public_key, size_t public_key_len ) {
+static int rsa_match ( const struct asn1_cursor *private_key,
+		       const struct asn1_cursor *public_key ) {
 	struct asn1_cursor private_modulus;
 	struct asn1_cursor private_exponent;
-	struct asn1_cursor private_cursor;
 	struct asn1_cursor public_modulus;
 	struct asn1_cursor public_exponent;
-	struct asn1_cursor public_cursor;
 	int rc;
 
-	/* Initialise cursors */
-	private_cursor.data = private_key;
-	private_cursor.len = private_key_len;
-	public_cursor.data = public_key;
-	public_cursor.len = public_key_len;
-
 	/* Parse moduli and exponents */
 	if ( ( rc = rsa_parse_mod_exp ( &private_modulus, &private_exponent,
-					&private_cursor ) ) != 0 )
+					private_key ) ) != 0 )
 		return rc;
 	if ( ( rc = rsa_parse_mod_exp ( &public_modulus, &public_exponent,
-					&public_cursor ) ) != 0 )
+					public_key ) ) != 0 )
 		return rc;
 
 	/* Compare moduli */
diff --git a/src/crypto/x509.c b/src/crypto/x509.c
index acb85620f..c0762740e 100644
--- a/src/crypto/x509.c
+++ b/src/crypto/x509.c
@@ -1149,8 +1149,8 @@ static int x509_check_signature ( struct x509_certificate *cert,
 	}
 
 	/* Verify signature using signer's public key */
-	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx, public_key->raw.data,
-				  public_key->raw.len ) ) != 0 ) {
+	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx,
+				  &public_key->raw ) ) != 0 ) {
 		DBGC ( cert, "X509 %p \"%s\" cannot initialise public key: "
 		       "%s\n", cert, x509_name ( cert ), strerror ( rc ) );
 		goto err_pubkey_init;
@@ -1842,9 +1842,8 @@ struct x509_certificate * x509_find_key ( struct x509_chain *store,
 		/* Check public key */
 		cert = link->cert;
 		if ( pubkey_match ( cert->signature_algorithm->pubkey,
-				    key->builder.data, key->builder.len,
-				    cert->subject.public_key.raw.data,
-				    cert->subject.public_key.raw.len ) == 0 )
+				    privkey_cursor ( key ),
+				    &cert->subject.public_key.raw ) == 0 )
 			return x509_found ( store, cert );
 	}
 
diff --git a/src/drivers/net/iphone.c b/src/drivers/net/iphone.c
index bbac527bd..96eb0952b 100644
--- a/src/drivers/net/iphone.c
+++ b/src/drivers/net/iphone.c
@@ -367,8 +367,7 @@ static int icert_cert ( struct icert *icert, struct asn1_cursor *subject,
 	int rc;
 
 	/* Initialise "private" key */
-	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx, private->data,
-				  private->len ) ) != 0 ) {
+	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx, private ) ) != 0 ) {
 		DBGC ( icert, "ICERT %p could not initialise private key: "
 		       "%s\n", icert, strerror ( rc ) );
 		goto err_pubkey_init;
diff --git a/src/include/ipxe/crypto.h b/src/include/ipxe/crypto.h
index a6f437655..8b6eb94f6 100644
--- a/src/include/ipxe/crypto.h
+++ b/src/include/ipxe/crypto.h
@@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <stdint.h>
 #include <stddef.h>
 #include <assert.h>
+#include <ipxe/asn1.h>
 
 /** A message digest algorithm */
 struct digest_algorithm {
@@ -126,10 +127,9 @@ struct pubkey_algorithm {
 	 *
 	 * @v ctx		Context
 	 * @v key		Key
-	 * @v key_len		Length of key
 	 * @ret rc		Return status code
 	 */
-	int ( * init ) ( void *ctx, const void *key, size_t key_len );
+	int ( * init ) ( void *ctx, const struct asn1_cursor *key );
 	/** Calculate maximum output length
 	 *
 	 * @v ctx		Context
@@ -186,13 +186,11 @@ struct pubkey_algorithm {
 	/** Check that public key matches private key
 	 *
 	 * @v private_key	Private key
-	 * @v private_key_len	Private key length
 	 * @v public_key	Public key
-	 * @v public_key_len	Public key length
 	 * @ret rc		Return status code
 	 */
-	int ( * match ) ( const void *private_key, size_t private_key_len,
-			  const void *public_key, size_t public_key_len );
+	int ( * match ) ( const struct asn1_cursor *private_key,
+			  const struct asn1_cursor *public_key );
 };
 
 /** An elliptic curve */
@@ -282,8 +280,8 @@ is_auth_cipher ( struct cipher_algorithm *cipher ) {
 
 static inline __attribute__ (( always_inline )) int
 pubkey_init ( struct pubkey_algorithm *pubkey, void *ctx,
-	      const void *key, size_t key_len ) {
-	return pubkey->init ( ctx, key, key_len );
+	      const struct asn1_cursor *key ) {
+	return pubkey->init ( ctx, key );
 }
 
 static inline __attribute__ (( always_inline )) size_t
@@ -324,10 +322,9 @@ pubkey_final ( struct pubkey_algorithm *pubkey, void *ctx ) {
 
 static inline __attribute__ (( always_inline )) int
 pubkey_match ( struct pubkey_algorithm *pubkey,
-	       const void *private_key, size_t private_key_len,
-	       const void *public_key, size_t public_key_len ) {
-	return pubkey->match ( private_key, private_key_len, public_key,
-			       public_key_len );
+	       const struct asn1_cursor *private_key,
+	       const struct asn1_cursor *public_key ) {
+	return pubkey->match ( private_key, public_key );
 }
 
 static inline __attribute__ (( always_inline )) int
@@ -348,7 +345,7 @@ extern void cipher_null_decrypt ( void *ctx, const void *src, void *dst,
 				  size_t len );
 extern void cipher_null_auth ( void *ctx, void *auth );
 
-extern int pubkey_null_init ( void *ctx, const void *key, size_t key_len );
+extern int pubkey_null_init ( void *ctx, const struct asn1_cursor *key );
 extern size_t pubkey_null_max_len ( void *ctx );
 extern int pubkey_null_encrypt ( void *ctx, const void *plaintext,
 				 size_t plaintext_len, void *ciphertext );
diff --git a/src/net/tls.c b/src/net/tls.c
index c08057103..a22626f41 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -1824,7 +1824,7 @@ static int tls_send_certificate_verify ( struct tls_connection *tls ) {
 	tls_verify_handshake ( tls, digest_out );
 
 	/* Initialise public-key algorithm */
-	if ( ( rc = pubkey_init ( pubkey, ctx, key->data, key->len ) ) != 0 ) {
+	if ( ( rc = pubkey_init ( pubkey, ctx, key ) ) != 0 ) {
 		DBGC ( tls, "TLS %p could not initialise %s client private "
 		       "key: %s\n", tls, pubkey->name, strerror ( rc ) );
 		goto err_pubkey_init;
@@ -3581,8 +3581,7 @@ static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 
 	/* Initialise public key algorithm */
 	if ( ( rc = pubkey_init ( pubkey, cipherspec->pubkey_ctx,
-				  cert->subject.public_key.raw.data,
-				  cert->subject.public_key.raw.len ) ) != 0 ) {
+				  &cert->subject.public_key.raw ) ) != 0 ) {
 		DBGC ( tls, "TLS %p cannot initialise public key: %s\n",
 		       tls, strerror ( rc ) );
 		goto err;
diff --git a/src/tests/pubkey_test.h b/src/tests/pubkey_test.h
index cd65b8703..214992238 100644
--- a/src/tests/pubkey_test.h
+++ b/src/tests/pubkey_test.h
@@ -12,17 +12,16 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @v pubkey		Public key algorithm
  * @v key		Key
- * @v key_len		Key length
  * @v ciphertext	Ciphertext
  * @v ciphertext_len	Ciphertext length
  * @v expected		Expected plaintext
  * @v expected_len	Expected plaintext length
  */
-#define pubkey_decrypt_ok( pubkey, key, key_len, ciphertext,		\
-			   ciphertext_len, expected, expected_len ) do {\
+#define pubkey_decrypt_ok( pubkey, key, ciphertext, ciphertext_len,	\
+			   expected, expected_len ) do {		\
 	uint8_t ctx[ (pubkey)->ctxsize ];				\
 									\
-	ok ( pubkey_init ( (pubkey), ctx, (key), (key_len) ) == 0 );	\
+	ok ( pubkey_init ( (pubkey), ctx, (key) ) == 0 );		\
 	{								\
 		size_t max_len = pubkey_max_len ( (pubkey), ctx );	\
 		uint8_t decrypted[ max_len ];				\
@@ -44,19 +43,15 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @v pubkey		Public key algorithm
  * @v encrypt_key	Encryption key
- * @v encrypt_key_len	Encryption key length
  * @v decrypt_key	Decryption key
- * @v decrypt_key_len	Decryption key length
  * @v plaintext		Plaintext
  * @v plaintext_len	Plaintext length
  */
-#define pubkey_encrypt_ok( pubkey, encrypt_key, encrypt_key_len,	\
-			   decrypt_key, decrypt_key_len, plaintext,	\
+#define pubkey_encrypt_ok( pubkey, encrypt_key, decrypt_key, plaintext,	\
 			   plaintext_len ) do {				\
 	uint8_t ctx[ (pubkey)->ctxsize ];				\
 									\
-	ok ( pubkey_init ( (pubkey), ctx, (encrypt_key),		\
-			   (encrypt_key_len) ) == 0 );			\
+	ok ( pubkey_init ( (pubkey), ctx, (encrypt_key) ) == 0 );	\
 	{								\
 		size_t max_len = pubkey_max_len ( (pubkey), ctx );	\
 		uint8_t encrypted[ max_len ];				\
@@ -68,9 +63,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 						 encrypted );		\
 		ok ( encrypted_len >= 0 );				\
 		pubkey_decrypt_ok ( (pubkey), (decrypt_key),		\
-				    (decrypt_key_len), encrypted,	\
-				    encrypted_len, (plaintext),		\
-				    (plaintext_len) );			\
+				    encrypted, encrypted_len,		\
+				    (plaintext), (plaintext_len) );	\
 	}								\
 	pubkey_final ( (pubkey), ctx );					\
 	} while ( 0 )
@@ -80,15 +74,14 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @v pubkey		Public key algorithm
  * @v key		Key
- * @v key_len		Key length
  * @v digest		Digest algorithm
  * @v plaintext		Plaintext
  * @v plaintext_len	Plaintext length
  * @v expected		Expected signature
  * @v expected_len	Expected signature length
  */
-#define pubkey_sign_ok( pubkey, key, key_len, digest, plaintext,	\
-			plaintext_len, expected, expected_len ) do {	\
+#define pubkey_sign_ok( pubkey, key, digest, plaintext,	plaintext_len,	\
+			expected, expected_len ) do {			\
 	uint8_t ctx[ (pubkey)->ctxsize ];				\
 	uint8_t digestctx[ (digest)->ctxsize ];				\
 	uint8_t digestout[ (digest)->digestsize ];			\
@@ -98,7 +91,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 			(plaintext_len) );				\
 	digest_final ( (digest), digestctx, digestout );		\
 									\
-	ok ( pubkey_init ( (pubkey), ctx, (key), (key_len) ) == 0 );	\
+	ok ( pubkey_init ( (pubkey), ctx, (key) ) == 0 );		\
 	{								\
 		size_t max_len = pubkey_max_len ( (pubkey), ctx );	\
 		uint8_t signature[ max_len ];				\
@@ -118,14 +111,13 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @v pubkey		Public key algorithm
  * @v key		Key
- * @v key_len		Key length
  * @v digest		Digest algorithm
  * @v plaintext		Plaintext
  * @v plaintext_len	Plaintext length
  * @v signature		Signature
  * @v signature_len	Signature length
  */
-#define pubkey_verify_ok( pubkey, key, key_len, digest, plaintext,	\
+#define pubkey_verify_ok( pubkey, key, digest, plaintext,		\
 			  plaintext_len, signature, signature_len ) do {\
 	uint8_t ctx[ (pubkey)->ctxsize ];				\
 	uint8_t digestctx[ (digest)->ctxsize ];				\
@@ -136,7 +128,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 			(plaintext_len) );				\
 	digest_final ( (digest), digestctx, digestout );		\
 									\
-	ok ( pubkey_init ( (pubkey), ctx, (key), (key_len) ) == 0 );	\
+	ok ( pubkey_init ( (pubkey), ctx, (key) ) == 0 );		\
 	ok ( pubkey_verify ( (pubkey), ctx, (digest), digestout,	\
 			     (signature), (signature_len) ) == 0 );	\
 	pubkey_final ( (pubkey), ctx );					\
@@ -147,14 +139,13 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @v pubkey		Public key algorithm
  * @v key		Key
- * @v key_len		Key length
  * @v digest		Digest algorithm
  * @v plaintext		Plaintext
  * @v plaintext_len	Plaintext length
  * @v signature		Signature
  * @v signature_len	Signature length
  */
-#define pubkey_verify_fail_ok( pubkey, key, key_len, digest, plaintext,	\
+#define pubkey_verify_fail_ok( pubkey, key, digest, plaintext,		\
 			       plaintext_len, signature,		\
 			       signature_len ) do {			\
 	uint8_t ctx[ (pubkey)->ctxsize ];				\
@@ -166,7 +157,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 			(plaintext_len) );				\
 	digest_final ( (digest), digestctx, digestout );		\
 									\
-	ok ( pubkey_init ( (pubkey), ctx, (key), (key_len) ) == 0 );	\
+	ok ( pubkey_init ( (pubkey), ctx, (key) ) == 0 );		\
 	ok ( pubkey_verify ( (pubkey), ctx, (digest), digestout,	\
 			     (signature), (signature_len) ) != 0 );	\
 	pubkey_final ( (pubkey), ctx );					\
diff --git a/src/tests/rsa_test.c b/src/tests/rsa_test.c
index 46894f603..b1d522bc0 100644
--- a/src/tests/rsa_test.c
+++ b/src/tests/rsa_test.c
@@ -61,13 +61,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** An RSA encryption and decryption self-test */
 struct rsa_encrypt_decrypt_test {
 	/** Private key */
-	const void *private;
-	/** Private key length */
-	size_t private_len;
+	const struct asn1_cursor private;
 	/** Public key */
-	const void *public;
-	/** Public key length */
-	size_t public_len;
+	const struct asn1_cursor public;
 	/** Plaintext */
 	const void *plaintext;
 	/** Plaintext length */
@@ -100,10 +96,14 @@ struct rsa_encrypt_decrypt_test {
 	static const uint8_t name ## _plaintext[] = PLAINTEXT;		\
 	static const uint8_t name ## _ciphertext[] = CIPHERTEXT;	\
 	static struct rsa_encrypt_decrypt_test name = {			\
-		.private = name ## _private,				\
-		.private_len = sizeof ( name ## _private ),		\
-		.public = name ## _public,				\
-		.public_len = sizeof ( name ## _public ),		\
+		.private = {						\
+			.data = name ## _private,			\
+			.len = sizeof ( name ## _private ),		\
+		},							\
+		.public = {						\
+			.data = name ## _public,			\
+			.len = sizeof ( name ## _public ),		\
+		},							\
 		.plaintext = name ## _plaintext,			\
 		.plaintext_len = sizeof ( name ## _plaintext ),		\
 		.ciphertext = name ## _ciphertext,			\
@@ -113,13 +113,9 @@ struct rsa_encrypt_decrypt_test {
 /** An RSA signature self-test */
 struct rsa_signature_test {
 	/** Private key */
-	const void *private;
-	/** Private key length */
-	size_t private_len;
+	const struct asn1_cursor private;
 	/** Public key */
-	const void *public;
-	/** Public key length */
-	size_t public_len;
+	const struct asn1_cursor public;
 	/** Plaintext */
 	const void *plaintext;
 	/** Plaintext length */
@@ -150,10 +146,14 @@ struct rsa_signature_test {
 	static const uint8_t name ## _plaintext[] = PLAINTEXT;		\
 	static const uint8_t name ## _signature[] = SIGNATURE;		\
 	static struct rsa_signature_test name = {			\
-		.private = name ## _private,				\
-		.private_len = sizeof ( name ## _private ),		\
-		.public = name ## _public,				\
-		.public_len = sizeof ( name ## _public ),		\
+		.private = {						\
+			.data = name ## _private,			\
+			.len = sizeof ( name ## _private ),		\
+		},							\
+		.public = {						\
+			.data = name ## _public,			\
+			.len = sizeof ( name ## _public ),		\
+		},							\
 		.plaintext = name ## _plaintext,			\
 		.plaintext_len = sizeof ( name ## _plaintext ),		\
 		.algorithm = ALGORITHM,					\
@@ -167,17 +167,14 @@ struct rsa_signature_test {
  * @v test		RSA encryption and decryption test
  */
 #define rsa_encrypt_decrypt_ok( test ) do {				\
-	pubkey_decrypt_ok ( &rsa_algorithm, (test)->private,		\
-			    (test)->private_len, (test)->ciphertext,	\
-			    (test)->ciphertext_len, (test)->plaintext,	\
+	pubkey_decrypt_ok ( &rsa_algorithm, &(test)->private,		\
+			    (test)->ciphertext,	(test)->ciphertext_len,	\
+			    (test)->plaintext,  (test)->plaintext_len );\
+	pubkey_encrypt_ok ( &rsa_algorithm, &(test)->private,		\
+			    &(test)->public, (test)->plaintext,		\
 			    (test)->plaintext_len );			\
-	pubkey_encrypt_ok ( &rsa_algorithm, (test)->private,		\
-			    (test)->private_len, (test)->public,	\
-			    (test)->public_len, (test)->plaintext,	\
-			    (test)->plaintext_len );			\
-	pubkey_encrypt_ok ( &rsa_algorithm, (test)->public,		\
-			    (test)->public_len, (test)->private,	\
-			    (test)->private_len, (test)->plaintext,	\
+	pubkey_encrypt_ok ( &rsa_algorithm, &(test)->public,		\
+			    &(test)->private, (test)->plaintext,	\
 			    (test)->plaintext_len );			\
 	} while ( 0 )
 
@@ -190,18 +187,15 @@ struct rsa_signature_test {
 #define rsa_signature_ok( test ) do {					\
 	struct digest_algorithm *digest = (test)->algorithm->digest;	\
 	uint8_t bad_signature[ (test)->signature_len ];			\
-	pubkey_sign_ok ( &rsa_algorithm, (test)->private,		\
-			 (test)->private_len, digest,			\
+	pubkey_sign_ok ( &rsa_algorithm, &(test)->private, digest,	\
 			 (test)->plaintext, (test)->plaintext_len,	\
 			 (test)->signature, (test)->signature_len );	\
-	pubkey_verify_ok ( &rsa_algorithm, (test)->public,		\
-			   (test)->public_len, digest,			\
+	pubkey_verify_ok ( &rsa_algorithm, &(test)->public, digest,	\
 			   (test)->plaintext, (test)->plaintext_len,	\
 			   (test)->signature, (test)->signature_len );	\
 	memset ( bad_signature, 0, sizeof ( bad_signature ) );		\
-	pubkey_verify_fail_ok ( &rsa_algorithm, (test)->public,		\
-				(test)->public_len, digest,		\
-				(test)->plaintext,			\
+	pubkey_verify_fail_ok ( &rsa_algorithm, &(test)->public,	\
+				digest, (test)->plaintext,		\
 				(test)->plaintext_len, bad_signature,	\
 				sizeof ( bad_signature ) );		\
 	} while ( 0 )

From 633f4f362dbdf1f3177ee0f6b5b63085af7c1be5 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 18 Aug 2024 20:01:08 +0100
Subject: [PATCH 173/237] [test] Generalise public-key algorithm tests and use
 okx()

Generalise the existing support for performing RSA public-key
encryption, decryption, signature, and verification tests, and update
the code to use okx() for neater reporting of test results.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/tests/pubkey_test.c | 187 ++++++++++++++++++++++++++++
 src/tests/pubkey_test.h | 269 +++++++++++++++++++---------------------
 src/tests/rsa_test.c    | 189 +++-------------------------
 3 files changed, 336 insertions(+), 309 deletions(-)
 create mode 100644 src/tests/pubkey_test.c

diff --git a/src/tests/pubkey_test.c b/src/tests/pubkey_test.c
new file mode 100644
index 000000000..93962516a
--- /dev/null
+++ b/src/tests/pubkey_test.c
@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Public key self-tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <ipxe/crypto.h>
+#include <ipxe/test.h>
+#include "pubkey_test.h"
+
+/**
+ * Report public key encryption and decryption test result
+ *
+ * @v test		Public key encryption and decryption test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+void pubkey_okx ( struct pubkey_test *test, const char *file,
+		  unsigned int line ) {
+	struct pubkey_algorithm *pubkey = test->pubkey;
+	uint8_t private_ctx[pubkey->ctxsize];
+	uint8_t public_ctx[pubkey->ctxsize];
+	size_t max_len;
+
+	/* Initialize contexts */
+	okx ( pubkey_init ( pubkey, private_ctx, &test->private ) == 0,
+	      file, line );
+	okx ( pubkey_init ( pubkey, public_ctx, &test->public ) == 0,
+	      file, line );
+	max_len = pubkey_max_len ( pubkey, private_ctx );
+
+	/* Test decrypting with private key to obtain known plaintext */
+	{
+		uint8_t decrypted[max_len];
+		int decrypted_len;
+
+		decrypted_len = pubkey_decrypt ( pubkey, private_ctx,
+						 test->ciphertext,
+						 test->ciphertext_len,
+						 decrypted );
+		okx ( decrypted_len == ( ( int ) test->plaintext_len ),
+		      file, line );
+		okx ( memcmp ( decrypted, test->plaintext,
+			       test->plaintext_len ) == 0, file, line );
+	}
+
+	/* Test encrypting with private key and decrypting with public key */
+	{
+		uint8_t encrypted[max_len];
+		uint8_t decrypted[max_len];
+		int encrypted_len;
+		int decrypted_len;
+
+		encrypted_len = pubkey_encrypt ( pubkey, private_ctx,
+						 test->plaintext,
+						 test->plaintext_len,
+						 encrypted );
+		okx ( encrypted_len >= 0, file, line );
+		decrypted_len = pubkey_decrypt ( pubkey, public_ctx,
+						 encrypted, encrypted_len,
+						 decrypted );
+		okx ( decrypted_len == ( ( int ) test->plaintext_len ),
+		      file, line );
+		okx ( memcmp ( decrypted, test->plaintext,
+			       test->plaintext_len ) == 0, file, line );
+	}
+
+	/* Test encrypting with public key and decrypting with private key */
+	{
+		uint8_t encrypted[max_len];
+		uint8_t decrypted[max_len];
+		int encrypted_len;
+		int decrypted_len;
+
+		encrypted_len = pubkey_encrypt ( pubkey, public_ctx,
+						 test->plaintext,
+						 test->plaintext_len,
+						 encrypted );
+		okx ( encrypted_len >= 0, file, line );
+		decrypted_len = pubkey_decrypt ( pubkey, private_ctx,
+						 encrypted, encrypted_len,
+						 decrypted );
+		okx ( decrypted_len == ( ( int ) test->plaintext_len ),
+		      file, line );
+		okx ( memcmp ( decrypted, test->plaintext,
+			       test->plaintext_len ) == 0, file, line );
+	}
+
+	/* Free contexts */
+	pubkey_final ( pubkey, public_ctx );
+	pubkey_final ( pubkey, private_ctx );
+}
+
+/**
+ * Report public key signature test result
+ *
+ * @v test		Public key signature test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+void pubkey_sign_okx ( struct pubkey_sign_test *test, const char *file,
+		       unsigned int line ) {
+	struct pubkey_algorithm *pubkey = test->pubkey;
+	struct digest_algorithm *digest = test->digest;
+	uint8_t private_ctx[pubkey->ctxsize];
+	uint8_t public_ctx[pubkey->ctxsize];
+	uint8_t digestctx[digest->ctxsize ];
+	uint8_t digestout[digest->digestsize];
+	size_t max_len;
+
+	/* Initialize contexts */
+	okx ( pubkey_init ( pubkey, private_ctx, &test->private ) == 0,
+	      file, line );
+	okx ( pubkey_init ( pubkey, public_ctx, &test->public ) == 0,
+	      file, line );
+	max_len = pubkey_max_len ( pubkey, private_ctx );
+
+	/* Construct digest over plaintext */
+	digest_init ( digest, digestctx );
+	digest_update ( digest, digestctx, test->plaintext,
+			test->plaintext_len );
+	digest_final ( digest, digestctx, digestout );
+
+	/* Test signing using private key */
+	{
+		uint8_t signature[max_len];
+		int signature_len;
+
+		signature_len = pubkey_sign ( pubkey, private_ctx, digest,
+					      digestout, signature );
+		okx ( signature_len == ( ( int ) test->signature_len ),
+		      file, line );
+		okx ( memcmp ( signature, test->signature,
+			       test->signature_len ) == 0, file, line );
+	}
+
+	/* Test verification using public key */
+	okx ( pubkey_verify ( pubkey, public_ctx, digest, digestout,
+			      test->signature, test->signature_len ) == 0,
+	      file, line );
+
+	/* Test verification failure of modified signature */
+	{
+		uint8_t bad[test->signature_len];
+
+		memcpy ( bad, test->signature, test->signature_len );
+		bad[ test->signature_len / 2 ] ^= 0x40;
+		okx ( pubkey_verify ( pubkey, public_ctx, digest, digestout,
+				      bad, sizeof ( bad ) ) != 0, file, line );
+	}
+
+	/* Free contexts */
+	pubkey_final ( pubkey, public_ctx );
+	pubkey_final ( pubkey, private_ctx );
+}
diff --git a/src/tests/pubkey_test.h b/src/tests/pubkey_test.h
index 214992238..20bb94355 100644
--- a/src/tests/pubkey_test.h
+++ b/src/tests/pubkey_test.h
@@ -7,160 +7,151 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/crypto.h>
 #include <ipxe/test.h>
 
-/**
- * Report public key decryption test result
- *
- * @v pubkey		Public key algorithm
- * @v key		Key
- * @v ciphertext	Ciphertext
- * @v ciphertext_len	Ciphertext length
- * @v expected		Expected plaintext
- * @v expected_len	Expected plaintext length
- */
-#define pubkey_decrypt_ok( pubkey, key, ciphertext, ciphertext_len,	\
-			   expected, expected_len ) do {		\
-	uint8_t ctx[ (pubkey)->ctxsize ];				\
-									\
-	ok ( pubkey_init ( (pubkey), ctx, (key) ) == 0 );		\
-	{								\
-		size_t max_len = pubkey_max_len ( (pubkey), ctx );	\
-		uint8_t decrypted[ max_len ];				\
-		int decrypted_len;					\
-									\
-		decrypted_len = pubkey_decrypt ( (pubkey), ctx,		\
-						 (ciphertext),		\
-						 (ciphertext_len),	\
-						 decrypted );		\
-		ok ( decrypted_len == ( ( int ) (expected_len) ) );	\
-		ok ( memcmp ( decrypted, (expected),			\
-			      (expected_len) ) == 0 );			\
-	}								\
-	pubkey_final ( (pubkey), ctx );					\
-	} while ( 0 )
+/** A public-key encryption and decryption test */
+struct pubkey_test {
+	/** Public-key algorithm */
+	struct pubkey_algorithm *pubkey;
+	/** Private key */
+	const struct asn1_cursor private;
+	/** Public key */
+	const struct asn1_cursor public;
+	/** Plaintext */
+	const void *plaintext;
+	/** Length of plaintext */
+	size_t plaintext_len;
+	/** Ciphertext
+	 *
+	 * Note that the encryption process may include some random
+	 * padding, so a given plaintext will encrypt to multiple
+	 * different ciphertexts.
+	 */
+	const void *ciphertext;
+	/** Length of ciphertext */
+	size_t ciphertext_len;
+};
+
+/** A public-key signature test */
+struct pubkey_sign_test {
+	/** Public-key algorithm */
+	struct pubkey_algorithm *pubkey;
+	/** Private key */
+	const struct asn1_cursor private;
+	/** Public key */
+	const struct asn1_cursor public;
+	/** Plaintext */
+	const void *plaintext;
+	/** Plaintext length */
+	size_t plaintext_len;
+	/** Signature algorithm */
+	struct digest_algorithm *digest;
+	/** Signature */
+	const void *signature;
+	/** Signature length */
+	size_t signature_len;
+};
+
+/** Define inline private key data */
+#define PRIVATE(...) { __VA_ARGS__ }
+
+/** Define inline public key data */
+#define PUBLIC(...) { __VA_ARGS__ }
+
+/** Define inline plaintext data */
+#define PLAINTEXT(...) { __VA_ARGS__ }
+
+/** Define inline ciphertext data */
+#define CIPHERTEXT(...) { __VA_ARGS__ }
+
+/** Define inline signature data */
+#define SIGNATURE(...) { __VA_ARGS__ }
 
 /**
- * Report public key encryption and decryption test result
+ * Define a public-key encryption and decryption test
  *
- * @v pubkey		Public key algorithm
- * @v encrypt_key	Encryption key
- * @v decrypt_key	Decryption key
- * @v plaintext		Plaintext
- * @v plaintext_len	Plaintext length
+ * @v name		Test name
+ * @v PUBKEY		Public-key algorithm
+ * @v PRIVATE		Private key
+ * @v PUBLIC		Public key
+ * @v PLAINTEXT		Plaintext
+ * @v CIPHERTEXT	Ciphertext
+ * @ret test		Encryption and decryption test
  */
-#define pubkey_encrypt_ok( pubkey, encrypt_key, decrypt_key, plaintext,	\
-			   plaintext_len ) do {				\
-	uint8_t ctx[ (pubkey)->ctxsize ];				\
-									\
-	ok ( pubkey_init ( (pubkey), ctx, (encrypt_key) ) == 0 );	\
-	{								\
-		size_t max_len = pubkey_max_len ( (pubkey), ctx );	\
-		uint8_t encrypted[ max_len ];				\
-		int encrypted_len;					\
-									\
-		encrypted_len = pubkey_encrypt ( (pubkey), ctx,		\
-						 (plaintext),		\
-						 (plaintext_len),	\
-						 encrypted );		\
-		ok ( encrypted_len >= 0 );				\
-		pubkey_decrypt_ok ( (pubkey), (decrypt_key),		\
-				    encrypted, encrypted_len,		\
-				    (plaintext), (plaintext_len) );	\
-	}								\
-	pubkey_final ( (pubkey), ctx );					\
-	} while ( 0 )
+#define PUBKEY_TEST( name, PUBKEY, PRIVATE, PUBLIC, PLAINTEXT,		\
+			   CIPHERTEXT )					\
+	static const uint8_t name ## _private[] = PRIVATE;		\
+	static const uint8_t name ## _public[] = PUBLIC;		\
+	static const uint8_t name ## _plaintext[] = PLAINTEXT;		\
+	static const uint8_t name ## _ciphertext[] = CIPHERTEXT;	\
+	static struct pubkey_test name = {				\
+		.pubkey = PUBKEY,					\
+		.private = {						\
+			.data = name ## _private,			\
+			.len = sizeof ( name ## _private ),		\
+		},							\
+		.public = {						\
+			.data = name ## _public,			\
+			.len = sizeof ( name ## _public ),		\
+		},							\
+		.plaintext = name ## _plaintext,			\
+		.plaintext_len = sizeof ( name ## _plaintext ),		\
+		.ciphertext = name ## _ciphertext,			\
+		.ciphertext_len = sizeof ( name ## _ciphertext ),	\
+	}
 
 /**
- * Report public key signature test result
+ * Define a public-key signature test
  *
- * @v pubkey		Public key algorithm
- * @v key		Key
- * @v digest		Digest algorithm
- * @v plaintext		Plaintext
- * @v plaintext_len	Plaintext length
- * @v expected		Expected signature
- * @v expected_len	Expected signature length
+ * @v name		Test name
+ * @v PUBKEY		Public-key algorithm
+ * @v PRIVATE		Private key
+ * @v PUBLIC		Public key
+ * @v PLAINTEXT		Plaintext
+ * @v DIGEST		Digest algorithm
+ * @v SIGNATURE		Signature
+ * @ret test		Signature test
  */
-#define pubkey_sign_ok( pubkey, key, digest, plaintext,	plaintext_len,	\
-			expected, expected_len ) do {			\
-	uint8_t ctx[ (pubkey)->ctxsize ];				\
-	uint8_t digestctx[ (digest)->ctxsize ];				\
-	uint8_t digestout[ (digest)->digestsize ];			\
-									\
-	digest_init ( (digest), digestctx );				\
-	digest_update ( (digest), digestctx, (plaintext),		\
-			(plaintext_len) );				\
-	digest_final ( (digest), digestctx, digestout );		\
-									\
-	ok ( pubkey_init ( (pubkey), ctx, (key) ) == 0 );		\
-	{								\
-		size_t max_len = pubkey_max_len ( (pubkey), ctx );	\
-		uint8_t signature[ max_len ];				\
-		int signature_len;					\
-									\
-		signature_len = pubkey_sign ( (pubkey), ctx, (digest),	\
-					      digestout, signature );	\
-		ok ( signature_len == ( ( int ) (expected_len) ) );	\
-		ok ( memcmp ( signature, (expected),			\
-			      (expected_len) ) == 0 );			\
-	}								\
-	pubkey_final ( (pubkey), ctx );					\
-	} while ( 0 )
+#define PUBKEY_SIGN_TEST( name, PUBKEY, PRIVATE, PUBLIC, PLAINTEXT,	\
+			  DIGEST, SIGNATURE )				\
+	static const uint8_t name ## _private[] = PRIVATE;		\
+	static const uint8_t name ## _public[] = PUBLIC;		\
+	static const uint8_t name ## _plaintext[] = PLAINTEXT;		\
+	static const uint8_t name ## _signature[] = SIGNATURE;		\
+	static struct pubkey_sign_test name = {				\
+		.pubkey = PUBKEY,					\
+		.private = {						\
+			.data = name ## _private,			\
+			.len = sizeof ( name ## _private ),		\
+		},							\
+		.public = {						\
+			.data = name ## _public,			\
+			.len = sizeof ( name ## _public ),		\
+		},							\
+		.plaintext = name ## _plaintext,			\
+		.plaintext_len = sizeof ( name ## _plaintext ),		\
+		.digest = DIGEST,					\
+		.signature = name ## _signature,			\
+		.signature_len = sizeof ( name ## _signature ),		\
+	}
+
+extern void pubkey_okx ( struct pubkey_test *test,
+			 const char *file, unsigned int line );
+extern void pubkey_sign_okx ( struct pubkey_sign_test *test,
+			      const char *file, unsigned int line );
 
 /**
- * Report public key verification test result
+ * Report a public key encryption and decryption test result
  *
- * @v pubkey		Public key algorithm
- * @v key		Key
- * @v digest		Digest algorithm
- * @v plaintext		Plaintext
- * @v plaintext_len	Plaintext length
- * @v signature		Signature
- * @v signature_len	Signature length
+ * @v test		Public key encryption and decryption test
  */
-#define pubkey_verify_ok( pubkey, key, digest, plaintext,		\
-			  plaintext_len, signature, signature_len ) do {\
-	uint8_t ctx[ (pubkey)->ctxsize ];				\
-	uint8_t digestctx[ (digest)->ctxsize ];				\
-	uint8_t digestout[ (digest)->digestsize ];			\
-									\
-	digest_init ( (digest), digestctx );				\
-	digest_update ( (digest), digestctx, (plaintext),		\
-			(plaintext_len) );				\
-	digest_final ( (digest), digestctx, digestout );		\
-									\
-	ok ( pubkey_init ( (pubkey), ctx, (key) ) == 0 );		\
-	ok ( pubkey_verify ( (pubkey), ctx, (digest), digestout,	\
-			     (signature), (signature_len) ) == 0 );	\
-	pubkey_final ( (pubkey), ctx );					\
-	} while ( 0 )
+#define pubkey_ok( test ) \
+	pubkey_okx ( test, __FILE__, __LINE__ )
 
 /**
- * Report public key verification test result
+ * Report a public key signature test result
  *
- * @v pubkey		Public key algorithm
- * @v key		Key
- * @v digest		Digest algorithm
- * @v plaintext		Plaintext
- * @v plaintext_len	Plaintext length
- * @v signature		Signature
- * @v signature_len	Signature length
+ * @v test		Public key signature test
  */
-#define pubkey_verify_fail_ok( pubkey, key, digest, plaintext,		\
-			       plaintext_len, signature,		\
-			       signature_len ) do {			\
-	uint8_t ctx[ (pubkey)->ctxsize ];				\
-	uint8_t digestctx[ (digest)->ctxsize ];				\
-	uint8_t digestout[ (digest)->digestsize ];			\
-									\
-	digest_init ( (digest), digestctx );				\
-	digest_update ( (digest), digestctx, (plaintext),		\
-			(plaintext_len) );				\
-	digest_final ( (digest), digestctx, digestout );		\
-									\
-	ok ( pubkey_init ( (pubkey), ctx, (key) ) == 0 );		\
-	ok ( pubkey_verify ( (pubkey), ctx, (digest), digestout,	\
-			     (signature), (signature_len) ) != 0 );	\
-	pubkey_final ( (pubkey), ctx );					\
-	} while ( 0 )
+#define pubkey_sign_ok( test ) \
+	pubkey_sign_okx ( test, __FILE__, __LINE__ )
 
 #endif /* _PUBKEY_TEST_H */
diff --git a/src/tests/rsa_test.c b/src/tests/rsa_test.c
index b1d522bc0..13160934a 100644
--- a/src/tests/rsa_test.c
+++ b/src/tests/rsa_test.c
@@ -43,165 +43,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/test.h>
 #include "pubkey_test.h"
 
-/** Define inline private key data */
-#define PRIVATE(...) { __VA_ARGS__ }
-
-/** Define inline public key data */
-#define PUBLIC(...) { __VA_ARGS__ }
-
-/** Define inline plaintext data */
-#define PLAINTEXT(...) { __VA_ARGS__ }
-
-/** Define inline ciphertext data */
-#define CIPHERTEXT(...) { __VA_ARGS__ }
-
-/** Define inline signature data */
-#define SIGNATURE(...) { __VA_ARGS__ }
-
-/** An RSA encryption and decryption self-test */
-struct rsa_encrypt_decrypt_test {
-	/** Private key */
-	const struct asn1_cursor private;
-	/** Public key */
-	const struct asn1_cursor public;
-	/** Plaintext */
-	const void *plaintext;
-	/** Plaintext length */
-	size_t plaintext_len;
-	/** Ciphertext
-	 *
-	 * Note that the encryption process includes some random
-	 * padding, so a given plaintext will encrypt to multiple
-	 * different ciphertexts.
-	 */
-	const void *ciphertext;
-	/** Ciphertext length */
-	size_t ciphertext_len;
-};
-
-/**
- * Define an RSA encryption and decryption test
- *
- * @v name		Test name
- * @v PRIVATE		Private key
- * @v PUBLIC		Public key
- * @v PLAINTEXT		Plaintext
- * @v CIPHERTEXT	Ciphertext
- * @ret test		Encryption and decryption test
- */
-#define RSA_ENCRYPT_DECRYPT_TEST( name, PRIVATE, PUBLIC, PLAINTEXT,	\
-				  CIPHERTEXT )				\
-	static const uint8_t name ## _private[] = PRIVATE;		\
-	static const uint8_t name ## _public[] = PUBLIC;		\
-	static const uint8_t name ## _plaintext[] = PLAINTEXT;		\
-	static const uint8_t name ## _ciphertext[] = CIPHERTEXT;	\
-	static struct rsa_encrypt_decrypt_test name = {			\
-		.private = {						\
-			.data = name ## _private,			\
-			.len = sizeof ( name ## _private ),		\
-		},							\
-		.public = {						\
-			.data = name ## _public,			\
-			.len = sizeof ( name ## _public ),		\
-		},							\
-		.plaintext = name ## _plaintext,			\
-		.plaintext_len = sizeof ( name ## _plaintext ),		\
-		.ciphertext = name ## _ciphertext,			\
-		.ciphertext_len = sizeof ( name ## _ciphertext ),	\
-	}
-
-/** An RSA signature self-test */
-struct rsa_signature_test {
-	/** Private key */
-	const struct asn1_cursor private;
-	/** Public key */
-	const struct asn1_cursor public;
-	/** Plaintext */
-	const void *plaintext;
-	/** Plaintext length */
-	size_t plaintext_len;
-	/** Signature algorithm */
-	struct asn1_algorithm *algorithm;
-	/** Signature */
-	const void *signature;
-	/** Signature length */
-	size_t signature_len;
-};
-
-/**
- * Define an RSA signature test
- *
- * @v name		Test name
- * @v PRIVATE		Private key
- * @v PUBLIC		Public key
- * @v PLAINTEXT		Plaintext
- * @v ALGORITHM		Signature algorithm
- * @v SIGNATURE		Signature
- * @ret test		Signature test
- */
-#define RSA_SIGNATURE_TEST( name, PRIVATE, PUBLIC, PLAINTEXT,		\
-			    ALGORITHM, SIGNATURE )			\
-	static const uint8_t name ## _private[] = PRIVATE;		\
-	static const uint8_t name ## _public[] = PUBLIC;		\
-	static const uint8_t name ## _plaintext[] = PLAINTEXT;		\
-	static const uint8_t name ## _signature[] = SIGNATURE;		\
-	static struct rsa_signature_test name = {			\
-		.private = {						\
-			.data = name ## _private,			\
-			.len = sizeof ( name ## _private ),		\
-		},							\
-		.public = {						\
-			.data = name ## _public,			\
-			.len = sizeof ( name ## _public ),		\
-		},							\
-		.plaintext = name ## _plaintext,			\
-		.plaintext_len = sizeof ( name ## _plaintext ),		\
-		.algorithm = ALGORITHM,					\
-		.signature = name ## _signature,			\
-		.signature_len = sizeof ( name ## _signature ),		\
-	}
-
-/**
- * Report RSA encryption and decryption test result
- *
- * @v test		RSA encryption and decryption test
- */
-#define rsa_encrypt_decrypt_ok( test ) do {				\
-	pubkey_decrypt_ok ( &rsa_algorithm, &(test)->private,		\
-			    (test)->ciphertext,	(test)->ciphertext_len,	\
-			    (test)->plaintext,  (test)->plaintext_len );\
-	pubkey_encrypt_ok ( &rsa_algorithm, &(test)->private,		\
-			    &(test)->public, (test)->plaintext,		\
-			    (test)->plaintext_len );			\
-	pubkey_encrypt_ok ( &rsa_algorithm, &(test)->public,		\
-			    &(test)->private, (test)->plaintext,	\
-			    (test)->plaintext_len );			\
-	} while ( 0 )
-
-
-/**
- * Report RSA signature test result
- *
- * @v test		RSA signature test
- */
-#define rsa_signature_ok( test ) do {					\
-	struct digest_algorithm *digest = (test)->algorithm->digest;	\
-	uint8_t bad_signature[ (test)->signature_len ];			\
-	pubkey_sign_ok ( &rsa_algorithm, &(test)->private, digest,	\
-			 (test)->plaintext, (test)->plaintext_len,	\
-			 (test)->signature, (test)->signature_len );	\
-	pubkey_verify_ok ( &rsa_algorithm, &(test)->public, digest,	\
-			   (test)->plaintext, (test)->plaintext_len,	\
-			   (test)->signature, (test)->signature_len );	\
-	memset ( bad_signature, 0, sizeof ( bad_signature ) );		\
-	pubkey_verify_fail_ok ( &rsa_algorithm, &(test)->public,	\
-				digest, (test)->plaintext,		\
-				(test)->plaintext_len, bad_signature,	\
-				sizeof ( bad_signature ) );		\
-	} while ( 0 )
-
 /** "Hello world" encryption and decryption test (traditional PKCS#1 key) */
-RSA_ENCRYPT_DECRYPT_TEST ( hw_test,
+PUBKEY_TEST ( hw_test, &rsa_algorithm,
 	PRIVATE ( 0x30, 0x82, 0x01, 0x3b, 0x02, 0x01, 0x00, 0x02, 0x41, 0x00,
 		  0xd2, 0xf1, 0x04, 0x67, 0xf6, 0x2c, 0x96, 0x07, 0xa6, 0xbd,
 		  0x85, 0xac, 0xc1, 0x17, 0x5d, 0xe8, 0xf0, 0x93, 0x94, 0x0c,
@@ -255,7 +98,7 @@ RSA_ENCRYPT_DECRYPT_TEST ( hw_test,
 		     0x38, 0x43, 0xf9, 0x41 ) );
 
 /** "Hello world" encryption and decryption test (PKCS#8 key) */
-RSA_ENCRYPT_DECRYPT_TEST ( hw_test_pkcs8,
+PUBKEY_TEST ( hw_test_pkcs8, &rsa_algorithm,
 	PRIVATE ( 0x30, 0x82, 0x01, 0x55, 0x02, 0x01, 0x00, 0x30, 0x0d, 0x06,
 		  0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01,
 		  0x05, 0x00, 0x04, 0x82, 0x01, 0x3f, 0x30, 0x82, 0x01, 0x3b,
@@ -312,7 +155,7 @@ RSA_ENCRYPT_DECRYPT_TEST ( hw_test_pkcs8,
 		     0x38, 0x43, 0xf9, 0x41 ) );
 
 /** Random message MD5 signature test */
-RSA_SIGNATURE_TEST ( md5_test,
+PUBKEY_SIGN_TEST ( md5_test, &rsa_algorithm,
 	PRIVATE ( 0x30, 0x82, 0x01, 0x3b, 0x02, 0x01, 0x00, 0x02, 0x41, 0x00,
 		  0xf9, 0x3f, 0x78, 0x44, 0xe2, 0x0e, 0x25, 0xf1, 0x0e, 0x94,
 		  0xcd, 0xca, 0x6f, 0x9e, 0xea, 0x6d, 0xdf, 0xcd, 0xa0, 0x7c,
@@ -375,7 +218,7 @@ RSA_SIGNATURE_TEST ( md5_test,
 		    0xf2, 0x8d, 0xfc, 0xfc, 0x37, 0xf7, 0xc7, 0x6d, 0x6c, 0xd8,
 		    0x24, 0x0c, 0x6a, 0xec, 0x82, 0x5c, 0x72, 0xf1, 0xfc, 0x05,
 		    0xed, 0x8e, 0xe8, 0xd9, 0x8b, 0x8b, 0x67, 0x02, 0x95 ),
-	&md5_with_rsa_encryption_algorithm,
+	&md5_algorithm,
 	SIGNATURE ( 0xdb, 0x56, 0x3d, 0xea, 0xae, 0x81, 0x4b, 0x3b, 0x2e, 0x8e,
 		    0xb8, 0xee, 0x13, 0x61, 0xc6, 0xe7, 0xd7, 0x50, 0xcd, 0x0d,
 		    0x34, 0x3a, 0xfe, 0x9a, 0x8d, 0xf8, 0xfb, 0xd6, 0x7e, 0xbd,
@@ -385,7 +228,7 @@ RSA_SIGNATURE_TEST ( md5_test,
 		    0xac, 0x45, 0x00, 0xa8 ) );
 
 /** Random message SHA-1 signature test */
-RSA_SIGNATURE_TEST ( sha1_test,
+PUBKEY_SIGN_TEST ( sha1_test, &rsa_algorithm,
 	PRIVATE ( 0x30, 0x82, 0x01, 0x3b, 0x02, 0x01, 0x00, 0x02, 0x41, 0x00,
 		  0xe0, 0x3a, 0x8d, 0x35, 0xe1, 0x92, 0x2f, 0xea, 0x0d, 0x82,
 		  0x60, 0x2e, 0xb6, 0x0b, 0x02, 0xd3, 0xf4, 0x39, 0xfb, 0x06,
@@ -448,7 +291,7 @@ RSA_SIGNATURE_TEST ( sha1_test,
 		    0x30, 0x91, 0x1c, 0xaa, 0x6c, 0x24, 0x42, 0x1b, 0x1a, 0xba,
 		    0x30, 0x40, 0x49, 0x83, 0xd9, 0xd7, 0x66, 0x7e, 0x5c, 0x1a,
 		    0x4b, 0x7f, 0xa6, 0x8e, 0x8a, 0xd6, 0x0c, 0x65, 0x75 ),
-	&sha1_with_rsa_encryption_algorithm,
+	&sha1_algorithm,
 	SIGNATURE ( 0xa5, 0x5a, 0x8a, 0x67, 0x81, 0x76, 0x7e, 0xad, 0x99, 0x22,
 		    0xf1, 0x47, 0x64, 0xd2, 0xfb, 0x81, 0x45, 0xeb, 0x85, 0x56,
 		    0xf8, 0x7d, 0xb8, 0xec, 0x41, 0x17, 0x84, 0xf7, 0x2b, 0xbb,
@@ -458,7 +301,7 @@ RSA_SIGNATURE_TEST ( sha1_test,
 		    0x0e, 0x3d, 0x80, 0x80 ) );
 
 /** Random message SHA-256 signature test */
-RSA_SIGNATURE_TEST ( sha256_test,
+PUBKEY_SIGN_TEST ( sha256_test, &rsa_algorithm,
 	PRIVATE ( 0x30, 0x82, 0x01, 0x3a, 0x02, 0x01, 0x00, 0x02, 0x41, 0x00,
 		  0xa5, 0xe9, 0xdb, 0xa9, 0x1a, 0x6e, 0xd6, 0x4c, 0x25, 0x50,
 		  0xfe, 0x61, 0x77, 0x08, 0x7a, 0x80, 0x36, 0xcb, 0x88, 0x49,
@@ -521,7 +364,7 @@ RSA_SIGNATURE_TEST ( sha256_test,
 		    0x91, 0x71, 0xd6, 0x2d, 0xa1, 0xae, 0x81, 0x0c, 0xed, 0x54,
 		    0x48, 0x79, 0x8a, 0x78, 0x05, 0x74, 0x4d, 0x4f, 0xf0, 0xe0,
 		    0x3c, 0x41, 0x5c, 0x04, 0x0b, 0x68, 0x57, 0xc5, 0xd6 ),
-	&sha256_with_rsa_encryption_algorithm,
+	&sha256_algorithm,
 	SIGNATURE ( 0x02, 0x2e, 0xc5, 0x2a, 0x2b, 0x7f, 0xb4, 0x80, 0xca, 0x9d,
 		    0x96, 0x5b, 0xaf, 0x1f, 0x72, 0x5b, 0x6e, 0xf1, 0x69, 0x7f,
 		    0x4d, 0x41, 0xd5, 0x9f, 0x00, 0xdc, 0x47, 0xf4, 0x68, 0x8f,
@@ -536,11 +379,11 @@ RSA_SIGNATURE_TEST ( sha256_test,
  */
 static void rsa_test_exec ( void ) {
 
-	rsa_encrypt_decrypt_ok ( &hw_test );
-	rsa_encrypt_decrypt_ok ( &hw_test_pkcs8 );
-	rsa_signature_ok ( &md5_test );
-	rsa_signature_ok ( &sha1_test );
-	rsa_signature_ok ( &sha256_test );
+	pubkey_ok ( &hw_test );
+	pubkey_ok ( &hw_test_pkcs8 );
+	pubkey_sign_ok ( &md5_test );
+	pubkey_sign_ok ( &sha1_test );
+	pubkey_sign_ok ( &sha256_test );
 }
 
 /** RSA self-test */
@@ -548,3 +391,9 @@ struct self_test rsa_test __self_test = {
 	.name = "rsa",
 	.exec = rsa_test_exec,
 };
+
+/* Drag in required ASN.1 OID-identified algorithms */
+REQUIRING_SYMBOL ( rsa_test );
+REQUIRE_OBJECT ( rsa_md5 );
+REQUIRE_OBJECT ( rsa_sha1 );
+REQUIRE_OBJECT ( rsa_sha256 );

From ab5743efc57e84a45754d8ca9a188b7178cefebd Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 20 Aug 2024 10:11:24 +0100
Subject: [PATCH 174/237] [contrib] Remove obsolete rom-o-matic code

The rom-o-matic code does not form part of the iPXE codebase, has not
been maintained for over a decade, and does not appear to still be in
use anywhere in the world.

It does, however, result in a large number of false positive security
vulnerability reports from some low quality automated code analysis
tools such as Fortify SCA.

Remove this unused and obsolete code to reduce the burden of
responding to these false positives.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 contrib/rom-o-matic/README                    |  62 --
 contrib/rom-o-matic/bottom.php                |  62 --
 contrib/rom-o-matic/build.php                 | 311 --------
 contrib/rom-o-matic/customize-flags.php       |  69 --
 contrib/rom-o-matic/directions.php            |  63 --
 contrib/rom-o-matic/doc/AUTOBOOT_CMD.html     |   1 -
 contrib/rom-o-matic/doc/BANNER_TIMEOUT.html   |   1 -
 contrib/rom-o-matic/doc/COMCONSOLE.html       |   3 -
 contrib/rom-o-matic/doc/COMDATA.html          |   1 -
 contrib/rom-o-matic/doc/COMPARITY.html        |   1 -
 contrib/rom-o-matic/doc/COMPRESERVE.html      |   1 -
 contrib/rom-o-matic/doc/COMSPEED.html         |   1 -
 contrib/rom-o-matic/doc/COMSTOP.html          |   1 -
 contrib/rom-o-matic/doc/CONFIG_CMD.html       |   1 -
 contrib/rom-o-matic/doc/CONSOLE_PC_BIOS.html  |   1 -
 contrib/rom-o-matic/doc/CONSOLE_SERIAL.html   |   1 -
 contrib/rom-o-matic/doc/CRYPTO_80211_WEP.html |   1 -
 contrib/rom-o-matic/doc/CRYPTO_80211_WPA.html |   1 -
 .../rom-o-matic/doc/CRYPTO_80211_WPA2.html    |   1 -
 contrib/rom-o-matic/doc/DHCP_CMD.html         |   1 -
 contrib/rom-o-matic/doc/DNS_RESOLVER.html     |   1 -
 .../rom-o-matic/doc/DOWNLOAD_PROTO_FTP.html   |   1 -
 .../rom-o-matic/doc/DOWNLOAD_PROTO_HTTP.html  |   1 -
 .../rom-o-matic/doc/DOWNLOAD_PROTO_TFTP.html  |   1 -
 contrib/rom-o-matic/doc/IFMGMT_CMD.html       |   1 -
 contrib/rom-o-matic/doc/IMAGE_BZIMAGE.html    |   1 -
 contrib/rom-o-matic/doc/IMAGE_CMD.html        |   1 -
 contrib/rom-o-matic/doc/IMAGE_ELF.html        |   1 -
 contrib/rom-o-matic/doc/IMAGE_MULTIBOOT.html  |   1 -
 contrib/rom-o-matic/doc/IMAGE_NBI.html        |   1 -
 contrib/rom-o-matic/doc/IMAGE_PXE.html        |   1 -
 contrib/rom-o-matic/doc/IMAGE_SCRIPT.html     |   1 -
 contrib/rom-o-matic/doc/IWMGMT_CMD.html       |   1 -
 contrib/rom-o-matic/doc/NMB_RESOLVER.html     |   1 -
 contrib/rom-o-matic/doc/NVO_CMD.html          |   1 -
 contrib/rom-o-matic/doc/ROUTE_CMD.html        |   1 -
 contrib/rom-o-matic/doc/SANBOOT_CMD.html      |   1 -
 contrib/rom-o-matic/flag-table.php            | 531 --------------
 contrib/rom-o-matic/globals.php               |  51 --
 contrib/rom-o-matic/index.php                 |  47 --
 contrib/rom-o-matic/top.php                   |  41 --
 contrib/rom-o-matic/utils.php                 | 684 ------------------
 42 files changed, 1955 deletions(-)
 delete mode 100644 contrib/rom-o-matic/README
 delete mode 100644 contrib/rom-o-matic/bottom.php
 delete mode 100644 contrib/rom-o-matic/build.php
 delete mode 100644 contrib/rom-o-matic/customize-flags.php
 delete mode 100644 contrib/rom-o-matic/directions.php
 delete mode 100644 contrib/rom-o-matic/doc/AUTOBOOT_CMD.html
 delete mode 100644 contrib/rom-o-matic/doc/BANNER_TIMEOUT.html
 delete mode 100644 contrib/rom-o-matic/doc/COMCONSOLE.html
 delete mode 100644 contrib/rom-o-matic/doc/COMDATA.html
 delete mode 100644 contrib/rom-o-matic/doc/COMPARITY.html
 delete mode 100644 contrib/rom-o-matic/doc/COMPRESERVE.html
 delete mode 100644 contrib/rom-o-matic/doc/COMSPEED.html
 delete mode 100644 contrib/rom-o-matic/doc/COMSTOP.html
 delete mode 100644 contrib/rom-o-matic/doc/CONFIG_CMD.html
 delete mode 100644 contrib/rom-o-matic/doc/CONSOLE_PC_BIOS.html
 delete mode 100644 contrib/rom-o-matic/doc/CONSOLE_SERIAL.html
 delete mode 100644 contrib/rom-o-matic/doc/CRYPTO_80211_WEP.html
 delete mode 100644 contrib/rom-o-matic/doc/CRYPTO_80211_WPA.html
 delete mode 100644 contrib/rom-o-matic/doc/CRYPTO_80211_WPA2.html
 delete mode 100644 contrib/rom-o-matic/doc/DHCP_CMD.html
 delete mode 100644 contrib/rom-o-matic/doc/DNS_RESOLVER.html
 delete mode 100644 contrib/rom-o-matic/doc/DOWNLOAD_PROTO_FTP.html
 delete mode 100644 contrib/rom-o-matic/doc/DOWNLOAD_PROTO_HTTP.html
 delete mode 100644 contrib/rom-o-matic/doc/DOWNLOAD_PROTO_TFTP.html
 delete mode 100644 contrib/rom-o-matic/doc/IFMGMT_CMD.html
 delete mode 100644 contrib/rom-o-matic/doc/IMAGE_BZIMAGE.html
 delete mode 100644 contrib/rom-o-matic/doc/IMAGE_CMD.html
 delete mode 100644 contrib/rom-o-matic/doc/IMAGE_ELF.html
 delete mode 100644 contrib/rom-o-matic/doc/IMAGE_MULTIBOOT.html
 delete mode 100644 contrib/rom-o-matic/doc/IMAGE_NBI.html
 delete mode 100644 contrib/rom-o-matic/doc/IMAGE_PXE.html
 delete mode 100644 contrib/rom-o-matic/doc/IMAGE_SCRIPT.html
 delete mode 100644 contrib/rom-o-matic/doc/IWMGMT_CMD.html
 delete mode 100644 contrib/rom-o-matic/doc/NMB_RESOLVER.html
 delete mode 100644 contrib/rom-o-matic/doc/NVO_CMD.html
 delete mode 100644 contrib/rom-o-matic/doc/ROUTE_CMD.html
 delete mode 100644 contrib/rom-o-matic/doc/SANBOOT_CMD.html
 delete mode 100644 contrib/rom-o-matic/flag-table.php
 delete mode 100644 contrib/rom-o-matic/globals.php
 delete mode 100644 contrib/rom-o-matic/index.php
 delete mode 100644 contrib/rom-o-matic/top.php
 delete mode 100644 contrib/rom-o-matic/utils.php

diff --git a/contrib/rom-o-matic/README b/contrib/rom-o-matic/README
deleted file mode 100644
index b68cf775e..000000000
--- a/contrib/rom-o-matic/README
+++ /dev/null
@@ -1,62 +0,0 @@
-ROM-o-matic web interface for building iPXE ROMs
-------------------------------------------------
-
-This web application generates iPXE images and sends them to a web
-browser.
-
-Available as part of the iPXE source code distribution, which can be
-downlaoded from http://etherboot.org/
-
-Author:  Marty Connor <mdc@etherboot.org>
-License: GPLv2
-Support: http://etherboot.org/mailman/listinfo/ipxe
-         Please send support questions to the iPXE mailing list
-
-System Requirements
--------------------
-- Apache web server
-- PHP 4+
-- Tools required to build iPXE installed on the server
-  - gcc, mtools, syslinux, perl, etc.
-
-Setup
------
-As distributed, it is expected that the rom-o-matic source code
-directory is in the contrib directory of a iPXE source distribution.
-
-The easiest way to do this is to simply put a iPXE source distribution
-in a web server accessible directory.
-
-If this is not the case, you will need to either edit the file
-
-    "globals.php"
-
-or create a file called
-
-    "local-config.php"
-
-containing the following lines:
-
-<?php
-$src_dir = "../../src";
-?>
-
-Then change the line beginning "$src_dir = " to the path of your iPXE
-source code tree.
-
-To make build times shorter, before you run rom-o-matic for the first time
-you should cd to the ipxe "src" directory and enter the following
-commands:
-
-  $ make
-  $ make bin/NIC
-
-This will pro-compile most object files and will make your rom-o-matic
-builds much faster.
-
-Running rom-o-matic from a web browser
---------------------------------------
-Enter a URL like:
-
-   http://example.com/ipxe-1.x.x/contrib/rom-o-matic
-
diff --git a/contrib/rom-o-matic/bottom.php b/contrib/rom-o-matic/bottom.php
deleted file mode 100644
index 9ba8e3190..000000000
--- a/contrib/rom-o-matic/bottom.php
+++ /dev/null
@@ -1,62 +0,0 @@
-<?php
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-?>
-<hr>
-<h4>
-Resources:
-</h4>
-<ul>
-  <li>
-    Source code for iPXE images is available at
-    <a href="http://www.ipxe.org/download" target="_blank">
-    http://www.ipxe.org/download</a>
-    <br><br>
-  </li>
-  <li>
-    For general information about using iPXE, please visit the
-    <a href="http://www.ipxe.org/" target="_blank">
-    iPXE Project Home Page</a>
-    <br><br>
-  </li>
-  <li>
-    For Email-based support for iPXE please join
-    <a href="http://www.ipxe.org/contact" target="_blank">
-    iPXE Project mailing lists.</a>
-    <br><br>
-  </li>
-  <li>
-    For real-time online iPXE support via IRC please visit the
-    <a href="irc://irc.freenode.net/%23ipxe"> #ipxe channel
-    of irc.freenode.net</a>.
-    <br><br>
-  </li>
-</ul>
-<hr>
-  <font size="-1">
-    <br>
-    Please email <a href="mailto:<?php echo "${webmaster_email}" ?>"><?php echo "${webmaster_email}"?></a>
-    with questions or comments about this website.
-  </font>
-  <br><br>
-<hr>
-</body>
-</html>
diff --git a/contrib/rom-o-matic/build.php b/contrib/rom-o-matic/build.php
deleted file mode 100644
index b2b5bb452..000000000
--- a/contrib/rom-o-matic/build.php
+++ /dev/null
@@ -1,311 +0,0 @@
-<?php // -*- Mode: PHP; -*-
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-// Get utility functions and set globals
-require_once "utils.php";
-
-// Make sure at least $A (action)  was supplied
-if ( ! isset ( $_POST['A'] ) ) {
-
-    // Present user with form to customize build options
-    require_once "customize-flags.php";
-
-    exit ();
-
-// If user chose "Customize" option on form
-} else if ( $_POST['A'] == "Customize" ) {
-
-    // Present user with form to customize build options
-    require_once "customize-flags.php";
-
-    exit ();
-
-// The following conditional includes all other cases except "Get Image"
-// particularly the explicit ($A == "Start Over") case
-} else if ( $_POST['A'] != "Get Image" ) {
-
-    // Note that this method of redirections discards all the
-    // configuration flags, which is intentional in this case.
-
-    $dest = curDirURL ();
-    header ( "Location: $dest" );
-
-    // This next "echo" should normally not be seen, because
-    // the "header" statement above should cause immediate
-    // redirection but just in case...
-
-    echo "Try this link: <a href=\"$dest\">$dest</a>";
-
-    exit ();
-}
-
-// OK, we're going to try to use whatever options have been set
-// to build an image.
-
-// Make sure at least $nic was supplied
-if ( ! isset ( $_POST['nic'] ) ) {
-    die ( "No NIC supplied!" );
-}
-if ( isset ( $nics[$_POST['nic']] ) ) {
-    $nic = $nics[$_POST['nic']];
-} else {
-    die ( "Invalid NIC \"${_POST['nic']}\" supplied!" );
-}
-
-// Fetch flags
-$flags = get_flags ();
-
-// Get requested format
-$ofmt = isset ( $_POST['ofmt'] ) ? $_POST['ofmt'] : "";
-$fmt_extension = isset ( $ofmts[$ofmt] ) ? $ofmts[$ofmt] : 'dsk';
-
-// Handle some special cases
-
-$pci_vendor_code = "";
-$pci_device_code = "";
-
-if ( $nic == 'undionly' && $fmt_extension == "pxe" ) {
-
-    // undionly.pxe can't work because it unloads the PXE stack
-    // that it needs to communicate with, so we set the extension
-    // to .kpxe, which has a chance of working. The extension
-    // .kkpxe is another option.
-
-    $fmt_extension = "kpxe";
-
-} else if ( $fmt_extension == "rom" ) {
-
-    if ( ! isset ( $_POST['pci_vendor_code'] )
-		 || ! isset ( $_POST['pci_device_code'] ) ) {
-		die ( "rom output format selected but PCI code(s) missing!" );
-	}
-
-	$pci_vendor_code = $_POST['pci_vendor_code'];
-	$pci_device_code = $_POST['pci_device_code'];
-
-    if ( $pci_vendor_code == ""
-		 || $pci_device_code == "" ) {
-		die ( "rom output format selected but PCI code(s) missing!" );
-	}
-
-	// Try to be forgiving of 0xAAAA format
-	if ( strtolower ( substr ( $pci_vendor_code, 0, 2 ) ) == "0x"
-		 && strlen ( $pci_vendor_code ) == 6 ) {
-		$pci_vendor_code = substr ( $pci_vendor_code, 2, 4 );
-	}
-	if ( strtolower ( substr ( $pci_device_code, 0, 2 ) ) == "0x"
-		 && strlen ( $pci_device_code ) == 6 ) {
-		$pci_device_code = substr ( $pci_device_code, 2, 4 );
-	}
-
-    // concatenate the pci codes to get the $nic part of the
-    // Make target
-    $pci_codes = strtolower (  $pci_vendor_code . $pci_device_code );
-
-    $nic = $pci_codes;
-    if ( ! isset ( $roms[$pci_codes] ) ) {
-        die (   "Sorry, no network driver supports PCI codes<br>"
-              . "${_POST['pci_vendor_code']}:"
-              . "${_POST['pci_device_code']}" );
-    }
-} else if ( $fmt_extension != "rom"
-            && ( $pci_vendor_code != "" || $pci_device_code != "" ) ) {
-    die (   "'$fmt_extension' format was selected but PCI IDs were"
-          . " also entered.<br>Did you mean to select 'rom' output format"
-          . " instead?" );
-}
-
-/**
- * remove temporary build directory
- *
- * @return bool true if removal is successful, false otherwise
- */
-function rm_build_dir ()
-{
-    global $build_dir;
-    global $keep_build_dir;
-
-    if ( $keep_build_dir !== true ) {
-        rm_file_or_dir ( $build_dir );
-    }
-}
-
-// Arrange for the build directory to always be removed on exit.
-$build_dir = "";
-$keep_build_dir = false;
-register_shutdown_function ( 'rm_build_dir' );
-
-// Make temporary copy of src directory
-$build_dir = mktempcopy ( "$src_dir", "/tmp", "MDCROM" );
-$config_dir = $build_dir . "/config";
-
-// Write config files with supplied flags
-write_ipxe_config_files ( $config_dir, $flags );
-
-// Handle a possible embedded script
-$emb_script_cmd = "";
-$embedded_script = isset ( $_POST['embedded_script'] ) ? $_POST['embedded_script'] : "";
-if ( $embedded_script != "" ) {
-    $emb_script_path = "$build_dir" . "/script0.ipxe";
-
-	if ( substr ( $embedded_script, 0, 5 ) != "#!ipxe" ) {
-		$embedded_script = "#!ipxe\n" . $embedded_script;
-	}
-
-    // iPXE 0.9.7 doesn't like '\r\n" in the shebang...
-    $embedded_script = str_replace ( "\r\n", "\n", $embedded_script );
-
-    write_file_from_string ( $emb_script_path, $embedded_script );
-    $emb_script_cmd = "EMBEDDED_IMAGE=${emb_script_path}";
-}
-
-// Make the requested image.  $status is set to 0 on success
-$make_target = "bin/${nic}.${fmt_extension}";
-$gitversion = exec('git describe --always --abbrev=1 --match "" 2>/dev/null');
-if ($gitversion) {
-	$gitversion = "GITVERSION=$gitversion";
-}
-
-$make_cmd = "make -C '$build_dir' '$make_target' $gitversion $emb_script_cmd 2>&1";
-
-exec ( $make_cmd, $maketxt, $status );
-
-// Uncomment the following section for debugging
-
-/**
-
-echo "<h2>build.php:</h2>";
-echo "<h3>Begin debugging output</h3>";
-
-//echo "<h3>\$_POST variables</h3>";
-//echo "<pre>"; var_dump ( $_POST ); echo "</pre>";
-
-echo "<h3>Build options:</h3>";
-echo "<strong>Build directory is:</strong> $build_dir" . "<br><br>";
-echo "\$_POST['ofmt'] = " . "\"${_POST['ofmt']}\"" . "<br>";
-echo "\$_POST['nic'] = " . "\"${_POST['nic']}\"" .  "<br>";
-echo "\$_POST['pci_vendor_code'] = " . "\"${_POST['pci_vendor_code']}\"" . "<br>";
-echo "\$_POST['pci_device_code'] = " . "\"${_POST['pci_device_code']}\"" . "<br>";
-
-echo "<h3>Flags:</h3>";
-show_flags ( $flags );
-
-if ( $embedded_script != "" ) {
-    echo "<h3>Embedded script:</h3>";
-    echo "<blockquote>"."<pre>";
-    echo $embedded_script;
-    echo "</pre>"."</blockquote>";
-}
-
-echo "<h3>Make output:</h3>";
-echo "Make command: " . $make_cmd . "<br>";
-echo "Build status = <? echo $status ?>" . "<br>";
-echo "<blockquote>"."<pre>";
-echo htmlentities ( implode ("\n", $maketxt ) );
-echo "</pre>"."</blockquote>";
-// Uncomment the next line if you want to keep the
-// build directory around for inspection after building.
-$keep_build_dir = true;
-die ( "<h3>End debugging output</h3>" );
-
-**/ //   End debugging section
-
-// Send ROM to browser (with extreme prejudice)
-
-if ( $status == 0 ) {
-
-    $fp = fopen("${build_dir}/${make_target}", "rb" );
-    if ( $fp > 0 ) {
-
-        $len = filesize ( "${build_dir}/${make_target}" );
-        if ( $len > 0 ) {
-
-            $buf = fread ( $fp, $len );
-            fclose ( $fp );
-
-            // Delete build directory as soon as it is not needed
-            rm_build_dir ();
-
-            $output_filename = preg_replace('/[^a-z0-9\+\.\-]/i', '', "ipxe-${version}-${nic}.${fmt_extension}");
-
-            // Try to force IE to handle downloading right.
-            Header ( "Cache-control: private");
-            Header ( "Content-Type: application/x-octet-stream; " .
-                     "name=$output_filename");
-            Header ( "Content-Disposition: attachment; " .
-                     "Filename=$output_filename");
-            Header ( "Content-Location: $output_filename");
-            Header ( "Content-Length: $len");
-
-            echo $buf;
-
-            exit ();
-        }
-    }
-}
-
-/*
- * If we reach this point, the build has failed, and we provide
- * debugging information for a potential bug report
- *
- */
-
-// Remove build directory
-rm_build_dir ();
-
-// Announce failure if $status from make was non-zero
-echo "<h2>Build failed.  Status = " . $status . "</h2>";
-echo "<h2>build.php:</h2>";
-echo "<h3>Build options:</h3>";
-echo "<strong>Build directory is:</strong> $build_dir" . "<br><br>";
-echo "\$_POST['ofmt'] = " . "\"${_POST['ofmt']}\"" . "<br>";
-echo "\$_POST['nic'] = " . "\"${_POST['nic']}\"" .  "<br>";
-echo "\$_POST['pci_vendor_code'] = " . "\"${_POST['pci_vendor_code']}\"" . "<br>";
-echo "\$_POST['pci_device_code'] = " . "\"${_POST['pci_device_code']}\"" . "<br>";
-
-echo "<h3>Flags:</h3>";
-show_flags ( $flags );
-
-if ( $embedded_script != "" ) {
-    echo "<h3>Embedded script:</h3>";
-    echo "<blockquote>"."<pre>";
-    echo $embedded_script;
-    echo "</pre>"."</blockquote>";
-}
-
-echo "<h3>Make output:</h3>";
-echo "Make command: " . $make_cmd . "<br>";
-echo "<blockquote>"."<pre>";
-echo htmlentities ( implode ("\n", $maketxt ) );
-echo "</pre>"."</blockquote>";
-
-echo "Please let us know that this happened, and paste the above output into your email message.<br>";
-
-include_once $bottom_inc;
-
-// For emacs:
-//  Local variables:
-//  c-basic-offset: 4
-//  c-indent-level: 4
-//  tab-width: 4
-//  End:
-
-?>
diff --git a/contrib/rom-o-matic/customize-flags.php b/contrib/rom-o-matic/customize-flags.php
deleted file mode 100644
index e283921a8..000000000
--- a/contrib/rom-o-matic/customize-flags.php
+++ /dev/null
@@ -1,69 +0,0 @@
-<?php // -*- Mode: PHP; -*-
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-// Get utility functions and set globals
-require_once "utils.php";
-
-// Prepare settable compile options for presentation to user
-$flags = default_flags ();
-
-$build = "<input type=\"submit\" name=\"A\" value=\"Get Image\">";
-$restart = "<input type=\"submit\" name=\"A\" value=\"Start Over\">";
-
-// Begin html output
-include_once $top_inc;
-
-?>
-
-<form action="build.php" method=POST>
-  <input type="hidden" name="version" value = "<?php echo $version ?>">
-  <input type="hidden" name="use_flags" value="1">
-  <h3>
-    Make changes below and press <?php echo $build ?> to create an image, <br>
-    Or press <?php echo $restart ?> to return to the main page.
-  </h3>
-  <hr>
-  <ul>
-  <?php require ( "directions.php" ); ?>
-  </ul>
-  <hr>
-  <?php echo_flags( $flags ); ?>
-  <hr>
-  <h3>Embedded Script:</h3>
-  <?php echo textarea ( "embedded_script", "", "10", "50" ); ?>
-  <br><br>
-  <hr>
-  <center><table width="35%"><tr>
-  <td align="left"> <?php echo $build; ?> </td>
-  <td align="right"> <?php echo $restart ?></td>
-  </tr></table></center>
-</form>
-
-<?php include_once $bottom_inc; ?>
-<?
-// For emacs:
-//
-// Local variables:
-//  c-basic-offset: 4
-//  c-indent-level: 4
-//  tab-width: 4
-// End:
-?>
diff --git a/contrib/rom-o-matic/directions.php b/contrib/rom-o-matic/directions.php
deleted file mode 100644
index 540121edd..000000000
--- a/contrib/rom-o-matic/directions.php
+++ /dev/null
@@ -1,63 +0,0 @@
-<?php
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-?>
-    <li>
-      Choose an output format: <?php echo keys_menubox ( "ofmt", $ofmts,
-      isset ( $_POST['ofmt'] ) ? $_POST['ofmt'] : "") ?>
-      <br><br>
-    </li>
-    <li>
-      Choose a NIC type: <?php echo keys_menubox ( "nic", $nics,
-      isset ( $_POST['nic'] ) ? $_POST['nic'] : "" ) ?>
-      <br><br>
-    </li>
-    <li>
-      <strong>( optional &mdash; for binary ROM image format only )</strong> <br><br>
-      If you choose <em>Binary ROM image</em> as your output format, you must<br>
-      enter <strong>4 hex digits</strong> below for
-      <em>PCI VENDOR CODE</em> and <em>PCI DEVICE CODE</em>  <br>
-      that match the NIC device for which you are making this image.<br><br>
-      Information on how to determine NIC PCI IDs may be found
-      <a href="http://www.ipxe.org/howto/romburning"
-      target="_blank">here</a>.
-      <br><br>
-      PCI VENDOR CODE:  <?php echo textbox ( "pci_vendor_code",
-      isset ( $_POST['pci_vendor_code'] ) ? $_POST['pci_vendor_code']
-              : "", 6 ); ?>
-      &nbsp;&nbsp;
-      PCI DEVICE CODE:  <?php echo textbox ( "pci_device_code",
-      isset ( $_POST['pci_device_code'] ) ? $_POST['pci_device_code']
-              : "", 6 ); ?>
-      <h4>Please note for ROM images:</h4>
-      <ul>
-        <li>
-          If you enter PCI IDs, we will attempt to determine the correct<br>
-          driver to support them, and will ignore any NIC type entered
-          above.<br><br>
-        </li>
-        <li>
-          iPXE does not support all possible PCI IDs for supported
-          NICs.
-          <br><br>
-        </li>
-      </ul>
-    </li>
diff --git a/contrib/rom-o-matic/doc/AUTOBOOT_CMD.html b/contrib/rom-o-matic/doc/AUTOBOOT_CMD.html
deleted file mode 100644
index 444c5e602..000000000
--- a/contrib/rom-o-matic/doc/AUTOBOOT_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-Automatic booting
diff --git a/contrib/rom-o-matic/doc/BANNER_TIMEOUT.html b/contrib/rom-o-matic/doc/BANNER_TIMEOUT.html
deleted file mode 100644
index e135897f1..000000000
--- a/contrib/rom-o-matic/doc/BANNER_TIMEOUT.html
+++ /dev/null
@@ -1 +0,0 @@
-Tenths of a second for which the shell banner should appear
diff --git a/contrib/rom-o-matic/doc/COMCONSOLE.html b/contrib/rom-o-matic/doc/COMCONSOLE.html
deleted file mode 100644
index e7036c00e..000000000
--- a/contrib/rom-o-matic/doc/COMCONSOLE.html
+++ /dev/null
@@ -1,3 +0,0 @@
-Serial Console I/O port address.  Common addresses are:<br>
-COM1 =>	0x3f8, COM2 => 0x2f8, COM3 => 0x3e8, COM4 => 0x2e8
-
diff --git a/contrib/rom-o-matic/doc/COMDATA.html b/contrib/rom-o-matic/doc/COMDATA.html
deleted file mode 100644
index a27e27510..000000000
--- a/contrib/rom-o-matic/doc/COMDATA.html
+++ /dev/null
@@ -1 +0,0 @@
-Serial Console Data bits
diff --git a/contrib/rom-o-matic/doc/COMPARITY.html b/contrib/rom-o-matic/doc/COMPARITY.html
deleted file mode 100644
index 14f359517..000000000
--- a/contrib/rom-o-matic/doc/COMPARITY.html
+++ /dev/null
@@ -1 +0,0 @@
-Serial Console Parity: 0=None, 1=Odd, 2=Even
diff --git a/contrib/rom-o-matic/doc/COMPRESERVE.html b/contrib/rom-o-matic/doc/COMPRESERVE.html
deleted file mode 100644
index 6e41a10b3..000000000
--- a/contrib/rom-o-matic/doc/COMPRESERVE.html
+++ /dev/null
@@ -1 +0,0 @@
-Keep settings from a previous user of the serial port
\ No newline at end of file
diff --git a/contrib/rom-o-matic/doc/COMSPEED.html b/contrib/rom-o-matic/doc/COMSPEED.html
deleted file mode 100644
index 32b685957..000000000
--- a/contrib/rom-o-matic/doc/COMSPEED.html
+++ /dev/null
@@ -1 +0,0 @@
-Serial Console Baud rate
diff --git a/contrib/rom-o-matic/doc/COMSTOP.html b/contrib/rom-o-matic/doc/COMSTOP.html
deleted file mode 100644
index ae3fd24fb..000000000
--- a/contrib/rom-o-matic/doc/COMSTOP.html
+++ /dev/null
@@ -1 +0,0 @@
-Serial Console Stop bits
diff --git a/contrib/rom-o-matic/doc/CONFIG_CMD.html b/contrib/rom-o-matic/doc/CONFIG_CMD.html
deleted file mode 100644
index 1256c0694..000000000
--- a/contrib/rom-o-matic/doc/CONFIG_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-Option configuration console
diff --git a/contrib/rom-o-matic/doc/CONSOLE_PC_BIOS.html b/contrib/rom-o-matic/doc/CONSOLE_PC_BIOS.html
deleted file mode 100644
index 144eea3b9..000000000
--- a/contrib/rom-o-matic/doc/CONSOLE_PC_BIOS.html
+++ /dev/null
@@ -1 +0,0 @@
-Enable Default BIOS console
diff --git a/contrib/rom-o-matic/doc/CONSOLE_SERIAL.html b/contrib/rom-o-matic/doc/CONSOLE_SERIAL.html
deleted file mode 100644
index f35e2ff5a..000000000
--- a/contrib/rom-o-matic/doc/CONSOLE_SERIAL.html
+++ /dev/null
@@ -1 +0,0 @@
-Enable Serial port console
diff --git a/contrib/rom-o-matic/doc/CRYPTO_80211_WEP.html b/contrib/rom-o-matic/doc/CRYPTO_80211_WEP.html
deleted file mode 100644
index 26fdf8a85..000000000
--- a/contrib/rom-o-matic/doc/CRYPTO_80211_WEP.html
+++ /dev/null
@@ -1 +0,0 @@
-Wireless WEP encryption support
diff --git a/contrib/rom-o-matic/doc/CRYPTO_80211_WPA.html b/contrib/rom-o-matic/doc/CRYPTO_80211_WPA.html
deleted file mode 100644
index b218a1e91..000000000
--- a/contrib/rom-o-matic/doc/CRYPTO_80211_WPA.html
+++ /dev/null
@@ -1 +0,0 @@
-Wireless WPA encryption support
diff --git a/contrib/rom-o-matic/doc/CRYPTO_80211_WPA2.html b/contrib/rom-o-matic/doc/CRYPTO_80211_WPA2.html
deleted file mode 100644
index 947597d13..000000000
--- a/contrib/rom-o-matic/doc/CRYPTO_80211_WPA2.html
+++ /dev/null
@@ -1 +0,0 @@
-Wireless WPA2 encryption support
diff --git a/contrib/rom-o-matic/doc/DHCP_CMD.html b/contrib/rom-o-matic/doc/DHCP_CMD.html
deleted file mode 100644
index a0c31c7c0..000000000
--- a/contrib/rom-o-matic/doc/DHCP_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-DHCP management commands
diff --git a/contrib/rom-o-matic/doc/DNS_RESOLVER.html b/contrib/rom-o-matic/doc/DNS_RESOLVER.html
deleted file mode 100644
index 1029b9c52..000000000
--- a/contrib/rom-o-matic/doc/DNS_RESOLVER.html
+++ /dev/null
@@ -1 +0,0 @@
-DNS resolver
diff --git a/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_FTP.html b/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_FTP.html
deleted file mode 100644
index 7686d5d8a..000000000
--- a/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_FTP.html
+++ /dev/null
@@ -1 +0,0 @@
-File Transfer Protocol
diff --git a/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_HTTP.html b/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_HTTP.html
deleted file mode 100644
index c28d88868..000000000
--- a/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_HTTP.html
+++ /dev/null
@@ -1 +0,0 @@
-Hypertext Transfer Protocol
diff --git a/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_TFTP.html b/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_TFTP.html
deleted file mode 100644
index f2b31b17a..000000000
--- a/contrib/rom-o-matic/doc/DOWNLOAD_PROTO_TFTP.html
+++ /dev/null
@@ -1 +0,0 @@
-Trivial File Transfer Protocol
diff --git a/contrib/rom-o-matic/doc/IFMGMT_CMD.html b/contrib/rom-o-matic/doc/IFMGMT_CMD.html
deleted file mode 100644
index 0e2b2a5e8..000000000
--- a/contrib/rom-o-matic/doc/IFMGMT_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-Interface management commands
diff --git a/contrib/rom-o-matic/doc/IMAGE_BZIMAGE.html b/contrib/rom-o-matic/doc/IMAGE_BZIMAGE.html
deleted file mode 100644
index d85e5d079..000000000
--- a/contrib/rom-o-matic/doc/IMAGE_BZIMAGE.html
+++ /dev/null
@@ -1 +0,0 @@
-Linux bzImage image support
diff --git a/contrib/rom-o-matic/doc/IMAGE_CMD.html b/contrib/rom-o-matic/doc/IMAGE_CMD.html
deleted file mode 100644
index 6f5acb538..000000000
--- a/contrib/rom-o-matic/doc/IMAGE_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-Image management commands
diff --git a/contrib/rom-o-matic/doc/IMAGE_ELF.html b/contrib/rom-o-matic/doc/IMAGE_ELF.html
deleted file mode 100644
index 5e39e8bd4..000000000
--- a/contrib/rom-o-matic/doc/IMAGE_ELF.html
+++ /dev/null
@@ -1 +0,0 @@
-ELF image support
diff --git a/contrib/rom-o-matic/doc/IMAGE_MULTIBOOT.html b/contrib/rom-o-matic/doc/IMAGE_MULTIBOOT.html
deleted file mode 100644
index 6a092a203..000000000
--- a/contrib/rom-o-matic/doc/IMAGE_MULTIBOOT.html
+++ /dev/null
@@ -1 +0,0 @@
-MultiBoot image support
diff --git a/contrib/rom-o-matic/doc/IMAGE_NBI.html b/contrib/rom-o-matic/doc/IMAGE_NBI.html
deleted file mode 100644
index eb78e03c3..000000000
--- a/contrib/rom-o-matic/doc/IMAGE_NBI.html
+++ /dev/null
@@ -1 +0,0 @@
-NBI image support
diff --git a/contrib/rom-o-matic/doc/IMAGE_PXE.html b/contrib/rom-o-matic/doc/IMAGE_PXE.html
deleted file mode 100644
index bdca3841d..000000000
--- a/contrib/rom-o-matic/doc/IMAGE_PXE.html
+++ /dev/null
@@ -1 +0,0 @@
-PXE image support
diff --git a/contrib/rom-o-matic/doc/IMAGE_SCRIPT.html b/contrib/rom-o-matic/doc/IMAGE_SCRIPT.html
deleted file mode 100644
index 87416727f..000000000
--- a/contrib/rom-o-matic/doc/IMAGE_SCRIPT.html
+++ /dev/null
@@ -1 +0,0 @@
-iPXE script image support
diff --git a/contrib/rom-o-matic/doc/IWMGMT_CMD.html b/contrib/rom-o-matic/doc/IWMGMT_CMD.html
deleted file mode 100644
index 0d5bd4a6f..000000000
--- a/contrib/rom-o-matic/doc/IWMGMT_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-Wireless interface management commands
diff --git a/contrib/rom-o-matic/doc/NMB_RESOLVER.html b/contrib/rom-o-matic/doc/NMB_RESOLVER.html
deleted file mode 100644
index a0bdc17d3..000000000
--- a/contrib/rom-o-matic/doc/NMB_RESOLVER.html
+++ /dev/null
@@ -1 +0,0 @@
-NMB resolver
diff --git a/contrib/rom-o-matic/doc/NVO_CMD.html b/contrib/rom-o-matic/doc/NVO_CMD.html
deleted file mode 100644
index 5346f3f57..000000000
--- a/contrib/rom-o-matic/doc/NVO_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-Non-volatile option storage commands
diff --git a/contrib/rom-o-matic/doc/ROUTE_CMD.html b/contrib/rom-o-matic/doc/ROUTE_CMD.html
deleted file mode 100644
index 8114c265b..000000000
--- a/contrib/rom-o-matic/doc/ROUTE_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-Routing table management commands
diff --git a/contrib/rom-o-matic/doc/SANBOOT_CMD.html b/contrib/rom-o-matic/doc/SANBOOT_CMD.html
deleted file mode 100644
index 2e9d84075..000000000
--- a/contrib/rom-o-matic/doc/SANBOOT_CMD.html
+++ /dev/null
@@ -1 +0,0 @@
-SAN boot commands
diff --git a/contrib/rom-o-matic/flag-table.php b/contrib/rom-o-matic/flag-table.php
deleted file mode 100644
index fe81c8029..000000000
--- a/contrib/rom-o-matic/flag-table.php
+++ /dev/null
@@ -1,531 +0,0 @@
-<?php // -*- Mode: PHP; -*-
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-$ofmts = array
-	( "Floppy bootable image (.dsk)" => "dsk",
-	  "SYSLINUX-based bootable floppy image (.sdsk)" => "sdsk",
-	  "ISO bootable image (.iso)" => "iso",
-	  "ISO bootable image with legacy floppy emulation (.liso)" => "liso",
-	  "Linux kernel (SYSLINUX/GRUB/LILO) loadable image (.lkrn)" => "lkrn",
-	  "USB Keychain disk image (.usb)" => "usb",
-	  "ROM binary (flashable) image (.rom)" => "rom",
-	  "ROM binary (flashable) for problem PMM BIOSES  (.hrom)" => "hrom",
-	  "PXE bootstrap loader image [Unload PXE stack] (.pxe)" => "pxe",
-	  "PXE bootstrap loader keep [Keep PXE stack method 1] (.kpxe)" => "kpxe",
-	  "PXE bootstrap loader keep [Keep PXE stack method 2] (.kkpxe)" => "kkpxe",
-	);
-
-$flag_table = array (
-
-	// Begin General Options:
-
-	"HDR_MISC_OPTIONS"
-	=> array (
-	   "flag" => "HDR_MISC_OPTIONS",
-	   "hide_from_user" => "yes",  // Hide even the header
-	   "type" => "header",
-	   "label" => "Miscellaneous Options"
-		),
-
-	"PRODUCT_NAME"
-	=> array (
-	   "flag" => "PRODUCT_NAME",
-	   "hide_from_user" => "yes",
-	   "type" => "string",
-	   "value" => "",
-	   "cfgsec" => "general"
-	   ),
-
-	"PRODUCT_SHORT_NAME"
-	=> array (
-	   "flag" => "PRODUCT_SHORT_NAME",
-	   "hide_from_user" => "yes",
-	   "type" => "string",
-	   "value" => "iPXE",
-	   "cfgsec" => "general"
-	   ),
-
-	// End General Options:
-
-	// Begin Console Options:
-
-	"HDR_CONSOLE_OPTIONS"
-	=> array (
-	   "flag" => "HDR_CONSOLE_OPTIONS",
-	   "type" => "header",
-	   "label" => "Console Options"
-		),
-
-	"CONSOLE_PCBIOS"
-	=> array (
-	   "flag" => "CONSOLE_PCBIOS",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "console"
-	   ),
-
-	"CONSOLE_SERIAL"
-	=> array (
-	   "flag" => "CONSOLE_SERIAL",
-	   "type" => "on/off",
-	   "value" => "off",
-	   "cfgsec" => "console"
-	   ),
-
-	"BANNER_TIMEOUT"
-	=> array (
-	   "flag" => "BANNER_TIMEOUT",
-	   "type" => "integer",
-	   "value" => "20",
-	   "cfgsec" => "general"
-	   ),
-
-        "KEYBOARD_MAP"
-        => array (
-           "flag" => "KEYBOARD_MAP",
-           "type" => "choice",
-	   "options" => array("al","az","bg","by","cf","cz","de","dk","es","et","fi","fr",
-	      "gr","hu","il","it","lt","mk","mt","nl","no","pl","pt","ro","ru","sg","sr",
-	      "th","ua","uk","us","wo"),
-           "value" => "us",
-           "cfgsec" => "console"
-           ),
-
-	"LOG_LEVEL"
-	=> array (
-	   "flag" => "LOG_LEVEL",
-	   "type" => "choice",
-	   "options" => array("LOG_NONE","LOG_EMERG","LOG_ALERT","LOG_CRIT","LOG_ERR",
-	      "LOG_WARNING","LOG_NOTICE","LOG_INFO","LOG_DEBUG","LOG_ALL"),
-	   "value" => "LOG_NONE",
-	   "cfgsec" => "console"
-	   ),
-
-	// End Console Options
-
-	// Begin Network Protocol Options:
-
-	"HDR_NETWORK_PROTOCOL_OPTIONS"
-	=> array (
-	   "flag" => "HDR_NETWORK_PROTOCOL_OPTIONS",
-	   "hide_from_user" => "yes",  // Hide even the header
-	   "type" => "header",
-	   "label" => "Network Protocol Options"
-		),
-
-	"NET_PROTO_IPV4"
-	=> array (
-	   "flag" => "NET_PROTO_IPV4",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "hide_from_user" => "yes",
-	   "cfgsec" => "general"
-	   ),
-
-	// End Network Protocol Options
-
-	// Begin Serial Port configuration
-
-	"HDR_SERIAL_PORT_OPTIONS"
-	=> array (
-	   "flag" => "HDR_SERIAL_PORT_OPTIONS",
-	   "type" => "header",
-	   "label" => "Serial Port Options"
-		),
-
-	"COMCONSOLE"
-	=> array (
-	   "flag" => "COMCONSOLE",
-	   "type" => "integer-hex", // e.g. 0x378
-	   "value" => "0x3F8",
-	   "cfgsec" => "serial"
-		),
-
-	"COMPRESERVE"
-	=> array (
-	   "flag" => "COMPRESERVE",
-	   "type" => "on/off",
-	   "value" => "off",
-	   "cfgsec" => "serial"
-	   ),
-
-	"COMSPEED"
-	=> array (
-	   "flag" => "COMSPEED",
-	   "type" => "integer",
-	   "value" => "115200",
-	   "cfgsec" => "serial"
-	   ),
-
-	"COMDATA"
-	=> array (
-	   "flag" => "COMDATA",
-	   "type" => "integer",
-	   "value" => "8",
-	   "cfgsec" => "serial"
-	   ),
-
-	"COMPARITY"
-	=> array (
-	   "flag" => "COMPARITY",
-	   "type" => "integer",
-	   "value" => "0",
-	   "cfgsec" => "serial"
-	   ),
-
-	"COMSTOP"
-	=> array (
-	   "flag" => "COMSTOP",
-	   "type" => "integer",
-	   "value" => "1",
-	   "cfgsec" => "serial"
-	   ),
-
-	// End Serial Options
-
-	// Begin Download Protocols
-
-	"HDR_DOWNLOAD_PROTOCOLS"
-	=> array (
-	   "flag" => "HDR_DOWNLOAD_PROTOCOLS",
-	   "type" => "header",
-	   "label" => "Download Protocols"
-		),
-
-	"DOWNLOAD_PROTO_TFTP"
-	=> array (
-	   "flag" => "DOWNLOAD_PROTO_TFTP",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"DOWNLOAD_PROTO_HTTP"
-	=> array (
-	   "flag" => "DOWNLOAD_PROTO_HTTP",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"DOWNLOAD_PROTO_HTTPS"
-	=> array (
-	   "flag" => "DOWNLOAD_PROTO_HTTPS",
-	   "type" => "on/off",
-	   "value" => "off",
-	   "cfgsec" => "general"
-	   ),
-
-	"DOWNLOAD_PROTO_FTP"
-	=> array (
-	   "flag" => "DOWNLOAD_PROTO_FTP",
-	   "type" => "on/off",
-	   "value" => "off",
-	   "cfgsec" => "general"
-	   ),
-
-	// End Download Protocols
-
-	// Begin SAN boot protocols
-
-	"HDR_SANBOOT_PROTOCOLS"
-	=> array (
-	   "flag" => "HDR_SANBOOT_PROTOCOLS",
-	   "type" => "header",
-	   "label" => "SAN Boot Protocols"
-		),
-
-	"SANBOOT_PROTO_ISCSI"
-	=> array (
-	   "flag" => "SANBOOT_PROTO_ISCSI",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"SANBOOT_PROTO_AOE"
-	=> array (
-	   "flag" => "SANBOOT_PROTO_AOE",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	// End SAN boot protocols
-
-	// Begin Name resolution modules
-
-	"HDR_NAME_RESOLUTION_MODULES"
-	=> array (
-	   "flag" => "HDR_NAME_RESOLUTION_MODULES",
-	   "type" => "header",
-	   "label" => "Name Resolution Modules"
-	   ),
-
-	"DNS_RESOLVER"
-	=> array (
-	   "flag" => "DNS_RESOLVER",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-		),
-
-	"NMB_RESOLVER"
-	=> array (
-	   "flag" => "NMB_RESOLVER",
-	   "type" => "on/off",
-	   "value" => "off",
-	   "hide_from_user" => "yes",
-	   "cfgsec" => "general"
-		),
-
-	// End Name resolution modules
-
-	// Begin Image types
-
-	"HDR_IMAGE_TYPES"
-	=> array (
-	   "flag" => "HDR_IMAGE_TYPES",
-	   "type" => "header",
-	   "label" => "Image Types",
-	   ),
-
-	"IMAGE_ELF"
-	=> array (
-	   "flag" => "IMAGE_ELF",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"IMAGE_NBI"
-	=> array (
-	   "flag" => "IMAGE_NBI",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-		),
-
-	"IMAGE_MULTIBOOT"
-	=> array (
-	   "flag" => "IMAGE_MULTIBOOT",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"IMAGE_PXE"
-	=> array (
-	   "flag" => "IMAGE_PXE",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"IMAGE_SCRIPT"
-	=> array (
-	   "flag" => "IMAGE_SCRIPT",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"IMAGE_BZIMAGE"
-	=> array (
-	   "flag" => "IMAGE_BZIMAGE",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"IMAGE_COMBOOT"
-	=> array (
-	   "flag" => "IMAGE_COMBOOT",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	// End Image types
-
-	// Begin Command-line commands to include
-
-	"HDR_COMMAND_LINE_OPTIONS"
-	=> array (
-	   "flag" => "HDR_COMMAND_LINE_OPTIONS",
-	   "type" => "header",
-	   "label" => "Command Line Options",
-	   ),
-
-	"AUTOBOOT_CMD"
-	=> array (
-	   "flag" => "AUTOBOOT_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"NVO_CMD"
-	=> array (
-	   "flag" => "NVO_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"CONFIG_CMD"
-	=> array (
-	   "flag" => "CONFIG_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"IFMGMT_CMD"
-	=> array (
-	   "flag" => "IFMGMT_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"IWMGMT_CMD"
-	=> array (
-	   "flag" => "IWMGMT_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"ROUTE_CMD"
-	=> array (
-	   "flag" => "ROUTE_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"IMAGE_CMD"
-	=> array (
-	   "flag" => "IMAGE_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"DHCP_CMD"
-	=> array (
-	   "flag" => "DHCP_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-		),
-
-	"SANBOOT_CMD"
-	=> array (
-	   "flag" => "SANBOOT_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-		),
-
-	"LOGIN_CMD"
-	=> array (
-	   "flag" => "LOGIN_CMD",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-		),
-
-	"TIME_CMD"
-	=> array (
-	   "flag" => "TIME_CMD",
-	   "type" => "on/off",
-	   "value" => "off",
-	   "cfgsec" => "general"
-		),
-
-	"DIGEST_CMD"
-	=> array (
-	   "flag" => "DIGEST_CMD",
-	   "type" => "on/off",
-	   "value" => "off",
-	   "cfgsec" => "general"
-		),
-
-	// End Command-line commands to include
-
-	// Begin Wireless options
-
-	"HDR_WIRELESS_OPTIONS"
-	=> array (
-	   "flag" => "HDR_WIRELESS_OPTIONS",
-	   "type" => "header",
-	   "label" => "Wireless Interface Options",
-	   ),
-
-	"CRYPTO_80211_WEP"
-	=> array (
-	   "flag" => "CRYPTO_80211_WEP",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"CRYPTO_80211_WPA"
-	=> array (
-	   "flag" => "CRYPTO_80211_WPA",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	"CRYPTO_80211_WPA2"
-	=> array (
-	   "flag" => "CRYPTO_80211_WPA2",
-	   "type" => "on/off",
-	   "value" => "on",
-	   "cfgsec" => "general"
-	   ),
-
-	// End Wireless options
-
-	// Obscure options required to compile
-	"NETDEV_DISCARD_RATE"
-	=> array (
-	   "flag" => "NETDEV_DISCARD_RATE",
-	   "type" => "integer",
-	   "value" => "0",
-	   "cfgsec" => "general",
-	   "hide_from_user" => true
-	   )
-
-	// End Obscure options
-);
-
-// For emacs:
-// Local variables:
-//	c-basic-offset: 4
-//	c-indent-level: 4
-//	tab-width: 4
-// End:
-
-?>
diff --git a/contrib/rom-o-matic/globals.php b/contrib/rom-o-matic/globals.php
deleted file mode 100644
index 822e4bc0b..000000000
--- a/contrib/rom-o-matic/globals.php
+++ /dev/null
@@ -1,51 +0,0 @@
-<?php // -*- Mode: PHP; -*-
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-// Directory containing iPXE source code tree
-$src_dir = "../../src";
-
-// Compute iPXE version based on source tree
-exec ( "make -C '$src_dir' version 2>&1", $make_output, $status );
-$version = ( $status == 0 && count ( $make_output  ) > 1 )
-           ? trim ( $make_output[count ( $make_output ) - 2] )
-           : "";
-
-// Email address of person responsible for this website
-$webmaster_email = "webmaster@example.com";
-
-// Files that header and footer text
-$top_inc = "top.php";
-$bottom_inc = "bottom.php";
-
-// Descriptive strings
-$header_title = "ROM-o-matic for iPXE $version";
-$html_tagline = "ROM-o-matic dynamically generates iPXE images";
-$html_title   = "ROM-o-matic for iPXE $version";
-$description  = "a dynamic iPXE image generator";
-
-// For emacs:
-// Local variables:
-//  c-basic-offset: 4
-//  c-indent-level: 4
-//  tab-width: 4
-// End:
-
-?>
diff --git a/contrib/rom-o-matic/index.php b/contrib/rom-o-matic/index.php
deleted file mode 100644
index 26585c975..000000000
--- a/contrib/rom-o-matic/index.php
+++ /dev/null
@@ -1,47 +0,0 @@
-<?php // -*- Mode: PHP; -*-
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-// Get utility functions and set globals
-require_once "utils.php";
-
-// Begin html output
-include_once $top_inc;
-
-?>
-<form action="build.php" method=POST>
-  <input type="hidden" name="version" value = "<?php echo $version ?>">
-  <h3>To create an image:</h3>
-  <ol>
-    <?php require ( "directions.php" ); ?>
-    <li>
-      Generate and download an image:
-      <input type="submit" name="A" value="Get Image">
-      <br><br>
-    </li>
-    <li>
-      (optional) Customize image configuration options:
-      <input type="submit" name="A" value="Customize">
-      <br><br>
-    </li>
-  </ol>
-</form>
-
-<?php include_once $bottom_inc ?>
diff --git a/contrib/rom-o-matic/top.php b/contrib/rom-o-matic/top.php
deleted file mode 100644
index 42a8e2d27..000000000
--- a/contrib/rom-o-matic/top.php
+++ /dev/null
@@ -1,41 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
-
-<?php
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-?>
-
-<html>
-<head>
-  <link rev="made" href="mailto:<?php echo "${webmaster_email}" ?>">
-  <meta name="keywords" content="rom, etherboot, ipxe, open source, rom-o-matic.net">
-  <title><?php echo $header_title ?></title>
-  <meta name="description" content="<?php echo $description ?>">
-</head>
-<h1>
-<?php echo "$html_title" ?>&nbsp;
-</h1>
-<hr>
-<h2>
-<?php echo "$html_tagline" ?>
-</h2>
-</form>
-<hr>
diff --git a/contrib/rom-o-matic/utils.php b/contrib/rom-o-matic/utils.php
deleted file mode 100644
index e0e62f447..000000000
--- a/contrib/rom-o-matic/utils.php
+++ /dev/null
@@ -1,684 +0,0 @@
-<?php // -*- Mode: PHP; -*-
-
-/**
- * Copyright (C) 2009 Marty Connor <mdc@etherboot.org>.
- * Copyright (C) 2009 Entity Cyber, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-// Include table of user-configurable iPXE options
-require_once "flag-table.php";
-
-// Include user-shadowable globals
-require_once "globals.php";
-
-// Allow user to shadow globals
-if ( is_file ( 'local-config.php' ) ) {
-    include_once "local-config.php";
-}
-
-////
-// General utility functions
-////
-
-/**
- * Remove undesirable characters from a given string
- *
- * Certain characters have the potential to be used for
- * malicious purposes by web-based attackers.  This routine
- * filters out such characters.
- *
- * @param string $s supplied string
- *
- * @return string returned string with unwanted characters
- *                removed
- */
-function cleanstring ( $s )
-{
-    $len = strlen ( $s );
-    if ( $len > 80 ) {
-        $s = substr ( $s, 0, 80 );
-    }
-
-    $s      = trim ( $s );
-    $pos    = 0;
-    $result = "";
-
-    while ( $pos < $len ) {
-        $ltr = ord ( ucfirst ( $s[$pos] ) );
-        if ( ( $ltr >= ord ( "A" ) ) && ( $ltr <= ord ( "Z" ) ) ||
-             ( $ltr >= ord ( "0" ) ) && ( $ltr <= ord ( "9" ) ) ||
-             ( $ltr == ord ( "." ) ) && ( strlen ( $result ) > 0 ) ||
-             ( $ltr == ord ( "_" ) ) ||
-             ( $ltr == ord ( "+" ) ) ||
-             ( $ltr == ord ( ":" ) ) ||
-             ( $ltr == ord ( "/" ) ) ||
-             ( $ltr == ord ( "-" ) ) ) {
-            $result .= $s[$pos];
-        }
-        $pos++;
-    }
-    return $result;
-}
-
-/**
- * Return URL of the currently running script, minus the filename
- *
- * @return string the URL of the currently running script, minus the filename
- */
-function curDirURL ()
-{
-        $dir = dirname ( $_SERVER['PHP_SELF'] );
-
-        if ( $dir == "." || $dir == "/" ) {
-                $dir = "";
-        }
-
-        $isHTTPS = ( isset ( $_SERVER["HTTPS"] ) && $_SERVER["HTTPS"] == "on" );
-        $port = ( isset($_SERVER["SERVER_PORT"] ) &&
-                          ( ( !$isHTTPS && $_SERVER["SERVER_PORT"] != "80" ) ||
-                                ( $isHTTPS  && $_SERVER["SERVER_PORT"] != "443" ) ) );
-
-        $port = ( $port ) ? ':' . $_SERVER["SERVER_PORT"] : '';
-
-        $dest = ( $isHTTPS ? 'https://' : 'http://' ) .
-                $_SERVER["SERVER_NAME"] . $dir . "/";
-
-        return $dest;
-}
-
-/**
- * Extract NIC families and associated ROM PCI IDs from the src/bin/NIC file.
- *
- * $src_dir must contain the path of the iPXE src directory for this build
- *
- * @return array[0] array $new_nics
- * @return array[1] array $roms
- */
-function parse_nic_file ()
-{
-    global $src_dir;
-
-    $fd = fopen ( "$src_dir/bin/NIC", "r" );
-    if ( ! $fd ) {
-        die ( "Missing src/bin/NIC file.  'make bin/NIC'" );
-    }
-
-    $nics = array ();
-    $roms = array ();
-    $nic = "";
-
-    while ( !feof ( $fd ) ) {
-
-        $line = trim ( fgets ( $fd, 200 ) );
-
-        $first_eight_chars = substr ( $line, 0, 8 );
-        settype ( $first_eight_chars, "string" );
-
-        if ( strpos ( $first_eight_chars, "family" ) === 0 ) {
-
-            // get pathname of NIC driver
-            #list ( $dummy, $nic ) = split( "[ \t]+", $line );
-            list ( $dummy, $nic ) = explode("\t", $line);
-            settype ( $nic, "string" );
-
-            // extract filename name of driver from pathname
-            $nic = substr ( $nic, strrpos ( $nic, "/" ) + 1,
-			   strlen ( $nic ) - strrpos ( $nic, "/" ) + 1 );
-
-            $nics[$nic] = $nic;
-
-            // For each ISA NIC, there can only be one ROM variant
-            $roms[$nic] = $nic;
-        }
-
-        // If the first 8 digits of the line are hex digits
-        // add this rom to the current nic family.
-
-        if (    ( strlen ( $first_eight_chars ) == 8 )
-             && ( ctype_xdigit ( $first_eight_chars ) )
-             && ( $nic != "" ) ) {
-
-            $roms[$first_eight_chars] = $nic;
-        }
-    }
-    fclose ( $fd );
-
-    // put most NICs in nice alpha order for menu
-    ksort ( $nics );
-
-    // add special cases to the top
-
-	$new_nics = array ( "all-drivers" => "ipxe",
-						"undionly" => "undionly",
-						"undi" => "undi",
-    );
-
-	foreach ( $nics as $key => $value ) {
-		// skip the undi driver
-		if ( $key != "undi" ) {
-			$new_nics[$key] = $value;
-		}
-	}
-
-	return array ( $new_nics, $roms );
-}
-
-////
-// HTML form utility functions
-////
-
-/**
- * Return html code to create hidden form input fields
- *
- * @param string $flag  name of form variable to set
- * @param string $value value to give form variable
- *
- * @return string html code for given hidden form input field
- */
-function hidden ( $flag, $value )
-{
-    $value = htmlentities ( $value );
-    return "<input type=\"hidden\" value=\"$value\" name=\"$flag\"></input>";
-}
-
-/**
- * Return html code to create checkbox form input fields
- *
- * @param string $flag  name of form variable to set
- * @param string $value "on" means box should be checked
- *
- * @return string html code for given hidden form input field
- */
-function checkbox ( $flag, $value )
-{
-    return "<input type=\"checkbox\" value=\"on\" name=\"$flag\"" .
-        ($value == "on" ? " checked>" : ">" );
-}
-
-/**
- * Return html code to create text form input fields
- *
- * @param string $flag  name of form variable to set
- * @param string $value initial contents of field
- * @param string $size  size in characters of text box
- *
- * @return string html code for given text input field
- */
-function textbox ( $flag, $value, $size )
-{
-    $value = htmlentities ( $value );
-    return "<input type=\"text\" size=\"$size\" value=\"$value\" name=\"$flag\">";
-}
-
-/**
- * Return html code to create textarea form fields
- *
- * @param string $flag  name of form variable to set
- * @param string $value initial contents of textarea
- * @param string $rows  height of text area in rows
- * @param string $cols  width of text area in columns
- *
- * @return string html code for given textarea input field
- */
-function textarea ( $flag, $value, $rows, $cols )
-{
-    $value = htmlentities ( $value );
-    return "<textarea name=\"$flag\" rows=\"$rows\" cols=\"$cols\">"
-            . $value . "</textarea>";
-}
-
-/**
- * Return html code to create select (menu) form fields
- *
- * Use array of strings as menu choices
- *
- * @param string $flag    name of form variable to set
- * @param array  $options array of strings representing choices
- * @param string $value   value of choice to select in menu
- *
- * @return string html code for given select (menu) input field
- */
-function menubox ( $name, $options, $value )
-{
-    $s="<select name=\"$name\">";
-
-	foreach ( $options as $ignore => $option ) {
-        if ( !$value ) $value = $option;
-        $s .= "<option" . ( $option == $value ? " selected>" : ">" ) .
-            htmlentities ( $option ) . "</option>";
-    }
-    return $s . "</select>";
-}
-
-/**
- * Return html code to create select (menu) form fields
- *
- * Use indices of array of strings as menu choices rather than
- * the values pointed to by the indicies.
- *
- * @param string $flag    name of form variable to set
- * @param array  $options array of strings representing choices
- * @param string $value   value of choice to select in menu
- *
- * @return string html code for given select (menu) input field
- */
-function keys_menubox ( $name, $options, $value )
-{
-    $s="<select name=\"$name\">";
-
-    foreach ( $options as $option => $ignore ) {
-        if ( !$value ) $value = $option;
-        $s .= "<option" . ( $option == $value ? " selected>" : ">" ) .
-            htmlentities ( $option ) . "</option>";
-    }
-    return $s . "</select>";
-}
-
-////
-// Flag (compile option) handling functions
-////
-
-/**
- * Return default compile options (flags)
- *
- * Initial compile options are in a global called $flag_table.
- * Create and return an array containing the ones we want.
- *
- * @return array default compile options (flags)
- */
-function default_flags ()
-{
-    global $flag_table;
-
-    $flags = array ();
-
-    foreach ( $flag_table as $key => $props ) {
-
-        $flag  = $props["flag"];
-        $type  = $props["type"];
-
-        // Fields like headers have no "value" property
-        if ( isset ( $props["value"] ) ) {
-            $flags[$flag] = $props["value"];
-        }
-    }
-    return $flags;
-}
-
-/**
- * Return combination of default and user compile options (flags)
- *
- * Initial compile options are in a global called $flag_table.
- * Compile options may have been changed via form input. We return
- * an array with either the default value of each option or a user
- * supplied value from form input.
- *
- * @return array combined default and user supplied compile options (flags)
- */
-function get_flags ()
-{
-    global $flag_table;
-
-    $flags = default_flags ();
-
-    if ( ! isset ( $_POST["use_flags"] ) )
-        return $flags;
-
-    foreach ( $flag_table as $key => $props ) {
-
-        $flag = $props["flag"];
-        $type = $props["type"];
-
-        if ( isset ( $_POST["$flag"] ) ) {
-            $flags[$flag] = $_POST["$flag"];
-            if ( $type == "integer-hex" ) {
-                if ( strtolower ( substr ( $flags[$flag], 0, 2 ) ) != "0x" ) {
-                    $flags[$flag] = "0x" . $flags[$flag];
-                }
-            }
-        } else if ( $type == "on/off" ) {
-			// Unchecked checkboxes don't pass any POST value
-			// so we must check for them specially.  At this
-			// point we know that there is no $_POST value set
-			// for this option.  If it is a checkbox, this means
-			// it is unchecked, so record that in $flags so we
-			// can later generate an #undef for this option.
-            $flags[$flag] = "off";
-        }
-    }
-    return $flags;
-}
-
-/**
- * Output given value in appropriate format for iPXE config file
- *
- * iPXE config/*.h files use C pre-processor syntax.  Output the given
- * compile option in a format appropriate to its type
- *
- * @param string $key   index into $flag_table for given compile option
- * @param string $value value we wish to set compile option to
- *
- * @return string code to set compile option to given value
- */
-function pprint_flag ( $key, $value )
-{
-    global $flag_table;
-
-    // Determine type of given compile option (flag)
-    $type = $flag_table[$key]["type"];
-    $s = "";
-
-    if ( $type == "on/off" && $value == "on" ) {
-        $s = "#define $key";
-    } else if ( $type == "on/off" && $value != "on" ) {
-        $s = "#undef $key";
-    } else if ( $type == "string" ) {
-        $s = ( "#define $key \"" . cleanstring ( $value ) . "\"" );
-    } else if ($type == "qstring" ) {
-        $s = ( "#define $key \\\"" . cleanstring ( $value ) . "\\\"" );
-    } else {
-        $s = "#define $key " . cleanstring ( $value );
-    }
-
-    return $s;
-}
-
-/**
- * Output html code to display all compile options as a table
- *
- * @param array $flags array of compile options
- *
- * @return void
- */
-function echo_flags ( $flags )
-{
-    global $flag_table;
-
-    echo "<table>\n";
-
-	foreach ( $flag_table as $key => $props ) {
-
-        // Hide parameters from users that should not be changed.
-        $hide_from_user = isset ( $props["hide_from_user"] ) ? $props["hide_from_user"] : "no";
-
-        $flag = $props["flag"];
-        $type = $props["type"];
-
-        $value = isset ( $flags[$flag] ) ? $flags[$flag] : '';
-
-        if ( $hide_from_user == "yes" ) {
-
-            // Hidden flags cannot not be set by the user.  We use hidden form
-            // fields to keep them at their default values.
-            if ( $type != "header" ) {
-                echo hidden ( $flag, $value );
-            }
-
-        } else {
-
-            // Flag (iPXE compile option) should be displayed to user
-
-            if ( $type == "header" ) {
-
-                $label = $props["label"];
-                echo "<td colspan=2><hr><h3>$label</h3><hr></td>";
-
-            } else if ($type == "on/off" ) {
-
-                echo "<td>", checkbox ( $flag, $value ), "</td><td><strong>$flag</strong></td>";
-
-            } else {   // don't display checkbox for non-on/off flags
-
-                echo "<td>&nbsp;</td><td><strong>$flag: </strong>";
-
-                if ($type == "choice" ) {
-                    $options = $props["options"];
-                    echo menubox($flag, $options, $value);
-
-                } else {
-
-                    echo textbox($flag, $value, ($type == "integer" ||
-                                                 $type == "integer-hex"
-                                                     ? 7 : 25));
-                }
-                echo "</td>";
-            }
-            echo "</tr>\n";
-
-            if ( $type != "header" ) {
-				echo "<tr><td>&nbsp;</td>";
-				echo "<td>\n";
-				if ( is_file ( "doc/$flag.html" ) ) {
-					include_once "doc/$flag.html";
-				}
-				echo "\n</td></tr>\n";
-            }
-        }
-    }
-    echo "</table>";
-}
-
-/**
- * Return an array of configuration sections used in all compile options
- *
- * $flag_table, the global list of compile options contains a 'cfgsec'
- * property for each flag we are interested in.  We return a list of
- * all the unique cfgsec options we find in $flag_table.
- *
- * @return array an array of strings representing all unique cfgsec values
- *               found in $flag_table
- */
-function get_flag_cfgsecs ()
-{
-    global $flag_table;
-    $cfgsecs = array ();
-
-    foreach ( $flag_table as $key => $props ) {
-        if ( isset ( $props['cfgsec'] ) ) {
-            $cfgsec = $props["cfgsec"];
-            $cfgsecs[$cfgsec] = $cfgsec;
-        }
-    }
-    return $cfgsecs;
-}
-
-////
-// File and directory handling functions
-////
-
-/**
- * Create a copy of a given source directory to a given destination
- *
- * Since we are going to modify the source directory, we create a copy
- * of the directory with a unique name in the given destination directory.
- * We supply a prefix for the tempnam call to prepend to the random filename
- * it generates.
- *
- * @param string $src    source directory
- * @param string $dst    destination directory
- * @param string $prefix string to append to directory created
- *
- * @return string absolute path to destination directory
- */
-function mktempcopy ( $src, $dst, $prefix )
-{
-    if ( $src[0] != "/" ) {
-        $src = dirname ( $_SERVER['SCRIPT_FILENAME'] ) . "/" . $src;
-    }
-
-    // Create a file in the given destination directory with a unique name
-    $dir = tempnam ( $dst, $prefix );
-
-    // Delete the file just created, since it would interfere with the copy we
-    // are about to do.  We only care that the dir name we copy to is unique.
-    unlink ( $dir );
-
-    exec ( "/bin/cp -a '$src' '$dir' 2>&1", $cpytxt, $status );
-
-    if ( $status != 0 ) {
-        die ( "src directory copy failed!" );
-    }
-    return $dir;
-}
-
-/**
- * Write iPXE config files based on value of given flags
- *
- * iPXE compile options are stored in src/config/*.h .
- * We write out a config file for each set of options.
- *
- * @param string $config_dir directory to write .h files to
- * @param array  $flags array of compile options for this build
- *
- * @return void
- */
-function write_ipxe_config_files ( $config_dir, $flags )
-{
-    global $flag_table;
-
-    $cfgsecs = get_flag_cfgsecs ();
-
-    foreach ( $cfgsecs as $cfgsec ) {
-
-        $fname = $config_dir . "/" . $cfgsec . ".h";
-
-        $fp = fopen ( $fname, "wb" );
-        if ( $fp <= 0 ) {
-            die ( "Unable to open $fname file for output!" );
-        }
-
-        $ifdef_secname = "CONFIG_" . strtoupper ( $cfgsec ) . "_H";
-
-        fwrite ( $fp, "#ifndef ${ifdef_secname}\n" );
-        fwrite ( $fp, "#define ${ifdef_secname}\n" );
-        fwrite ( $fp, "#include <config/defaults.h>\n" );
-
-        foreach ( $flags as $key => $value ) {
-            // When the flag matches this section name, write it out
-            if ( $flag_table[$key]["cfgsec"] == $cfgsec ) {
-                fwrite ( $fp, pprint_flag ( $key, $value ) . "\n" );
-            }
-        }
-        fwrite ( $fp, "#endif /* ${ifdef_secname} */\n" );
-        fclose ( $fp );
-    }
-}
-
-/**
- * Output a string to a file
- *
- * Output a given string to a given pathname. The file will be created if
- * necessary, and the string will replace the file's contents in all cases.
- *
- * @param string $fname pathname of file to output string to
- * @param string $ftext text to output to file
- *
- * @return void
- */
-function write_file_from_string ( $fname, $ftext )
-{
-        $fp = fopen ( $fname, "wb" );
-        if ( ! $fp ) {
-            die ( "Unable to open $fname file for output!" );
-        }
-        fwrite ( $fp, $ftext );
-        fclose ( $fp );
-}
-
-/**
- * Delete a file or recursively delete a directory tree
- *
- * @param   string   $file_or_dir_name  name of file or directory to delete
- * @return  bool     Returns TRUE on success, FALSE on failure
- */
-function rm_file_or_dir ( $file_or_dir_name )
-{
-    if ( ! file_exists ( $file_or_dir_name ) ) {
-        return false;
-    }
-
-    if ( is_file ( $file_or_dir_name ) || is_link ( $file_or_dir_name ) ) {
-        return unlink ( $file_or_dir_name );
-    }
-
-    $dir = dir ( $file_or_dir_name );
-    while ( ( $dir_entry = $dir->read () ) !== false ) {
-
-        if ( $dir_entry == '.' || $dir_entry == '..') {
-            continue;
-        }
-        rm_file_or_dir ( $file_or_dir_name . '/' . $dir_entry );
-    }
-    $dir->close();
-
-    return rmdir ( $file_or_dir_name );
-}
-
-////
-// Debugging functions
-////
-
-/**
- * Emit html code to display given array of compile options (flags)
- *
- * @param array  $flags array of compile options for this build
- *
- * @return void
- */
-function show_flags ( $flags )
-{
-    echo ( "\$flags contains " . count ( $flags ) . " elements:" . "<br>" );
-
-	foreach ( $flags as $key => $flag ) {
-        echo ( "\$flags[" . $key . "]=" . "\"$flag\"" . "<br>" );
-    }
-}
-
-/**
- * Emit HTML code to display default array of compile options (flags)
- *
- * $flag_table contains default compile options and properties.  This
- * routine outputs HTML code to display all properties of $flag_table.
- *
- * @return void
- */
-function dump_flag_table ()
-{
-    global $flag_table;
-
-    echo ( "\$flag_table contains " . count ( $flag_table ) . " elements:" . "<br>" );
-
-	foreach ( $flag_table as $key => $props ) {
-        print ( "flag_table[" . $key . "] = " . "<br>" );
-
-		foreach ( $props as $key2 => $props2 ) {
-            print ( "&nbsp;&nbsp;&nbsp;" . $key2 . " = " . $props2 . "<br>" );
-        }
-    }
-}
-
-// Parse src/bin/NIC file
-list ( $nics, $roms ) = parse_nic_file ();
-
-// For emacs:
-// Local variables:
-//  c-basic-offset: 4
-//  c-indent-level: 4
-//  tab-width: 4
-// End:
-
-?>

From be2784649dcc5f57108eb2ab86c8d80f3f96b664 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 20 Aug 2024 22:44:15 +0100
Subject: [PATCH 175/237] [gve] Add missing error codes in EUNIQ() list of
 potential errors

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/gve.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/drivers/net/gve.c b/src/drivers/net/gve.c
index 7f86dd219..0193e769f 100644
--- a/src/drivers/net/gve.c
+++ b/src/drivers/net/gve.c
@@ -117,10 +117,11 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	EUNIQ ( EINFO_EIO, ( (status) & 0x1f ),				\
 		EIO_ADMIN_UNSET, EIO_ADMIN_ABORTED, EIO_ADMIN_EXISTS,	\
 		EIO_ADMIN_CANCELLED, EIO_ADMIN_DATALOSS,		\
-		EIO_ADMIN_DEADLINE, EIO_ADMIN_NOT_FOUND,		\
-		EIO_ADMIN_RANGE, EIO_ADMIN_PERM, EIO_ADMIN_UNAUTH,	\
-		EIO_ADMIN_RESOURCE, EIO_ADMIN_UNAVAIL,			\
-		EIO_ADMIN_NOTSUP, EIO_ADMIN_UNKNOWN )
+		EIO_ADMIN_DEADLINE, EIO_ADMIN_PRECONDITION,		\
+		EIO_ADMIN_INTERNAL, EIO_ADMIN_INVAL,			\
+		EIO_ADMIN_NOT_FOUND, EIO_ADMIN_RANGE, EIO_ADMIN_PERM,	\
+		EIO_ADMIN_UNAUTH, EIO_ADMIN_RESOURCE,			\
+		EIO_ADMIN_UNAVAIL, EIO_ADMIN_NOTSUP, EIO_ADMIN_UNKNOWN )
 
 /******************************************************************************
  *

From c9cac76a5c07536a466bdcbb15c69e090f0bb9f7 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 21 Aug 2024 11:45:36 +0100
Subject: [PATCH 176/237] [tls] Group transmit and receive state in TLS
 connection structure

The TLS connection structure has grown to become unmanageably large as
new features and support for new TLS protocol versions have been added
over time.

Split out the portions of struct tls_connection that are specific to
transmit and receive operations into separate structures, and simplify
some structure field names.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/tls.h |  69 +++++++++++-------
 src/net/tls.c          | 158 ++++++++++++++++++++---------------------
 2 files changed, 119 insertions(+), 108 deletions(-)

diff --git a/src/include/ipxe/tls.h b/src/include/ipxe/tls.h
index cf3277820..b4e41ccc2 100644
--- a/src/include/ipxe/tls.h
+++ b/src/include/ipxe/tls.h
@@ -250,6 +250,14 @@ struct tls_cipherspec {
 	void *fixed_iv;
 };
 
+/** A TLS cipher specification pair */
+struct tls_cipherspec_pair {
+	/** Current cipher specification */
+	struct tls_cipherspec active;
+	/** Next cipher specification */
+	struct tls_cipherspec pending;
+};
+
 /** A TLS signature and hash algorithm identifier */
 struct tls_signature_hash_id {
 	/** Hash algorithm */
@@ -340,6 +348,36 @@ struct tls_session {
 	struct list_head conn;
 };
 
+/** TLS transmit state */
+struct tls_tx {
+	/** Cipher specifications */
+	struct tls_cipherspec_pair cipherspec;
+	/** Sequence number */
+	uint64_t seq;
+	/** Pending transmissions */
+	unsigned int pending;
+	/** Transmit process */
+	struct process process;
+};
+
+/** TLS receive state */
+struct tls_rx {
+	/** Cipher specifications */
+	struct tls_cipherspec_pair cipherspec;
+	/** Sequence number */
+	uint64_t seq;
+	/** State machine current state */
+	enum tls_rx_state state;
+	/** Current received record header */
+	struct tls_header header;
+	/** Current received record header (static I/O buffer) */
+	struct io_buffer iobuf;
+	/** List of received data buffers */
+	struct list_head data;
+	/** Received handshake fragment */
+	struct io_buffer *handshake;
+};
+
 /** A TLS connection */
 struct tls_connection {
 	/** Reference counter */
@@ -365,14 +403,6 @@ struct tls_connection {
 
 	/** Protocol version */
 	uint16_t version;
-	/** Current TX cipher specification */
-	struct tls_cipherspec tx_cipherspec;
-	/** Next TX cipher specification */
-	struct tls_cipherspec tx_cipherspec_pending;
-	/** Current RX cipher specification */
-	struct tls_cipherspec rx_cipherspec;
-	/** Next RX cipher specification */
-	struct tls_cipherspec rx_cipherspec_pending;
 	/** Master secret */
 	uint8_t master_secret[48];
 	/** Server random bytes */
@@ -410,25 +440,10 @@ struct tls_connection {
 	/** Certificate validation pending operation */
 	struct pending_operation validation;
 
-	/** TX sequence number */
-	uint64_t tx_seq;
-	/** TX pending transmissions */
-	unsigned int tx_pending;
-	/** TX process */
-	struct process process;
-
-	/** RX sequence number */
-	uint64_t rx_seq;
-	/** RX state */
-	enum tls_rx_state rx_state;
-	/** Current received record header */
-	struct tls_header rx_header;
-	/** Current received record header (static I/O buffer) */
-	struct io_buffer rx_header_iobuf;
-	/** List of received data buffers */
-	struct list_head rx_data;
-	/** Received handshake fragment */
-	struct io_buffer *rx_handshake;
+	/** Transmit state */
+	struct tls_tx tx;
+	/** Receive state */
+	struct tls_rx rx;
 };
 
 /** RX I/O buffer size
diff --git a/src/net/tls.c b/src/net/tls.c
index a22626f41..ec985e332 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -382,17 +382,17 @@ static void free_tls ( struct refcnt *refcnt ) {
 
 	/* Free dynamically-allocated resources */
 	free ( tls->new_session_ticket );
-	tls_clear_cipher ( tls, &tls->tx_cipherspec );
-	tls_clear_cipher ( tls, &tls->tx_cipherspec_pending );
-	tls_clear_cipher ( tls, &tls->rx_cipherspec );
-	tls_clear_cipher ( tls, &tls->rx_cipherspec_pending );
+	tls_clear_cipher ( tls, &tls->tx.cipherspec.active );
+	tls_clear_cipher ( tls, &tls->tx.cipherspec.pending );
+	tls_clear_cipher ( tls, &tls->rx.cipherspec.active );
+	tls_clear_cipher ( tls, &tls->rx.cipherspec.pending );
 	free ( tls->server_key );
 	free ( tls->handshake_ctx );
-	list_for_each_entry_safe ( iobuf, tmp, &tls->rx_data, list ) {
+	list_for_each_entry_safe ( iobuf, tmp, &tls->rx.data, list ) {
 		list_del ( &iobuf->list );
 		free_iob ( iobuf );
 	}
-	free_iob ( tls->rx_handshake );
+	free_iob ( tls->rx.handshake );
 	x509_chain_put ( tls->certs );
 	x509_chain_put ( tls->chain );
 	x509_root_put ( tls->root );
@@ -420,7 +420,7 @@ static void tls_close ( struct tls_connection *tls, int rc ) {
 	pending_put ( &tls->validation );
 
 	/* Remove process */
-	process_del ( &tls->process );
+	process_del ( &tls->tx.process );
 
 	/* Close all interfaces */
 	intf_shutdown ( &tls->cipherstream, rc );
@@ -662,8 +662,8 @@ static void tls_generate_master_secret ( struct tls_connection *tls,
  * The master secret must already be known.
  */
 static int tls_generate_keys ( struct tls_connection *tls ) {
-	struct tls_cipherspec *tx_cipherspec = &tls->tx_cipherspec_pending;
-	struct tls_cipherspec *rx_cipherspec = &tls->rx_cipherspec_pending;
+	struct tls_cipherspec *tx_cipherspec = &tls->tx.cipherspec.pending;
+	struct tls_cipherspec *rx_cipherspec = &tls->rx.cipherspec.pending;
 	size_t hash_size = tx_cipherspec->suite->mac_len;
 	size_t key_size = tx_cipherspec->suite->key_len;
 	size_t iv_size = tx_cipherspec->suite->fixed_iv_len;
@@ -936,10 +936,10 @@ static int tls_select_cipher ( struct tls_connection *tls,
 		return rc;
 
 	/* Set ciphers */
-	if ( ( rc = tls_set_cipher ( tls, &tls->tx_cipherspec_pending,
+	if ( ( rc = tls_set_cipher ( tls, &tls->tx.cipherspec.pending,
 				     suite ) ) != 0 )
 		return rc;
-	if ( ( rc = tls_set_cipher ( tls, &tls->rx_cipherspec_pending,
+	if ( ( rc = tls_set_cipher ( tls, &tls->rx.cipherspec.pending,
 				     suite ) ) != 0 )
 		return rc;
 
@@ -955,22 +955,20 @@ static int tls_select_cipher ( struct tls_connection *tls,
  * Activate next cipher suite
  *
  * @v tls		TLS connection
- * @v pending		Pending cipher specification
- * @v active		Active cipher specification to replace
+ * @v pair		Cipher specification pair
  * @ret rc		Return status code
  */
 static int tls_change_cipher ( struct tls_connection *tls,
-			       struct tls_cipherspec *pending,
-			       struct tls_cipherspec *active ) {
+			       struct tls_cipherspec_pair *pair ) {
 
 	/* Sanity check */
-	if ( pending->suite == &tls_cipher_suite_null ) {
+	if ( pair->pending.suite == &tls_cipher_suite_null ) {
 		DBGC ( tls, "TLS %p refusing to use null cipher\n", tls );
 		return -ENOTSUP_NULL;
 	}
 
-	tls_clear_cipher ( tls, active );
-	memswap ( active, pending, sizeof ( *active ) );
+	tls_clear_cipher ( tls, &pair->active );
+	memswap ( &pair->active, &pair->pending, sizeof ( pair->active ) );
 	return 0;
 }
 
@@ -1088,7 +1086,7 @@ tls_find_named_curve ( unsigned int named_curve ) {
  * @v tls		TLS connection
  */
 static void tls_tx_resume ( struct tls_connection *tls ) {
-	process_add ( &tls->process );
+	process_add ( &tls->tx.process );
 }
 
 /**
@@ -1111,13 +1109,13 @@ static void tls_tx_resume_all ( struct tls_session *session ) {
 static void tls_restart ( struct tls_connection *tls ) {
 
 	/* Sanity check */
-	assert ( ! tls->tx_pending );
+	assert ( ! tls->tx.pending );
 	assert ( ! is_pending ( &tls->client_negotiation ) );
 	assert ( ! is_pending ( &tls->server_negotiation ) );
 	assert ( ! is_pending ( &tls->validation ) );
 
 	/* (Re)start negotiation */
-	tls->tx_pending = TLS_TX_CLIENT_HELLO;
+	tls->tx.pending = TLS_TX_CLIENT_HELLO;
 	tls_tx_resume ( tls );
 	pending_get ( &tls->client_negotiation );
 	pending_get ( &tls->server_negotiation );
@@ -1392,7 +1390,7 @@ static int tls_send_certificate ( struct tls_connection *tls ) {
  * @ret rc		Return status code
  */
 static int tls_send_client_key_exchange_pubkey ( struct tls_connection *tls ) {
-	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec_pending;
+	struct tls_cipherspec *cipherspec = &tls->tx.cipherspec.pending;
 	struct pubkey_algorithm *pubkey = cipherspec->suite->pubkey;
 	size_t max_len = pubkey_max_len ( pubkey, cipherspec->pubkey_ctx );
 	struct {
@@ -1458,7 +1456,7 @@ struct tls_key_exchange_algorithm tls_pubkey_exchange_algorithm = {
  */
 static int tls_verify_dh_params ( struct tls_connection *tls,
 				  size_t param_len ) {
-	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec_pending;
+	struct tls_cipherspec *cipherspec = &tls->tx.cipherspec.pending;
 	struct pubkey_algorithm *pubkey;
 	struct digest_algorithm *digest;
 	int use_sig_hash = tls_version ( tls, TLS_VERSION_TLS_1_2 );
@@ -1783,7 +1781,7 @@ struct tls_key_exchange_algorithm tls_ecdhe_exchange_algorithm = {
  * @ret rc		Return status code
  */
 static int tls_send_client_key_exchange ( struct tls_connection *tls ) {
-	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec_pending;
+	struct tls_cipherspec *cipherspec = &tls->tx.cipherspec.pending;
 	struct tls_cipher_suite *suite = cipherspec->suite;
 	int rc;
 
@@ -1976,13 +1974,12 @@ static int tls_new_change_cipher ( struct tls_connection *tls,
 	iob_pull ( iobuf, sizeof ( *change_cipher ) );
 
 	/* Change receive cipher spec */
-	if ( ( rc = tls_change_cipher ( tls, &tls->rx_cipherspec_pending,
-					&tls->rx_cipherspec ) ) != 0 ) {
+	if ( ( rc = tls_change_cipher ( tls, &tls->rx.cipherspec ) ) != 0 ) {
 		DBGC ( tls, "TLS %p could not activate RX cipher: %s\n",
 		       tls, strerror ( rc ) );
 		return rc;
 	}
-	tls->rx_seq = ~( ( uint64_t ) 0 );
+	tls->rx.seq = ~( ( uint64_t ) 0 );
 
 	return 0;
 }
@@ -2587,7 +2584,7 @@ static int tls_new_finished ( struct tls_connection *tls,
 	 * transmission of Change Cipher and Finished.
 	 */
 	if ( is_pending ( &tls->client_negotiation ) ) {
-		tls->tx_pending |= ( TLS_TX_CHANGE_CIPHER | TLS_TX_FINISHED );
+		tls->tx.pending |= ( TLS_TX_CHANGE_CIPHER | TLS_TX_FINISHED );
 		tls_tx_resume ( tls );
 	}
 
@@ -2788,7 +2785,7 @@ static int tls_new_record ( struct tls_connection *tls, unsigned int type,
 		break;
 	case TLS_TYPE_HANDSHAKE:
 		handler = tls_new_handshake;
-		iobuf = &tls->rx_handshake;
+		iobuf = &tls->rx.handshake;
 		break;
 	default:
 		DBGC ( tls, "TLS %p unknown record type %d\n", tls, type );
@@ -2935,7 +2932,7 @@ static void tls_hmac_list ( struct tls_cipherspec *cipherspec,
  */
 static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 				const void *data, size_t len ) {
-	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec;
+	struct tls_cipherspec *cipherspec = &tls->tx.cipherspec.active;
 	struct tls_cipher_suite *suite = cipherspec->suite;
 	struct cipher_algorithm *cipher = suite->cipher;
 	struct digest_algorithm *digest = suite->digest;
@@ -2962,7 +2959,7 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 	}
 
 	/* Construct authentication data */
-	authhdr.seq = cpu_to_be64 ( tls->tx_seq );
+	authhdr.seq = cpu_to_be64 ( tls->tx.seq );
 	authhdr.header.type = type;
 	authhdr.header.version = htons ( tls->version );
 	authhdr.header.length = htons ( len );
@@ -3046,7 +3043,7 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 	}
 
 	/* Update TX state machine to next record */
-	tls->tx_seq += 1;
+	tls->tx.seq += 1;
 
 	assert ( plaintext == NULL );
 	assert ( ciphertext == NULL );
@@ -3107,7 +3104,7 @@ static int tls_verify_padding ( struct tls_connection *tls,
 static int tls_new_ciphertext ( struct tls_connection *tls,
 				struct tls_header *tlshdr,
 				struct list_head *rx_data ) {
-	struct tls_cipherspec *cipherspec = &tls->rx_cipherspec;
+	struct tls_cipherspec *cipherspec = &tls->rx.cipherspec.active;
 	struct tls_cipher_suite *suite = cipherspec->suite;
 	struct cipher_algorithm *cipher = suite->cipher;
 	struct digest_algorithm *digest = suite->digest;
@@ -3156,7 +3153,7 @@ static int tls_new_ciphertext ( struct tls_connection *tls,
 	auth = last->tail;
 
 	/* Construct authentication data */
-	authhdr.seq = cpu_to_be64 ( tls->rx_seq );
+	authhdr.seq = cpu_to_be64 ( tls->rx.seq );
 	authhdr.header.type = tlshdr->type;
 	authhdr.header.version = tlshdr->version;
 	authhdr.header.length = htons ( len );
@@ -3172,7 +3169,7 @@ static int tls_new_ciphertext ( struct tls_connection *tls,
 
 	/* Decrypt the received data */
 	check_len = 0;
-	list_for_each_entry ( iobuf, &tls->rx_data, list ) {
+	list_for_each_entry ( iobuf, &tls->rx.data, list ) {
 		cipher_decrypt ( cipher, cipherspec->cipher_ctx,
 				 iobuf->data, iobuf->data, iob_len ( iobuf ) );
 		check_len += iob_len ( iobuf );
@@ -3334,10 +3331,10 @@ static struct interface_descriptor tls_plainstream_desc =
  * @ret rc		Returned status code
  */
 static int tls_newdata_process_header ( struct tls_connection *tls ) {
-	struct tls_cipherspec *cipherspec = &tls->rx_cipherspec;
+	struct tls_cipherspec *cipherspec = &tls->rx.cipherspec.active;
 	struct cipher_algorithm *cipher = cipherspec->suite->cipher;
 	size_t iv_len = cipherspec->suite->record_iv_len;
-	size_t data_len = ntohs ( tls->rx_header.length );
+	size_t data_len = ntohs ( tls->rx.header.length );
 	size_t remaining = data_len;
 	size_t frag_len;
 	size_t reserve;
@@ -3353,7 +3350,7 @@ static int tls_newdata_process_header ( struct tls_connection *tls ) {
 	remaining += reserve;
 
 	/* Allocate data buffers now that we know the length */
-	assert ( list_empty ( &tls->rx_data ) );
+	assert ( list_empty ( &tls->rx.data ) );
 	while ( remaining ) {
 
 		/* Calculate fragment length.  Ensure that no block is
@@ -3394,16 +3391,16 @@ static int tls_newdata_process_header ( struct tls_connection *tls ) {
 		reserve = 0;
 
 		/* Add I/O buffer to list */
-		list_add_tail ( &iobuf->list, &tls->rx_data );
+		list_add_tail ( &iobuf->list, &tls->rx.data );
 	}
 
 	/* Move to data state */
-	tls->rx_state = TLS_RX_DATA;
+	tls->rx.state = TLS_RX_DATA;
 
 	return 0;
 
  err:
-	list_for_each_entry_safe ( iobuf, tmp, &tls->rx_data, list ) {
+	list_for_each_entry_safe ( iobuf, tmp, &tls->rx.data, list ) {
 		list_del ( &iobuf->list );
 		free_iob ( iobuf );
 	}
@@ -3421,27 +3418,27 @@ static int tls_newdata_process_data ( struct tls_connection *tls ) {
 	int rc;
 
 	/* Move current buffer to end of list */
-	iobuf = list_first_entry ( &tls->rx_data, struct io_buffer, list );
+	iobuf = list_first_entry ( &tls->rx.data, struct io_buffer, list );
 	list_del ( &iobuf->list );
-	list_add_tail ( &iobuf->list, &tls->rx_data );
+	list_add_tail ( &iobuf->list, &tls->rx.data );
 
 	/* Continue receiving data if any space remains */
-	iobuf = list_first_entry ( &tls->rx_data, struct io_buffer, list );
+	iobuf = list_first_entry ( &tls->rx.data, struct io_buffer, list );
 	if ( iob_tailroom ( iobuf ) )
 		return 0;
 
 	/* Process record */
-	if ( ( rc = tls_new_ciphertext ( tls, &tls->rx_header,
-					 &tls->rx_data ) ) != 0 )
+	if ( ( rc = tls_new_ciphertext ( tls, &tls->rx.header,
+					 &tls->rx.data ) ) != 0 )
 		return rc;
 
 	/* Increment RX sequence number */
-	tls->rx_seq += 1;
+	tls->rx.seq += 1;
 
 	/* Return to header state */
-	assert ( list_empty ( &tls->rx_data ) );
-	tls->rx_state = TLS_RX_HEADER;
-	iob_unput ( &tls->rx_header_iobuf, sizeof ( tls->rx_header ) );
+	assert ( list_empty ( &tls->rx.data ) );
+	tls->rx.state = TLS_RX_HEADER;
+	iob_unput ( &tls->rx.iobuf, sizeof ( tls->rx.header ) );
 
 	return 0;
 }
@@ -3480,13 +3477,13 @@ static int tls_cipherstream_deliver ( struct tls_connection *tls,
 	while ( iob_len ( iobuf ) ) {
 
 		/* Select buffer according to current state */
-		switch ( tls->rx_state ) {
+		switch ( tls->rx.state ) {
 		case TLS_RX_HEADER:
-			dest = &tls->rx_header_iobuf;
+			dest = &tls->rx.iobuf;
 			process = tls_newdata_process_header;
 			break;
 		case TLS_RX_DATA:
-			dest = list_first_entry ( &tls->rx_data,
+			dest = list_first_entry ( &tls->rx.data,
 						  struct io_buffer, list );
 			assert ( dest != NULL );
 			process = tls_newdata_process_data;
@@ -3550,7 +3547,7 @@ static struct interface_descriptor tls_cipherstream_desc =
  */
 static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 	struct tls_session *session = tls->session;
-	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec_pending;
+	struct tls_cipherspec *cipherspec = &tls->tx.cipherspec.pending;
 	struct pubkey_algorithm *pubkey = cipherspec->suite->pubkey;
 	struct x509_certificate *cert;
 
@@ -3588,11 +3585,11 @@ static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 	}
 
 	/* Schedule Client Key Exchange, Change Cipher, and Finished */
-	tls->tx_pending |= ( TLS_TX_CLIENT_KEY_EXCHANGE |
+	tls->tx.pending |= ( TLS_TX_CLIENT_KEY_EXCHANGE |
 			     TLS_TX_CHANGE_CIPHER |
 			     TLS_TX_FINISHED );
 	if ( tls->certs ) {
-		tls->tx_pending |= ( TLS_TX_CERTIFICATE |
+		tls->tx.pending |= ( TLS_TX_CERTIFICATE |
 				     TLS_TX_CERTIFICATE_VERIFY );
 	}
 	tls_tx_resume ( tls );
@@ -3635,7 +3632,7 @@ static void tls_tx_step ( struct tls_connection *tls ) {
 		return;
 
 	/* Send first pending transmission */
-	if ( tls->tx_pending & TLS_TX_CLIENT_HELLO ) {
+	if ( tls->tx.pending & TLS_TX_CLIENT_HELLO ) {
 		/* Serialise server negotiations within a session, to
 		 * provide a consistent view of session IDs and
 		 * session tickets.
@@ -3668,32 +3665,32 @@ static void tls_tx_step ( struct tls_connection *tls ) {
 			       tls, strerror ( rc ) );
 			goto err;
 		}
-		tls->tx_pending &= ~TLS_TX_CLIENT_HELLO;
-	} else if ( tls->tx_pending & TLS_TX_CERTIFICATE ) {
+		tls->tx.pending &= ~TLS_TX_CLIENT_HELLO;
+	} else if ( tls->tx.pending & TLS_TX_CERTIFICATE ) {
 		/* Send Certificate */
 		if ( ( rc = tls_send_certificate ( tls ) ) != 0 ) {
 			DBGC ( tls, "TLS %p cold not send Certificate: %s\n",
 			       tls, strerror ( rc ) );
 			goto err;
 		}
-		tls->tx_pending &= ~TLS_TX_CERTIFICATE;
-	} else if ( tls->tx_pending & TLS_TX_CLIENT_KEY_EXCHANGE ) {
+		tls->tx.pending &= ~TLS_TX_CERTIFICATE;
+	} else if ( tls->tx.pending & TLS_TX_CLIENT_KEY_EXCHANGE ) {
 		/* Send Client Key Exchange */
 		if ( ( rc = tls_send_client_key_exchange ( tls ) ) != 0 ) {
 			DBGC ( tls, "TLS %p could not send Client Key "
 			       "Exchange: %s\n", tls, strerror ( rc ) );
 			goto err;
 		}
-		tls->tx_pending &= ~TLS_TX_CLIENT_KEY_EXCHANGE;
-	} else if ( tls->tx_pending & TLS_TX_CERTIFICATE_VERIFY ) {
+		tls->tx.pending &= ~TLS_TX_CLIENT_KEY_EXCHANGE;
+	} else if ( tls->tx.pending & TLS_TX_CERTIFICATE_VERIFY ) {
 		/* Send Certificate Verify */
 		if ( ( rc = tls_send_certificate_verify ( tls ) ) != 0 ) {
 			DBGC ( tls, "TLS %p could not send Certificate "
 			       "Verify: %s\n", tls, strerror ( rc ) );
 			goto err;
 		}
-		tls->tx_pending &= ~TLS_TX_CERTIFICATE_VERIFY;
-	} else if ( tls->tx_pending & TLS_TX_CHANGE_CIPHER ) {
+		tls->tx.pending &= ~TLS_TX_CERTIFICATE_VERIFY;
+	} else if ( tls->tx.pending & TLS_TX_CHANGE_CIPHER ) {
 		/* Send Change Cipher, and then change the cipher in use */
 		if ( ( rc = tls_send_change_cipher ( tls ) ) != 0 ) {
 			DBGC ( tls, "TLS %p could not send Change Cipher: "
@@ -3701,28 +3698,27 @@ static void tls_tx_step ( struct tls_connection *tls ) {
 			goto err;
 		}
 		if ( ( rc = tls_change_cipher ( tls,
-						&tls->tx_cipherspec_pending,
-						&tls->tx_cipherspec )) != 0 ){
+						&tls->tx.cipherspec ) ) != 0 ){
 			DBGC ( tls, "TLS %p could not activate TX cipher: "
 			       "%s\n", tls, strerror ( rc ) );
 			goto err;
 		}
-		tls->tx_seq = 0;
-		tls->tx_pending &= ~TLS_TX_CHANGE_CIPHER;
-	} else if ( tls->tx_pending & TLS_TX_FINISHED ) {
+		tls->tx.seq = 0;
+		tls->tx.pending &= ~TLS_TX_CHANGE_CIPHER;
+	} else if ( tls->tx.pending & TLS_TX_FINISHED ) {
 		/* Send Finished */
 		if ( ( rc = tls_send_finished ( tls ) ) != 0 ) {
 			DBGC ( tls, "TLS %p could not send Finished: %s\n",
 			       tls, strerror ( rc ) );
 			goto err;
 		}
-		tls->tx_pending &= ~TLS_TX_FINISHED;
+		tls->tx.pending &= ~TLS_TX_FINISHED;
 	}
 
 	/* Reschedule process if pending transmissions remain,
 	 * otherwise send notification of a window change.
 	 */
-	if ( tls->tx_pending ) {
+	if ( tls->tx.pending ) {
 		tls_tx_resume ( tls );
 	} else {
 		xfer_window_changed ( &tls->plainstream );
@@ -3736,7 +3732,7 @@ static void tls_tx_step ( struct tls_connection *tls ) {
 
 /** TLS TX process descriptor */
 static struct process_descriptor tls_process_desc =
-	PROC_DESC_ONCE ( struct tls_connection, process, tls_tx_step );
+	PROC_DESC_ONCE ( struct tls_connection, tx.process, tls_tx_step );
 
 /******************************************************************************
  *
@@ -3829,20 +3825,20 @@ int add_tls ( struct interface *xfer, const char *name,
 	intf_init ( &tls->plainstream, &tls_plainstream_desc, &tls->refcnt );
 	intf_init ( &tls->cipherstream, &tls_cipherstream_desc, &tls->refcnt );
 	intf_init ( &tls->validator, &tls_validator_desc, &tls->refcnt );
-	process_init_stopped ( &tls->process, &tls_process_desc,
+	process_init_stopped ( &tls->tx.process, &tls_process_desc,
 			       &tls->refcnt );
 	tls->key = privkey_get ( key ? key : &private_key );
 	tls->root = x509_root_get ( root ? root : &root_certificates );
 	tls->version = TLS_VERSION_MAX;
-	tls_clear_cipher ( tls, &tls->tx_cipherspec );
-	tls_clear_cipher ( tls, &tls->tx_cipherspec_pending );
-	tls_clear_cipher ( tls, &tls->rx_cipherspec );
-	tls_clear_cipher ( tls, &tls->rx_cipherspec_pending );
+	tls_clear_cipher ( tls, &tls->tx.cipherspec.active );
+	tls_clear_cipher ( tls, &tls->tx.cipherspec.pending );
+	tls_clear_cipher ( tls, &tls->rx.cipherspec.active );
+	tls_clear_cipher ( tls, &tls->rx.cipherspec.pending );
 	tls_clear_handshake ( tls );
 	tls->client_random.gmt_unix_time = time ( NULL );
-	iob_populate ( &tls->rx_header_iobuf, &tls->rx_header, 0,
-		       sizeof ( tls->rx_header ) );
-	INIT_LIST_HEAD ( &tls->rx_data );
+	iob_populate ( &tls->rx.iobuf, &tls->rx.header, 0,
+		       sizeof ( tls->rx.header ) );
+	INIT_LIST_HEAD ( &tls->rx.data );
 	if ( ( rc = tls_generate_random ( tls, &tls->client_random.random,
 			  ( sizeof ( tls->client_random.random ) ) ) ) != 0 ) {
 		goto err_random;

From acbabdb335f47eb8246188a23ed7e3997da6e8ba Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 21 Aug 2024 12:15:24 +0100
Subject: [PATCH 177/237] [tls] Group client and server state in TLS connection
 structure

The TLS connection structure has grown to become unmanageably large as
new features and support for new TLS protocol versions have been added
over time.

Split out the portions of struct tls_connection that are specific to
client and server operations into separate structures, and simplify
some structure field names.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/tls.h |  62 +++++++-----
 src/net/tls.c          | 213 +++++++++++++++++++++--------------------
 2 files changed, 147 insertions(+), 128 deletions(-)

diff --git a/src/include/ipxe/tls.h b/src/include/ipxe/tls.h
index b4e41ccc2..9494eaa05 100644
--- a/src/include/ipxe/tls.h
+++ b/src/include/ipxe/tls.h
@@ -378,6 +378,38 @@ struct tls_rx {
 	struct io_buffer *handshake;
 };
 
+/** TLS client state */
+struct tls_client {
+	/** Random bytes */
+	struct tls_client_random random;
+	/** Private key (if used) */
+	struct private_key *key;
+	/** Certificate chain (if used) */
+	struct x509_chain *chain;
+	/** Security negotiation pending operation */
+	struct pending_operation negotiation;
+};
+
+/** TLS server state */
+struct tls_server {
+	/** Random bytes */
+	uint8_t random[32];
+	/** Server Key Exchange record (if any) */
+	void *exchange;
+	/** Server Key Exchange record length */
+	size_t exchange_len;
+	/** Root of trust */
+	struct x509_root *root;
+	/** Certificate chain */
+	struct x509_chain *chain;
+	/** Certificate validator */
+	struct interface validator;
+	/** Certificate validation pending operation */
+	struct pending_operation validation;
+	/** Security negotiation pending operation */
+	struct pending_operation negotiation;
+};
+
 /** A TLS connection */
 struct tls_connection {
 	/** Reference counter */
@@ -405,45 +437,23 @@ struct tls_connection {
 	uint16_t version;
 	/** Master secret */
 	uint8_t master_secret[48];
-	/** Server random bytes */
-	uint8_t server_random[32];
-	/** Client random bytes */
-	struct tls_client_random client_random;
-	/** Server Key Exchange record (if any) */
-	void *server_key;
-	/** Server Key Exchange record length */
-	size_t server_key_len;
 	/** Digest algorithm used for handshake verification */
 	struct digest_algorithm *handshake_digest;
 	/** Digest algorithm context used for handshake verification */
 	uint8_t *handshake_ctx;
-	/** Private key */
-	struct private_key *key;
-	/** Client certificate chain (if used) */
-	struct x509_chain *certs;
 	/** Secure renegotiation flag */
 	int secure_renegotiation;
 	/** Verification data */
 	struct tls_verify_data verify;
 
-	/** Root of trust */
-	struct x509_root *root;
-	/** Server certificate chain */
-	struct x509_chain *chain;
-	/** Certificate validator */
-	struct interface validator;
-
-	/** Client security negotiation pending operation */
-	struct pending_operation client_negotiation;
-	/** Server security negotiation pending operation */
-	struct pending_operation server_negotiation;
-	/** Certificate validation pending operation */
-	struct pending_operation validation;
-
 	/** Transmit state */
 	struct tls_tx tx;
 	/** Receive state */
 	struct tls_rx rx;
+	/** Client state */
+	struct tls_client client;
+	/** Server state */
+	struct tls_server server;
 };
 
 /** RX I/O buffer size
diff --git a/src/net/tls.c b/src/net/tls.c
index ec985e332..ec503e43d 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -251,8 +251,8 @@ static void tls_set_uint24 ( tls24_t *field24, unsigned long value ) {
  * @ret is_ready	TLS connection is ready
  */
 static int tls_ready ( struct tls_connection *tls ) {
-	return ( ( ! is_pending ( &tls->client_negotiation ) ) &&
-		 ( ! is_pending ( &tls->server_negotiation ) ) );
+	return ( ( ! is_pending ( &tls->client.negotiation ) ) &&
+		 ( ! is_pending ( &tls->server.negotiation ) ) );
 }
 
 /**
@@ -386,17 +386,17 @@ static void free_tls ( struct refcnt *refcnt ) {
 	tls_clear_cipher ( tls, &tls->tx.cipherspec.pending );
 	tls_clear_cipher ( tls, &tls->rx.cipherspec.active );
 	tls_clear_cipher ( tls, &tls->rx.cipherspec.pending );
-	free ( tls->server_key );
+	free ( tls->server.exchange );
 	free ( tls->handshake_ctx );
 	list_for_each_entry_safe ( iobuf, tmp, &tls->rx.data, list ) {
 		list_del ( &iobuf->list );
 		free_iob ( iobuf );
 	}
 	free_iob ( tls->rx.handshake );
-	x509_chain_put ( tls->certs );
-	x509_chain_put ( tls->chain );
-	x509_root_put ( tls->root );
-	privkey_put ( tls->key );
+	privkey_put ( tls->client.key );
+	x509_chain_put ( tls->client.chain );
+	x509_chain_put ( tls->server.chain );
+	x509_root_put ( tls->server.root );
 
 	/* Drop reference to session */
 	assert ( list_empty ( &tls->list ) );
@@ -415,9 +415,9 @@ static void free_tls ( struct refcnt *refcnt ) {
 static void tls_close ( struct tls_connection *tls, int rc ) {
 
 	/* Remove pending operations, if applicable */
-	pending_put ( &tls->client_negotiation );
-	pending_put ( &tls->server_negotiation );
-	pending_put ( &tls->validation );
+	pending_put ( &tls->client.negotiation );
+	pending_put ( &tls->server.negotiation );
+	pending_put ( &tls->server.validation );
 
 	/* Remove process */
 	process_del ( &tls->tx.process );
@@ -425,7 +425,7 @@ static void tls_close ( struct tls_connection *tls, int rc ) {
 	/* Close all interfaces */
 	intf_shutdown ( &tls->cipherstream, rc );
 	intf_shutdown ( &tls->plainstream, rc );
-	intf_shutdown ( &tls->validator, rc );
+	intf_shutdown ( &tls->server.validator, rc );
 
 	/* Remove from session */
 	list_del ( &tls->list );
@@ -640,15 +640,15 @@ static void tls_generate_master_secret ( struct tls_connection *tls,
 	DBGC ( tls, "TLS %p pre-master-secret:\n", tls );
 	DBGC_HD ( tls, pre_master_secret, pre_master_secret_len );
 	DBGC ( tls, "TLS %p client random bytes:\n", tls );
-	DBGC_HD ( tls, &tls->client_random, sizeof ( tls->client_random ) );
+	DBGC_HD ( tls, &tls->client.random, sizeof ( tls->client.random ) );
 	DBGC ( tls, "TLS %p server random bytes:\n", tls );
-	DBGC_HD ( tls, &tls->server_random, sizeof ( tls->server_random ) );
+	DBGC_HD ( tls, &tls->server.random, sizeof ( tls->server.random ) );
 
 	tls_prf_label ( tls, pre_master_secret, pre_master_secret_len,
 			&tls->master_secret, sizeof ( tls->master_secret ),
 			"master secret",
-			&tls->client_random, sizeof ( tls->client_random ),
-			&tls->server_random, sizeof ( tls->server_random ) );
+			&tls->client.random, sizeof ( tls->client.random ),
+			&tls->server.random, sizeof ( tls->server.random ) );
 
 	DBGC ( tls, "TLS %p generated master secret:\n", tls );
 	DBGC_HD ( tls, &tls->master_secret, sizeof ( tls->master_secret ) );
@@ -675,8 +675,8 @@ static int tls_generate_keys ( struct tls_connection *tls ) {
 	/* Generate key block */
 	tls_prf_label ( tls, &tls->master_secret, sizeof ( tls->master_secret ),
 			key_block, sizeof ( key_block ), "key expansion",
-			&tls->server_random, sizeof ( tls->server_random ),
-			&tls->client_random, sizeof ( tls->client_random ) );
+			&tls->server.random, sizeof ( tls->server.random ),
+			&tls->client.random, sizeof ( tls->client.random ) );
 
 	/* Split key block into portions */
 	key = key_block;
@@ -1110,15 +1110,15 @@ static void tls_restart ( struct tls_connection *tls ) {
 
 	/* Sanity check */
 	assert ( ! tls->tx.pending );
-	assert ( ! is_pending ( &tls->client_negotiation ) );
-	assert ( ! is_pending ( &tls->server_negotiation ) );
-	assert ( ! is_pending ( &tls->validation ) );
+	assert ( ! is_pending ( &tls->client.negotiation ) );
+	assert ( ! is_pending ( &tls->server.negotiation ) );
+	assert ( ! is_pending ( &tls->server.validation ) );
 
 	/* (Re)start negotiation */
 	tls->tx.pending = TLS_TX_CLIENT_HELLO;
 	tls_tx_resume ( tls );
-	pending_get ( &tls->client_negotiation );
-	pending_get ( &tls->server_negotiation );
+	pending_get ( &tls->client.negotiation );
+	pending_get ( &tls->server.negotiation );
 }
 
 /**
@@ -1237,7 +1237,7 @@ static int tls_client_hello ( struct tls_connection *tls,
 			      htonl ( sizeof ( hello ) -
 				      sizeof ( hello.type_length ) ) );
 	hello.version = htons ( TLS_VERSION_MAX );
-	memcpy ( &hello.random, &tls->client_random, sizeof ( hello.random ) );
+	memcpy ( &hello.random, &tls->client.random, sizeof ( hello.random ) );
 	hello.session_id_len = tls->session_id_len;
 	memcpy ( hello.session_id, tls->session_id,
 		 sizeof ( hello.session_id ) );
@@ -1344,7 +1344,7 @@ static int tls_send_certificate ( struct tls_connection *tls ) {
 
 	/* Calculate length of client certificates */
 	len = 0;
-	list_for_each_entry ( link, &tls->certs->links, list ) {
+	list_for_each_entry ( link, &tls->client.chain->links, list ) {
 		cert = link->cert;
 		len += ( sizeof ( *certificate ) + cert->raw.len );
 		DBGC ( tls, "TLS %p sending client certificate %s\n",
@@ -1365,7 +1365,7 @@ static int tls_send_certificate ( struct tls_connection *tls ) {
 			  sizeof ( certificates->type_length ) ) );
 	tls_set_uint24 ( &certificates->length, len );
 	certificate = &certificates->certificates[0];
-	list_for_each_entry ( link, &tls->certs->links, list ) {
+	list_for_each_entry ( link, &tls->client.chain->links, list ) {
 		cert = link->cert;
 		tls_set_uint24 ( &certificate->length, cert->raw.len );
 		memcpy ( certificate->data, cert->raw.data, cert->raw.len );
@@ -1470,9 +1470,9 @@ static int tls_verify_dh_params ( struct tls_connection *tls,
 	int rc;
 
 	/* Signature follows parameters */
-	assert ( param_len <= tls->server_key_len );
-	data = ( tls->server_key + param_len );
-	remaining = ( tls->server_key_len - param_len );
+	assert ( param_len <= tls->server.exchange_len );
+	data = ( tls->server.exchange + param_len );
+	remaining = ( tls->server.exchange_len - param_len );
 
 	/* Parse signature from ServerKeyExchange */
 	sig = data;
@@ -1481,7 +1481,8 @@ static int tls_verify_dh_params ( struct tls_connection *tls,
 						sizeof ( *sig ) ) ) ) {
 		DBGC ( tls, "TLS %p received underlength ServerKeyExchange\n",
 		       tls );
-		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		DBGC_HDA ( tls, 0, tls->server.exchange,
+			   tls->server.exchange_len );
 		return -EINVAL_KEY_EXCHANGE;
 	}
 
@@ -1514,11 +1515,11 @@ static int tls_verify_dh_params ( struct tls_connection *tls,
 
 		/* Calculate digest */
 		digest_init ( digest, ctx );
-		digest_update ( digest, ctx, &tls->client_random,
-				sizeof ( tls->client_random ) );
-		digest_update ( digest, ctx, tls->server_random,
-				sizeof ( tls->server_random ) );
-		digest_update ( digest, ctx, tls->server_key, param_len );
+		digest_update ( digest, ctx, &tls->client.random,
+				sizeof ( tls->client.random ) );
+		digest_update ( digest, ctx, tls->server.random,
+				sizeof ( tls->server.random ) );
+		digest_update ( digest, ctx, tls->server.exchange, param_len );
 		digest_final ( digest, ctx, hash );
 
 		/* Verify signature */
@@ -1527,8 +1528,8 @@ static int tls_verify_dh_params ( struct tls_connection *tls,
 					    signature_len ) ) != 0 ) {
 			DBGC ( tls, "TLS %p ServerKeyExchange failed "
 			       "verification\n", tls );
-			DBGC_HDA ( tls, 0, tls->server_key,
-				   tls->server_key_len );
+			DBGC_HDA ( tls, 0, tls->server.exchange,
+				   tls->server.exchange_len );
 			return -EPERM_KEY_EXCHANGE;
 		}
 	}
@@ -1543,7 +1544,7 @@ static int tls_verify_dh_params ( struct tls_connection *tls,
  * @ret rc		Return status code
  */
 static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
-	uint8_t private[ sizeof ( tls->client_random.random ) ];
+	uint8_t private[ sizeof ( tls->client.random.random ) ];
 	const struct {
 		uint16_t len;
 		uint8_t data[0];
@@ -1556,8 +1557,8 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 	int rc;
 
 	/* Parse ServerKeyExchange */
-	data = tls->server_key;
-	remaining = tls->server_key_len;
+	data = tls->server.exchange;
+	remaining = tls->server.exchange_len;
 	for ( i = 0 ; i < ( sizeof ( dh_val ) / sizeof ( dh_val[0] ) ) ; i++ ){
 		dh_val[i] = data;
 		if ( ( sizeof ( *dh_val[i] ) > remaining ) ||
@@ -1565,8 +1566,8 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 						    sizeof ( *dh_val[i] ) ) )){
 			DBGC ( tls, "TLS %p received underlength "
 			       "ServerKeyExchange\n", tls );
-			DBGC_HDA ( tls, 0, tls->server_key,
-				   tls->server_key_len );
+			DBGC_HDA ( tls, 0, tls->server.exchange,
+				   tls->server.exchange_len );
 			rc = -EINVAL_KEY_EXCHANGE;
 			goto err_header;
 		}
@@ -1574,7 +1575,7 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 		data += frag_len;
 		remaining -= frag_len;
 	}
-	param_len = ( tls->server_key_len - remaining );
+	param_len = ( tls->server.exchange_len - remaining );
 
 	/* Verify parameter signature */
 	if ( ( rc = tls_verify_dh_params ( tls, param_len ) ) != 0 )
@@ -1679,12 +1680,14 @@ static int tls_send_client_key_exchange_ecdhe ( struct tls_connection *tls ) {
 	int rc;
 
 	/* Parse ServerKeyExchange record */
-	ecdh = tls->server_key;
-	if ( ( sizeof ( *ecdh ) > tls->server_key_len ) ||
-	     ( ecdh->public_len > ( tls->server_key_len - sizeof ( *ecdh ) ))){
+	ecdh = tls->server.exchange;
+	if ( ( sizeof ( *ecdh ) > tls->server.exchange_len ) ||
+	     ( ecdh->public_len > ( tls->server.exchange_len -
+				    sizeof ( *ecdh ) ) ) ) {
 		DBGC ( tls, "TLS %p received underlength ServerKeyExchange\n",
 		       tls );
-		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		DBGC_HDA ( tls, 0, tls->server.exchange,
+			   tls->server.exchange_len );
 		return -EINVAL_KEY_EXCHANGE;
 	}
 	param_len = ( sizeof ( *ecdh ) + ecdh->public_len );
@@ -1697,14 +1700,16 @@ static int tls_send_client_key_exchange_ecdhe ( struct tls_connection *tls ) {
 	if ( ecdh->curve_type != TLS_NAMED_CURVE_TYPE ) {
 		DBGC ( tls, "TLS %p unsupported curve type %d\n",
 		       tls, ecdh->curve_type );
-		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		DBGC_HDA ( tls, 0, tls->server.exchange,
+			   tls->server.exchange_len );
 		return -ENOTSUP_CURVE;
 	}
 	curve = tls_find_named_curve ( ecdh->named_curve );
 	if ( ! curve ) {
 		DBGC ( tls, "TLS %p unsupported named curve %d\n",
 		       tls, ntohs ( ecdh->named_curve ) );
-		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		DBGC_HDA ( tls, 0, tls->server.exchange,
+			   tls->server.exchange_len );
 		return -ENOTSUP_CURVE;
 	}
 
@@ -1712,7 +1717,8 @@ static int tls_send_client_key_exchange_ecdhe ( struct tls_connection *tls ) {
 	if ( ecdh->public_len != curve->curve->keysize ) {
 		DBGC ( tls, "TLS %p invalid %s key\n",
 		       tls, curve->curve->name );
-		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		DBGC_HDA ( tls, 0, tls->server.exchange,
+			   tls->server.exchange_len );
 		return -EINVAL_KEY_EXCHANGE;
 	}
 
@@ -1810,9 +1816,9 @@ static int tls_send_client_key_exchange ( struct tls_connection *tls ) {
  */
 static int tls_send_certificate_verify ( struct tls_connection *tls ) {
 	struct digest_algorithm *digest = tls->handshake_digest;
-	struct x509_certificate *cert = x509_first ( tls->certs );
+	struct x509_certificate *cert = x509_first ( tls->client.chain );
 	struct pubkey_algorithm *pubkey = cert->signature_algorithm->pubkey;
-	struct asn1_cursor *key = privkey_cursor ( tls->key );
+	struct asn1_cursor *key = privkey_cursor ( tls->client.key );
 	uint8_t digest_out[ digest->digestsize ];
 	uint8_t ctx[ pubkey->ctxsize ];
 	struct tls_signature_hash_algorithm *sig_hash = NULL;
@@ -1944,7 +1950,7 @@ static int tls_send_finished ( struct tls_connection *tls ) {
 		return rc;
 
 	/* Mark client as finished */
-	pending_put ( &tls->client_negotiation );
+	pending_put ( &tls->client.negotiation );
 
 	return 0;
 }
@@ -2185,8 +2191,8 @@ static int tls_new_server_hello ( struct tls_connection *tls,
 		return rc;
 
 	/* Copy out server random bytes */
-	memcpy ( &tls->server_random, &hello_a->random,
-		 sizeof ( tls->server_random ) );
+	memcpy ( &tls->server.random, &hello_a->random,
+		 sizeof ( tls->server.random ) );
 
 	/* Check session ID */
 	if ( hello_a->session_id_len &&
@@ -2306,12 +2312,12 @@ static int tls_parse_chain ( struct tls_connection *tls,
 	int rc;
 
 	/* Free any existing certificate chain */
-	x509_chain_put ( tls->chain );
-	tls->chain = NULL;
+	x509_chain_put ( tls->server.chain );
+	tls->server.chain = NULL;
 
 	/* Create certificate chain */
-	tls->chain = x509_alloc_chain();
-	if ( ! tls->chain ) {
+	tls->server.chain = x509_alloc_chain();
+	if ( ! tls->server.chain ) {
 		rc = -ENOMEM_CHAIN;
 		goto err_alloc_chain;
 	}
@@ -2343,14 +2349,15 @@ static int tls_parse_chain ( struct tls_connection *tls,
 		record_len = ( sizeof ( *certificate ) + certificate_len );
 
 		/* Add certificate to chain */
-		if ( ( rc = x509_append_raw ( tls->chain, certificate->data,
+		if ( ( rc = x509_append_raw ( tls->server.chain,
+					      certificate->data,
 					      certificate_len ) ) != 0 ) {
 			DBGC ( tls, "TLS %p could not append certificate: %s\n",
 			       tls, strerror ( rc ) );
 			DBGC_HDA ( tls, 0, data, remaining );
 			goto err_parse;
 		}
-		cert = x509_last ( tls->chain );
+		cert = x509_last ( tls->server.chain );
 		DBGC ( tls, "TLS %p found certificate %s\n",
 		       tls, x509_name ( cert ) );
 
@@ -2364,8 +2371,8 @@ static int tls_parse_chain ( struct tls_connection *tls,
  err_parse:
  err_overlength:
  err_underlength:
-	x509_chain_put ( tls->chain );
-	tls->chain = NULL;
+	x509_chain_put ( tls->server.chain );
+	tls->server.chain = NULL;
  err_alloc_chain:
 	return rc;
 }
@@ -2422,12 +2429,12 @@ static int tls_new_server_key_exchange ( struct tls_connection *tls,
 					 const void *data, size_t len ) {
 
 	/* Free any existing server key exchange record */
-	free ( tls->server_key );
-	tls->server_key_len = 0;
+	free ( tls->server.exchange );
+	tls->server.exchange_len = 0;
 
 	/* Allocate copy of server key exchange record */
-	tls->server_key = malloc ( len );
-	if ( ! tls->server_key )
+	tls->server.exchange = malloc ( len );
+	if ( ! tls->server.exchange )
 		return -ENOMEM;
 
 	/* Store copy of server key exchange record for later
@@ -2435,8 +2442,8 @@ static int tls_new_server_key_exchange ( struct tls_connection *tls,
 	 * since the certificate validation will not yet have
 	 * completed.
 	 */
-	memcpy ( tls->server_key, data, len );
-	tls->server_key_len = len;
+	memcpy ( tls->server.exchange, data, len );
+	tls->server.exchange_len = len;
 
 	return 0;
 }
@@ -2460,11 +2467,11 @@ static int tls_new_certificate_request ( struct tls_connection *tls,
 	 */
 
 	/* Free any existing client certificate chain */
-	x509_chain_put ( tls->certs );
-	tls->certs = NULL;
+	x509_chain_put ( tls->client.chain );
+	tls->client.chain = NULL;
 
 	/* Determine client certificate to be sent */
-	cert = x509_find_key ( NULL, tls->key );
+	cert = x509_find_key ( NULL, tls->client.key );
 	if ( ! cert ) {
 		DBGC ( tls, "TLS %p could not find certificate corresponding "
 		       "to private key\n", tls );
@@ -2476,18 +2483,18 @@ static int tls_new_certificate_request ( struct tls_connection *tls,
 	       tls, x509_name ( cert ) );
 
 	/* Create client certificate chain */
-	tls->certs = x509_alloc_chain();
-	if ( ! tls->certs ) {
+	tls->client.chain = x509_alloc_chain();
+	if ( ! tls->client.chain ) {
 		rc = -ENOMEM;
 		goto err_alloc;
 	}
 
 	/* Append client certificate to chain */
-	if ( ( rc = x509_append ( tls->certs, cert ) ) != 0 )
+	if ( ( rc = x509_append ( tls->client.chain, cert ) ) != 0 )
 		goto err_append;
 
 	/* Append any relevant issuer certificates */
-	if ( ( rc = x509_auto_append ( tls->certs, &certstore ) ) != 0 )
+	if ( ( rc = x509_auto_append ( tls->client.chain, &certstore ) ) != 0 )
 		goto err_auto_append;
 
 	/* Drop local reference to client certificate */
@@ -2497,8 +2504,8 @@ static int tls_new_certificate_request ( struct tls_connection *tls,
 
  err_auto_append:
  err_append:
-	x509_chain_put ( tls->certs );
-	tls->certs = NULL;
+	x509_chain_put ( tls->client.chain );
+	tls->client.chain = NULL;
  err_alloc:
 	x509_put ( cert );
  err_find:
@@ -2529,13 +2536,14 @@ static int tls_new_server_hello_done ( struct tls_connection *tls,
 	}
 
 	/* Begin certificate validation */
-	if ( ( rc = create_validator ( &tls->validator, tls->chain,
-				       tls->root ) ) != 0 ) {
+	if ( ( rc = create_validator ( &tls->server.validator,
+				       tls->server.chain,
+				       tls->server.root ) ) != 0 ) {
 		DBGC ( tls, "TLS %p could not start certificate validation: "
 		       "%s\n", tls, strerror ( rc ) );
 		return rc;
 	}
-	pending_get ( &tls->validation );
+	pending_get ( &tls->server.validation );
 
 	return 0;
 }
@@ -2577,13 +2585,13 @@ static int tls_new_finished ( struct tls_connection *tls,
 	}
 
 	/* Mark server as finished */
-	pending_put ( &tls->server_negotiation );
+	pending_put ( &tls->server.negotiation );
 
 	/* If we are resuming a session (i.e. if the server Finished
 	 * arrives before the client Finished is sent), then schedule
 	 * transmission of Change Cipher and Finished.
 	 */
-	if ( is_pending ( &tls->client_negotiation ) ) {
+	if ( is_pending ( &tls->client.negotiation ) ) {
 		tls->tx.pending |= ( TLS_TX_CHANGE_CIPHER | TLS_TX_FINISHED );
 		tls_tx_resume ( tls );
 	}
@@ -3295,8 +3303,8 @@ static int tls_progress ( struct tls_connection *tls,
 			  struct job_progress *progress ) {
 
 	/* Return cipherstream or validator progress as applicable */
-	if ( is_pending ( &tls->validation ) ) {
-		return job_progress ( &tls->validator, progress );
+	if ( is_pending ( &tls->server.validation ) ) {
+		return job_progress ( &tls->server.validator, progress );
 	} else {
 		return job_progress ( &tls->cipherstream, progress );
 	}
@@ -3552,10 +3560,10 @@ static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 	struct x509_certificate *cert;
 
 	/* Mark validation as complete */
-	pending_put ( &tls->validation );
+	pending_put ( &tls->server.validation );
 
 	/* Close validator interface */
-	intf_restart ( &tls->validator, rc );
+	intf_restart ( &tls->server.validator, rc );
 
 	/* Check for validation failure */
 	if ( rc != 0 ) {
@@ -3566,7 +3574,7 @@ static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 	DBGC ( tls, "TLS %p certificate validation succeeded\n", tls );
 
 	/* Extract first certificate */
-	cert = x509_first ( tls->chain );
+	cert = x509_first ( tls->server.chain );
 	assert ( cert != NULL );
 
 	/* Verify server name */
@@ -3588,7 +3596,7 @@ static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 	tls->tx.pending |= ( TLS_TX_CLIENT_KEY_EXCHANGE |
 			     TLS_TX_CHANGE_CIPHER |
 			     TLS_TX_FINISHED );
-	if ( tls->certs ) {
+	if ( tls->client.chain ) {
 		tls->tx.pending |= ( TLS_TX_CERTIFICATE |
 				     TLS_TX_CERTIFICATE_VERIFY );
 	}
@@ -3608,7 +3616,8 @@ static struct interface_operation tls_validator_ops[] = {
 
 /** TLS certificate validator interface descriptor */
 static struct interface_descriptor tls_validator_desc =
-	INTF_DESC ( struct tls_connection, validator, tls_validator_ops );
+	INTF_DESC ( struct tls_connection, server.validator,
+		    tls_validator_ops );
 
 /******************************************************************************
  *
@@ -3640,7 +3649,7 @@ static void tls_tx_step ( struct tls_connection *tls ) {
 		list_for_each_entry ( conn, &session->conn, list ) {
 			if ( conn == tls )
 				break;
-			if ( is_pending ( &conn->server_negotiation ) )
+			if ( is_pending ( &conn->server.negotiation ) )
 				return;
 		}
 		/* Record or generate session ID and associated master secret */
@@ -3654,8 +3663,8 @@ static void tls_tx_step ( struct tls_connection *tls ) {
 		} else {
 			/* No existing session: use a random session ID */
 			assert ( sizeof ( tls->session_id ) ==
-				 sizeof ( tls->client_random ) );
-			memcpy ( tls->session_id, &tls->client_random,
+				 sizeof ( tls->client.random ) );
+			memcpy ( tls->session_id, &tls->client.random,
 				 sizeof ( tls->session_id ) );
 			tls->session_id_len = sizeof ( tls->session_id );
 		}
@@ -3756,8 +3765,8 @@ static int tls_session ( struct tls_connection *tls, const char *name ) {
 	/* Find existing matching session, if any */
 	list_for_each_entry ( session, &tls_sessions, list ) {
 		if ( ( strcmp ( name, session->name ) == 0 ) &&
-		     ( tls->root == session->root ) &&
-		     ( tls->key == session->key ) ) {
+		     ( tls->server.root == session->root ) &&
+		     ( tls->client.key == session->key ) ) {
 			ref_get ( &session->refcnt );
 			tls->session = session;
 			DBGC ( tls, "TLS %p joining session %s\n", tls, name );
@@ -3776,8 +3785,8 @@ static int tls_session ( struct tls_connection *tls, const char *name ) {
 	name_copy = ( ( ( void * ) session ) + sizeof ( *session ) );
 	strcpy ( name_copy, name );
 	session->name = name_copy;
-	session->root = x509_root_get ( tls->root );
-	session->key = privkey_get ( tls->key );
+	session->root = x509_root_get ( tls->server.root );
+	session->key = privkey_get ( tls->client.key );
 	INIT_LIST_HEAD ( &session->conn );
 	list_add ( &session->list, &tls_sessions );
 
@@ -3824,23 +3833,23 @@ int add_tls ( struct interface *xfer, const char *name,
 	INIT_LIST_HEAD ( &tls->list );
 	intf_init ( &tls->plainstream, &tls_plainstream_desc, &tls->refcnt );
 	intf_init ( &tls->cipherstream, &tls_cipherstream_desc, &tls->refcnt );
-	intf_init ( &tls->validator, &tls_validator_desc, &tls->refcnt );
+	intf_init ( &tls->server.validator, &tls_validator_desc, &tls->refcnt );
 	process_init_stopped ( &tls->tx.process, &tls_process_desc,
 			       &tls->refcnt );
-	tls->key = privkey_get ( key ? key : &private_key );
-	tls->root = x509_root_get ( root ? root : &root_certificates );
+	tls->client.key = privkey_get ( key ? key : &private_key );
+	tls->server.root = x509_root_get ( root ? root : &root_certificates );
 	tls->version = TLS_VERSION_MAX;
 	tls_clear_cipher ( tls, &tls->tx.cipherspec.active );
 	tls_clear_cipher ( tls, &tls->tx.cipherspec.pending );
 	tls_clear_cipher ( tls, &tls->rx.cipherspec.active );
 	tls_clear_cipher ( tls, &tls->rx.cipherspec.pending );
 	tls_clear_handshake ( tls );
-	tls->client_random.gmt_unix_time = time ( NULL );
+	tls->client.random.gmt_unix_time = time ( NULL );
 	iob_populate ( &tls->rx.iobuf, &tls->rx.header, 0,
 		       sizeof ( tls->rx.header ) );
 	INIT_LIST_HEAD ( &tls->rx.data );
-	if ( ( rc = tls_generate_random ( tls, &tls->client_random.random,
-			  ( sizeof ( tls->client_random.random ) ) ) ) != 0 ) {
+	if ( ( rc = tls_generate_random ( tls, &tls->client.random.random,
+			  ( sizeof ( tls->client.random.random ) ) ) ) != 0 ) {
 		goto err_random;
 	}
 	if ( ( rc = tls_session ( tls, name ) ) != 0 )

From 46937a9df622d1e9fb5b1e926a04176b8855fdce Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 21 Aug 2024 16:25:10 +0100
Subject: [PATCH 178/237] [crypto] Remove the concept of a public-key algorithm
 reusable context

Instances of cipher and digest algorithms tend to get called
repeatedly to process substantial amounts of data.  This is not true
for public-key algorithms, which tend to get called only once or twice
for a given key.

Simplify the public-key algorithm API so that there is no reusable
algorithm context.  In particular, this allows callers to omit the
error handling currently required to handle memory allocation (or key
parsing) errors from pubkey_init(), and to omit the cleanup calls to
pubkey_final().

This change does remove the ability for a caller to distinguish
between a verification failure due to a memory allocation failure and
a verification failure due to a bad signature.  This difference is not
material in practice: in both cases, for whatever reason, the caller
was unable to verify the signature and so cannot proceed further, and
the cause of the error will be visible to the user via the return
status code.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/cms.c          |  19 +--
 src/crypto/crypto_null.c  |  24 +---
 src/crypto/ocsp.c         |  21 +--
 src/crypto/rsa.c          | 295 ++++++++++++++++++++++++--------------
 src/crypto/x509.c         |  13 +-
 src/drivers/net/iphone.c  |  18 +--
 src/include/ipxe/crypto.h |  96 +++++--------
 src/include/ipxe/rsa.h    |  25 ----
 src/include/ipxe/tls.h    |   4 +-
 src/net/tls.c             |  45 ++----
 src/tests/pubkey_test.c   | 142 ++++++------------
 11 files changed, 304 insertions(+), 398 deletions(-)

diff --git a/src/crypto/cms.c b/src/crypto/cms.c
index 0b772f1cf..2e153d819 100644
--- a/src/crypto/cms.c
+++ b/src/crypto/cms.c
@@ -612,33 +612,22 @@ static int cms_verify_digest ( struct cms_message *cms,
 			       userptr_t data, size_t len ) {
 	struct digest_algorithm *digest = part->digest;
 	struct pubkey_algorithm *pubkey = part->pubkey;
-	struct x509_public_key *public_key = &cert->subject.public_key;
+	struct asn1_cursor *key = &cert->subject.public_key.raw;
 	uint8_t digest_out[ digest->digestsize ];
-	uint8_t ctx[ pubkey->ctxsize ];
 	int rc;
 
 	/* Generate digest */
 	cms_digest ( cms, part, data, len, digest_out );
 
-	/* Initialise public-key algorithm */
-	if ( ( rc = pubkey_init ( pubkey, ctx, &public_key->raw ) ) != 0 ) {
-		DBGC ( cms, "CMS %p/%p could not initialise public key: %s\n",
-		       cms, part, strerror ( rc ) );
-		goto err_init;
-	}
-
 	/* Verify digest */
-	if ( ( rc = pubkey_verify ( pubkey, ctx, digest, digest_out,
+	if ( ( rc = pubkey_verify ( pubkey, key, digest, digest_out,
 				    part->value, part->len ) ) != 0 ) {
 		DBGC ( cms, "CMS %p/%p signature verification failed: %s\n",
 		       cms, part, strerror ( rc ) );
-		goto err_verify;
+		return rc;
 	}
 
- err_verify:
-	pubkey_final ( pubkey, ctx );
- err_init:
-	return rc;
+	return 0;
 }
 
 /**
diff --git a/src/crypto/crypto_null.c b/src/crypto/crypto_null.c
index b4169382b..d5863f958 100644
--- a/src/crypto/crypto_null.c
+++ b/src/crypto/crypto_null.c
@@ -93,34 +93,31 @@ struct cipher_algorithm cipher_null = {
 	.auth = cipher_null_auth,
 };
 
-int pubkey_null_init ( void *ctx __unused,
-		       const struct asn1_cursor *key __unused ) {
+size_t pubkey_null_max_len ( const struct asn1_cursor *key __unused ) {
 	return 0;
 }
 
-size_t pubkey_null_max_len ( void *ctx __unused ) {
-	return 0;
-}
-
-int pubkey_null_encrypt ( void *ctx __unused, const void *plaintext __unused,
+int pubkey_null_encrypt ( const struct asn1_cursor *key __unused,
+			  const void *plaintext __unused,
 			  size_t plaintext_len __unused,
 			  void *ciphertext __unused ) {
 	return 0;
 }
 
-int pubkey_null_decrypt ( void *ctx __unused, const void *ciphertext __unused,
+int pubkey_null_decrypt ( const struct asn1_cursor *key __unused,
+			  const void *ciphertext __unused,
 			  size_t ciphertext_len __unused,
 			  void *plaintext __unused ) {
 	return 0;
 }
 
-int pubkey_null_sign ( void *ctx __unused,
+int pubkey_null_sign ( const struct asn1_cursor *key __unused,
 		       struct digest_algorithm *digest __unused,
 		       const void *value __unused, void *signature __unused ) {
 	return 0;
 }
 
-int pubkey_null_verify ( void *ctx __unused,
+int pubkey_null_verify ( const struct asn1_cursor *key __unused,
 			 struct digest_algorithm *digest __unused,
 			 const void *value __unused,
 			 const void *signature __unused ,
@@ -128,18 +125,11 @@ int pubkey_null_verify ( void *ctx __unused,
 	return 0;
 }
 
-void pubkey_null_final ( void *ctx __unused ) {
-	/* Do nothing */
-}
-
 struct pubkey_algorithm pubkey_null = {
 	.name = "null",
-	.ctxsize = 0,
-	.init = pubkey_null_init,
 	.max_len = pubkey_null_max_len,
 	.encrypt = pubkey_null_encrypt,
 	.decrypt = pubkey_null_decrypt,
 	.sign = pubkey_null_sign,
 	.verify = pubkey_null_verify,
-	.final = pubkey_null_final,
 };
diff --git a/src/crypto/ocsp.c b/src/crypto/ocsp.c
index f35593454..e65f7180a 100644
--- a/src/crypto/ocsp.c
+++ b/src/crypto/ocsp.c
@@ -844,10 +844,9 @@ static int ocsp_check_signature ( struct ocsp_check *ocsp,
 	struct ocsp_response *response = &ocsp->response;
 	struct digest_algorithm *digest = response->algorithm->digest;
 	struct pubkey_algorithm *pubkey = response->algorithm->pubkey;
-	struct x509_public_key *public_key = &signer->subject.public_key;
+	struct asn1_cursor *key = &signer->subject.public_key.raw;
 	uint8_t digest_ctx[ digest->ctxsize ];
 	uint8_t digest_out[ digest->digestsize ];
-	uint8_t pubkey_ctx[ pubkey->ctxsize ];
 	int rc;
 
 	/* Generate digest */
@@ -856,30 +855,18 @@ static int ocsp_check_signature ( struct ocsp_check *ocsp,
 			response->tbs.len );
 	digest_final ( digest, digest_ctx, digest_out );
 
-	/* Initialise public-key algorithm */
-	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx,
-				  &public_key->raw ) ) != 0 ) {
-		DBGC ( ocsp, "OCSP %p \"%s\" could not initialise public key: "
-		       "%s\n", ocsp, x509_name ( ocsp->cert ), strerror ( rc ));
-		goto err_init;
-	}
-
 	/* Verify digest */
-	if ( ( rc = pubkey_verify ( pubkey, pubkey_ctx, digest, digest_out,
+	if ( ( rc = pubkey_verify ( pubkey, key, digest, digest_out,
 				    response->signature.data,
 				    response->signature.len ) ) != 0 ) {
 		DBGC ( ocsp, "OCSP %p \"%s\" signature verification failed: "
 		       "%s\n", ocsp, x509_name ( ocsp->cert ), strerror ( rc ));
-		goto err_verify;
+		return rc;
 	}
 
 	DBGC2 ( ocsp, "OCSP %p \"%s\" signature is correct\n",
 		ocsp, x509_name ( ocsp->cert ) );
-
- err_verify:
-	pubkey_final ( pubkey, pubkey_ctx );
- err_init:
-	return rc;
+	return 0;
 }
 
 /**
diff --git a/src/crypto/rsa.c b/src/crypto/rsa.c
index 2d288a953..19472c121 100644
--- a/src/crypto/rsa.c
+++ b/src/crypto/rsa.c
@@ -47,6 +47,28 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define EINFO_EACCES_VERIFY \
 	__einfo_uniqify ( EINFO_EACCES, 0x01, "RSA signature incorrect" )
 
+/** An RSA context */
+struct rsa_context {
+	/** Allocated memory */
+	void *dynamic;
+	/** Modulus */
+	bigint_element_t *modulus0;
+	/** Modulus size */
+	unsigned int size;
+	/** Modulus length */
+	size_t max_len;
+	/** Exponent */
+	bigint_element_t *exponent0;
+	/** Exponent size */
+	unsigned int exponent_size;
+	/** Input buffer */
+	bigint_element_t *input0;
+	/** Output buffer */
+	bigint_element_t *output0;
+	/** Temporary working space for modular exponentiation */
+	void *tmp;
+};
+
 /**
  * Identify RSA prefix
  *
@@ -69,10 +91,9 @@ rsa_find_prefix ( struct digest_algorithm *digest ) {
  *
  * @v context		RSA context
  */
-static void rsa_free ( struct rsa_context *context ) {
+static inline void rsa_free ( struct rsa_context *context ) {
 
 	free ( context->dynamic );
-	context->dynamic = NULL;
 }
 
 /**
@@ -98,9 +119,6 @@ static int rsa_alloc ( struct rsa_context *context, size_t modulus_len,
 		uint8_t tmp[tmp_len];
 	} __attribute__ (( packed )) *dynamic;
 
-	/* Free any existing dynamic storage */
-	rsa_free ( context );
-
 	/* Allocate dynamic storage */
 	dynamic = malloc ( sizeof ( *dynamic ) );
 	if ( ! dynamic )
@@ -231,12 +249,12 @@ static int rsa_parse_mod_exp ( struct asn1_cursor *modulus,
 /**
  * Initialise RSA cipher
  *
- * @v ctx		RSA context
+ * @v context		RSA context
  * @v key		Key
  * @ret rc		Return status code
  */
-static int rsa_init ( void *ctx, const struct asn1_cursor *key ) {
-	struct rsa_context *context = ctx;
+static int rsa_init ( struct rsa_context *context,
+		      const struct asn1_cursor *key ) {
 	struct asn1_cursor modulus;
 	struct asn1_cursor exponent;
 	int rc;
@@ -277,13 +295,22 @@ static int rsa_init ( void *ctx, const struct asn1_cursor *key ) {
 /**
  * Calculate RSA maximum output length
  *
- * @v ctx		RSA context
+ * @v key		Key
  * @ret max_len		Maximum output length
  */
-static size_t rsa_max_len ( void *ctx ) {
-	struct rsa_context *context = ctx;
+static size_t rsa_max_len ( const struct asn1_cursor *key ) {
+	struct asn1_cursor modulus;
+	struct asn1_cursor exponent;
+	int rc;
 
-	return context->max_len;
+	/* Parse moduli and exponents */
+	if ( ( rc = rsa_parse_mod_exp ( &modulus, &exponent, key ) ) != 0 ) {
+		/* Return a zero maximum length on error */
+		return 0;
+	}
+
+	/* Output length can never exceed modulus length */
+	return modulus.len;
 }
 
 /**
@@ -314,111 +341,147 @@ static void rsa_cipher ( struct rsa_context *context,
 /**
  * Encrypt using RSA
  *
- * @v ctx		RSA context
+ * @v key		Key
  * @v plaintext		Plaintext
  * @v plaintext_len	Length of plaintext
  * @v ciphertext	Ciphertext
  * @ret ciphertext_len	Length of ciphertext, or negative error
  */
-static int rsa_encrypt ( void *ctx, const void *plaintext,
+static int rsa_encrypt ( const struct asn1_cursor *key, const void *plaintext,
 			 size_t plaintext_len, void *ciphertext ) {
-	struct rsa_context *context = ctx;
+	struct rsa_context context;
 	void *temp;
 	uint8_t *encoded;
-	size_t max_len = ( context->max_len - 11 );
-	size_t random_nz_len = ( max_len - plaintext_len + 8 );
+	size_t max_len;
+	size_t random_nz_len;
 	int rc;
 
+	DBGC ( &context, "RSA %p encrypting:\n", &context );
+	DBGC_HDA ( &context, 0, plaintext, plaintext_len );
+
+	/* Initialise context */
+	if ( ( rc = rsa_init ( &context, key ) ) != 0 )
+		goto err_init;
+
+	/* Calculate lengths */
+	max_len = ( context.max_len - 11 );
+	random_nz_len = ( max_len - plaintext_len + 8 );
+
 	/* Sanity check */
 	if ( plaintext_len > max_len ) {
-		DBGC ( context, "RSA %p plaintext too long (%zd bytes, max "
-		       "%zd)\n", context, plaintext_len, max_len );
-		return -ERANGE;
+		DBGC ( &context, "RSA %p plaintext too long (%zd bytes, max "
+		       "%zd)\n", &context, plaintext_len, max_len );
+		rc = -ERANGE;
+		goto err_sanity;
 	}
-	DBGC ( context, "RSA %p encrypting:\n", context );
-	DBGC_HDA ( context, 0, plaintext, plaintext_len );
 
 	/* Construct encoded message (using the big integer output
 	 * buffer as temporary storage)
 	 */
-	temp = context->output0;
+	temp = context.output0;
 	encoded = temp;
 	encoded[0] = 0x00;
 	encoded[1] = 0x02;
 	if ( ( rc = get_random_nz ( &encoded[2], random_nz_len ) ) != 0 ) {
-		DBGC ( context, "RSA %p could not generate random data: %s\n",
-		       context, strerror ( rc ) );
-		return rc;
+		DBGC ( &context, "RSA %p could not generate random data: %s\n",
+		       &context, strerror ( rc ) );
+		goto err_random;
 	}
 	encoded[ 2 + random_nz_len ] = 0x00;
-	memcpy ( &encoded[ context->max_len - plaintext_len ],
+	memcpy ( &encoded[ context.max_len - plaintext_len ],
 		 plaintext, plaintext_len );
 
 	/* Encipher the encoded message */
-	rsa_cipher ( context, encoded, ciphertext );
-	DBGC ( context, "RSA %p encrypted:\n", context );
-	DBGC_HDA ( context, 0, ciphertext, context->max_len );
+	rsa_cipher ( &context, encoded, ciphertext );
+	DBGC ( &context, "RSA %p encrypted:\n", &context );
+	DBGC_HDA ( &context, 0, ciphertext, context.max_len );
 
-	return context->max_len;
+	/* Free context */
+	rsa_free ( &context );
+
+	return context.max_len;
+
+ err_random:
+ err_sanity:
+	rsa_free ( &context );
+ err_init:
+	return rc;
 }
 
 /**
  * Decrypt using RSA
  *
- * @v ctx		RSA context
+ * @v key		Key
  * @v ciphertext	Ciphertext
  * @v ciphertext_len	Ciphertext length
  * @v plaintext		Plaintext
  * @ret plaintext_len	Plaintext length, or negative error
  */
-static int rsa_decrypt ( void *ctx, const void *ciphertext,
+static int rsa_decrypt ( const struct asn1_cursor *key, const void *ciphertext,
 			 size_t ciphertext_len, void *plaintext ) {
-	struct rsa_context *context = ctx;
+	struct rsa_context context;
 	void *temp;
 	uint8_t *encoded;
 	uint8_t *end;
 	uint8_t *zero;
 	uint8_t *start;
 	size_t plaintext_len;
+	int rc;
+
+	DBGC ( &context, "RSA %p decrypting:\n", &context );
+	DBGC_HDA ( &context, 0, ciphertext, ciphertext_len );
+
+	/* Initialise context */
+	if ( ( rc = rsa_init ( &context, key ) ) != 0 )
+		goto err_init;
 
 	/* Sanity check */
-	if ( ciphertext_len != context->max_len ) {
-		DBGC ( context, "RSA %p ciphertext incorrect length (%zd "
+	if ( ciphertext_len != context.max_len ) {
+		DBGC ( &context, "RSA %p ciphertext incorrect length (%zd "
 		       "bytes, should be %zd)\n",
-		       context, ciphertext_len, context->max_len );
-		return -ERANGE;
+		       &context, ciphertext_len, context.max_len );
+		rc = -ERANGE;
+		goto err_sanity;
 	}
-	DBGC ( context, "RSA %p decrypting:\n", context );
-	DBGC_HDA ( context, 0, ciphertext, ciphertext_len );
 
 	/* Decipher the message (using the big integer input buffer as
 	 * temporary storage)
 	 */
-	temp = context->input0;
+	temp = context.input0;
 	encoded = temp;
-	rsa_cipher ( context, ciphertext, encoded );
+	rsa_cipher ( &context, ciphertext, encoded );
 
 	/* Parse the message */
-	end = ( encoded + context->max_len );
-	if ( ( encoded[0] != 0x00 ) || ( encoded[1] != 0x02 ) )
-		goto invalid;
+	end = ( encoded + context.max_len );
+	if ( ( encoded[0] != 0x00 ) || ( encoded[1] != 0x02 ) ) {
+		rc = -EINVAL;
+		goto err_invalid;
+	}
 	zero = memchr ( &encoded[2], 0, ( end - &encoded[2] ) );
-	if ( ! zero )
-		goto invalid;
+	if ( ! zero ) {
+		rc = -EINVAL;
+		goto err_invalid;
+	}
 	start = ( zero + 1 );
 	plaintext_len = ( end - start );
 
 	/* Copy out message */
 	memcpy ( plaintext, start, plaintext_len );
-	DBGC ( context, "RSA %p decrypted:\n", context );
-	DBGC_HDA ( context, 0, plaintext, plaintext_len );
+	DBGC ( &context, "RSA %p decrypted:\n", &context );
+	DBGC_HDA ( &context, 0, plaintext, plaintext_len );
+
+	/* Free context */
+	rsa_free ( &context );
 
 	return plaintext_len;
 
- invalid:
-	DBGC ( context, "RSA %p invalid decrypted message:\n", context );
-	DBGC_HDA ( context, 0, encoded, context->max_len );
-	return -EINVAL;
+ err_invalid:
+	DBGC ( &context, "RSA %p invalid decrypted message:\n", &context );
+	DBGC_HDA ( &context, 0, encoded, context.max_len );
+ err_sanity:
+	rsa_free ( &context );
+ err_init:
+	return rc;
 }
 
 /**
@@ -452,9 +515,9 @@ static int rsa_encode_digest ( struct rsa_context *context,
 	/* Sanity check */
 	max_len = ( context->max_len - 11 );
 	if ( digestinfo_len > max_len ) {
-		DBGC ( context, "RSA %p %s digestInfo too long (%zd bytes, max"
-		       "%zd)\n",
-		       context, digest->name, digestinfo_len, max_len );
+		DBGC ( context, "RSA %p %s digestInfo too long (%zd bytes, "
+		       "max %zd)\n", context, digest->name, digestinfo_len,
+		       max_len );
 		return -ERANGE;
 	}
 	DBGC ( context, "RSA %p encoding %s digest:\n",
@@ -482,104 +545,125 @@ static int rsa_encode_digest ( struct rsa_context *context,
 /**
  * Sign digest value using RSA
  *
- * @v ctx		RSA context
+ * @v key		Key
  * @v digest		Digest algorithm
  * @v value		Digest value
  * @v signature		Signature
  * @ret signature_len	Signature length, or negative error
  */
-static int rsa_sign ( void *ctx, struct digest_algorithm *digest,
-		      const void *value, void *signature ) {
-	struct rsa_context *context = ctx;
+static int rsa_sign ( const struct asn1_cursor *key,
+		      struct digest_algorithm *digest, const void *value,
+		      void *signature ) {
+	struct rsa_context context;
 	void *temp;
 	int rc;
 
-	DBGC ( context, "RSA %p signing %s digest:\n", context, digest->name );
-	DBGC_HDA ( context, 0, value, digest->digestsize );
+	DBGC ( &context, "RSA %p signing %s digest:\n",
+	       &context, digest->name );
+	DBGC_HDA ( &context, 0, value, digest->digestsize );
+
+	/* Initialise context */
+	if ( ( rc = rsa_init ( &context, key ) ) != 0 )
+		goto err_init;
 
 	/* Encode digest (using the big integer output buffer as
 	 * temporary storage)
 	 */
-	temp = context->output0;
-	if ( ( rc = rsa_encode_digest ( context, digest, value, temp ) ) != 0 )
-		return rc;
+	temp = context.output0;
+	if ( ( rc = rsa_encode_digest ( &context, digest, value, temp ) ) != 0 )
+		goto err_encode;
 
 	/* Encipher the encoded digest */
-	rsa_cipher ( context, temp, signature );
-	DBGC ( context, "RSA %p signed %s digest:\n", context, digest->name );
-	DBGC_HDA ( context, 0, signature, context->max_len );
+	rsa_cipher ( &context, temp, signature );
+	DBGC ( &context, "RSA %p signed %s digest:\n", &context, digest->name );
+	DBGC_HDA ( &context, 0, signature, context.max_len );
 
-	return context->max_len;
+	/* Free context */
+	rsa_free ( &context );
+
+	return context.max_len;
+
+ err_encode:
+	rsa_free ( &context );
+ err_init:
+	return rc;
 }
 
 /**
  * Verify signed digest value using RSA
  *
- * @v ctx		RSA context
+ * @v key		Key
  * @v digest		Digest algorithm
  * @v value		Digest value
  * @v signature		Signature
  * @v signature_len	Signature length
  * @ret rc		Return status code
  */
-static int rsa_verify ( void *ctx, struct digest_algorithm *digest,
-			const void *value, const void *signature,
-			size_t signature_len ) {
-	struct rsa_context *context = ctx;
+static int rsa_verify ( const struct asn1_cursor *key,
+			struct digest_algorithm *digest, const void *value,
+			const void *signature, size_t signature_len ) {
+	struct rsa_context context;
 	void *temp;
 	void *expected;
 	void *actual;
 	int rc;
 
+	DBGC ( &context, "RSA %p verifying %s digest:\n",
+	       &context, digest->name );
+	DBGC_HDA ( &context, 0, value, digest->digestsize );
+	DBGC_HDA ( &context, 0, signature, signature_len );
+
+	/* Initialise context */
+	if ( ( rc = rsa_init ( &context, key ) ) != 0 )
+		goto err_init;
+
 	/* Sanity check */
-	if ( signature_len != context->max_len ) {
-		DBGC ( context, "RSA %p signature incorrect length (%zd "
+	if ( signature_len != context.max_len ) {
+		DBGC ( &context, "RSA %p signature incorrect length (%zd "
 		       "bytes, should be %zd)\n",
-		       context, signature_len, context->max_len );
-		return -ERANGE;
+		       &context, signature_len, context.max_len );
+		rc = -ERANGE;
+		goto err_sanity;
 	}
-	DBGC ( context, "RSA %p verifying %s digest:\n",
-	       context, digest->name );
-	DBGC_HDA ( context, 0, value, digest->digestsize );
-	DBGC_HDA ( context, 0, signature, signature_len );
 
 	/* Decipher the signature (using the big integer input buffer
 	 * as temporary storage)
 	 */
-	temp = context->input0;
+	temp = context.input0;
 	expected = temp;
-	rsa_cipher ( context, signature, expected );
-	DBGC ( context, "RSA %p deciphered signature:\n", context );
-	DBGC_HDA ( context, 0, expected, context->max_len );
+	rsa_cipher ( &context, signature, expected );
+	DBGC ( &context, "RSA %p deciphered signature:\n", &context );
+	DBGC_HDA ( &context, 0, expected, context.max_len );
 
 	/* Encode digest (using the big integer output buffer as
 	 * temporary storage)
 	 */
-	temp = context->output0;
+	temp = context.output0;
 	actual = temp;
-	if ( ( rc = rsa_encode_digest ( context, digest, value, actual ) ) !=0 )
-		return rc;
+	if ( ( rc = rsa_encode_digest ( &context, digest, value,
+					actual ) ) != 0 )
+		goto err_encode;
 
 	/* Verify the signature */
-	if ( memcmp ( actual, expected, context->max_len ) != 0 ) {
-		DBGC ( context, "RSA %p signature verification failed\n",
-		       context );
-		return -EACCES_VERIFY;
+	if ( memcmp ( actual, expected, context.max_len ) != 0 ) {
+		DBGC ( &context, "RSA %p signature verification failed\n",
+		       &context );
+		rc = -EACCES_VERIFY;
+		goto err_verify;
 	}
 
-	DBGC ( context, "RSA %p signature verified successfully\n", context );
+	/* Free context */
+	rsa_free ( &context );
+
+	DBGC ( &context, "RSA %p signature verified successfully\n", &context );
 	return 0;
-}
 
-/**
- * Finalise RSA cipher
- *
- * @v ctx		RSA context
- */
-static void rsa_final ( void *ctx ) {
-	struct rsa_context *context = ctx;
-
-	rsa_free ( context );
+ err_verify:
+ err_encode:
+ err_sanity:
+	rsa_free ( &context );
+ err_init:
+	return rc;
 }
 
 /**
@@ -615,14 +699,11 @@ static int rsa_match ( const struct asn1_cursor *private_key,
 /** RSA public-key algorithm */
 struct pubkey_algorithm rsa_algorithm = {
 	.name		= "rsa",
-	.ctxsize	= RSA_CTX_SIZE,
-	.init		= rsa_init,
 	.max_len	= rsa_max_len,
 	.encrypt	= rsa_encrypt,
 	.decrypt	= rsa_decrypt,
 	.sign		= rsa_sign,
 	.verify		= rsa_verify,
-	.final		= rsa_final,
 	.match		= rsa_match,
 };
 
diff --git a/src/crypto/x509.c b/src/crypto/x509.c
index c0762740e..4101c8094 100644
--- a/src/crypto/x509.c
+++ b/src/crypto/x509.c
@@ -1125,7 +1125,6 @@ static int x509_check_signature ( struct x509_certificate *cert,
 	struct pubkey_algorithm *pubkey = algorithm->pubkey;
 	uint8_t digest_ctx[ digest->ctxsize ];
 	uint8_t digest_out[ digest->digestsize ];
-	uint8_t pubkey_ctx[ pubkey->ctxsize ];
 	int rc;
 
 	/* Sanity check */
@@ -1149,14 +1148,8 @@ static int x509_check_signature ( struct x509_certificate *cert,
 	}
 
 	/* Verify signature using signer's public key */
-	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx,
-				  &public_key->raw ) ) != 0 ) {
-		DBGC ( cert, "X509 %p \"%s\" cannot initialise public key: "
-		       "%s\n", cert, x509_name ( cert ), strerror ( rc ) );
-		goto err_pubkey_init;
-	}
-	if ( ( rc = pubkey_verify ( pubkey, pubkey_ctx, digest, digest_out,
-				    signature->value.data,
+	if ( ( rc = pubkey_verify ( pubkey, &public_key->raw, digest,
+				    digest_out, signature->value.data,
 				    signature->value.len ) ) != 0 ) {
 		DBGC ( cert, "X509 %p \"%s\" signature verification failed: "
 		       "%s\n", cert, x509_name ( cert ), strerror ( rc ) );
@@ -1167,8 +1160,6 @@ static int x509_check_signature ( struct x509_certificate *cert,
 	rc = 0;
 
  err_pubkey_verify:
-	pubkey_final ( pubkey, pubkey_ctx );
- err_pubkey_init:
  err_mismatch:
 	return rc;
 }
diff --git a/src/drivers/net/iphone.c b/src/drivers/net/iphone.c
index 96eb0952b..08459a6e2 100644
--- a/src/drivers/net/iphone.c
+++ b/src/drivers/net/iphone.c
@@ -362,17 +362,9 @@ static int icert_cert ( struct icert *icert, struct asn1_cursor *subject,
 	struct asn1_builder raw = { NULL, 0 };
 	uint8_t digest_ctx[SHA256_CTX_SIZE];
 	uint8_t digest_out[SHA256_DIGEST_SIZE];
-	uint8_t pubkey_ctx[RSA_CTX_SIZE];
 	int len;
 	int rc;
 
-	/* Initialise "private" key */
-	if ( ( rc = pubkey_init ( pubkey, pubkey_ctx, private ) ) != 0 ) {
-		DBGC ( icert, "ICERT %p could not initialise private key: "
-		       "%s\n", icert, strerror ( rc ) );
-		goto err_pubkey_init;
-	}
-
 	/* Construct subjectPublicKeyInfo */
 	if ( ( rc = ( asn1_prepend_raw ( &spki, public->data, public->len ),
 		      asn1_prepend_raw ( &spki, icert_nul,
@@ -406,14 +398,14 @@ static int icert_cert ( struct icert *icert, struct asn1_cursor *subject,
 	digest_update ( digest, digest_ctx, tbs.data, tbs.len );
 	digest_final ( digest, digest_ctx, digest_out );
 
-	/* Construct signature */
-	if ( ( rc = asn1_grow ( &raw, pubkey_max_len ( pubkey,
-						       pubkey_ctx ) ) ) != 0 ) {
+	/* Construct signature using "private" key */
+	if ( ( rc = asn1_grow ( &raw,
+				pubkey_max_len ( pubkey, private ) ) ) != 0 ) {
 		DBGC ( icert, "ICERT %p could not build signature: %s\n",
 		       icert, strerror ( rc ) );
 		goto err_grow;
 	}
-	if ( ( len = pubkey_sign ( pubkey, pubkey_ctx, digest, digest_out,
+	if ( ( len = pubkey_sign ( pubkey, private, digest, digest_out,
 				   raw.data ) ) < 0 ) {
 		rc = len;
 		DBGC ( icert, "ICERT %p could not sign: %s\n",
@@ -452,8 +444,6 @@ static int icert_cert ( struct icert *icert, struct asn1_cursor *subject,
  err_tbs:
 	free ( spki.data );
  err_spki:
-	pubkey_final ( pubkey, pubkey_ctx );
- err_pubkey_init:
 	return rc;
 }
 
diff --git a/src/include/ipxe/crypto.h b/src/include/ipxe/crypto.h
index 8b6eb94f6..dcc73f3ef 100644
--- a/src/include/ipxe/crypto.h
+++ b/src/include/ipxe/crypto.h
@@ -121,68 +121,55 @@ struct cipher_algorithm {
 struct pubkey_algorithm {
 	/** Algorithm name */
 	const char *name;
-	/** Context size */
-	size_t ctxsize;
-	/** Initialise algorithm
-	 *
-	 * @v ctx		Context
-	 * @v key		Key
-	 * @ret rc		Return status code
-	 */
-	int ( * init ) ( void *ctx, const struct asn1_cursor *key );
 	/** Calculate maximum output length
 	 *
-	 * @v ctx		Context
+	 * @v key		Key
 	 * @ret max_len		Maximum output length
 	 */
-	size_t ( * max_len ) ( void *ctx );
+	size_t ( * max_len ) ( const struct asn1_cursor *key );
 	/** Encrypt
 	 *
-	 * @v ctx		Context
+	 * @v key		Key
 	 * @v plaintext		Plaintext
 	 * @v plaintext_len	Length of plaintext
 	 * @v ciphertext	Ciphertext
 	 * @ret ciphertext_len	Length of ciphertext, or negative error
 	 */
-	int ( * encrypt ) ( void *ctx, const void *data, size_t len,
-			    void *out );
+	int ( * encrypt ) ( const struct asn1_cursor *key, const void *data,
+			    size_t len, void *out );
 	/** Decrypt
 	 *
-	 * @v ctx		Context
+	 * @v key		Key
 	 * @v ciphertext	Ciphertext
 	 * @v ciphertext_len	Ciphertext length
 	 * @v plaintext		Plaintext
 	 * @ret plaintext_len	Plaintext length, or negative error
 	 */
-	int ( * decrypt ) ( void *ctx, const void *data, size_t len,
-			    void *out );
+	int ( * decrypt ) ( const struct asn1_cursor *key, const void *data,
+			    size_t len, void *out );
 	/** Sign digest value
 	 *
-	 * @v ctx		Context
+	 * @v key		Key
 	 * @v digest		Digest algorithm
 	 * @v value		Digest value
 	 * @v signature		Signature
 	 * @ret signature_len	Signature length, or negative error
 	 */
-	int ( * sign ) ( void *ctx, struct digest_algorithm *digest,
-			 const void *value, void *signature );
+	int ( * sign ) ( const struct asn1_cursor *key,
+			 struct digest_algorithm *digest, const void *value,
+			 void *signature );
 	/** Verify signed digest value
 	 *
-	 * @v ctx		Context
+	 * @v key		Key
 	 * @v digest		Digest algorithm
 	 * @v value		Digest value
 	 * @v signature		Signature
 	 * @v signature_len	Signature length
 	 * @ret rc		Return status code
 	 */
-	int ( * verify ) ( void *ctx, struct digest_algorithm *digest,
-			   const void *value, const void *signature,
-			   size_t signature_len );
-	/** Finalise algorithm
-	 *
-	 * @v ctx		Context
-	 */
-	void ( * final ) ( void *ctx );
+	int ( * verify ) ( const struct asn1_cursor *key,
+			   struct digest_algorithm *digest, const void *value,
+			   const void *signature, size_t signature_len );
 	/** Check that public key matches private key
 	 *
 	 * @v private_key	Private key
@@ -278,46 +265,36 @@ is_auth_cipher ( struct cipher_algorithm *cipher ) {
 	return cipher->authsize;
 }
 
-static inline __attribute__ (( always_inline )) int
-pubkey_init ( struct pubkey_algorithm *pubkey, void *ctx,
-	      const struct asn1_cursor *key ) {
-	return pubkey->init ( ctx, key );
-}
-
 static inline __attribute__ (( always_inline )) size_t
-pubkey_max_len ( struct pubkey_algorithm *pubkey, void *ctx ) {
-	return pubkey->max_len ( ctx );
+pubkey_max_len ( struct pubkey_algorithm *pubkey,
+		 const struct asn1_cursor *key ) {
+	return pubkey->max_len ( key );
 }
 
 static inline __attribute__ (( always_inline )) int
-pubkey_encrypt ( struct pubkey_algorithm *pubkey, void *ctx,
+pubkey_encrypt ( struct pubkey_algorithm *pubkey, const struct asn1_cursor *key,
 		 const void *data, size_t len, void *out ) {
-	return pubkey->encrypt ( ctx, data, len, out );
+	return pubkey->encrypt ( key, data, len, out );
 }
 
 static inline __attribute__ (( always_inline )) int
-pubkey_decrypt ( struct pubkey_algorithm *pubkey, void *ctx,
+pubkey_decrypt ( struct pubkey_algorithm *pubkey, const struct asn1_cursor *key,
 		 const void *data, size_t len, void *out ) {
-	return pubkey->decrypt ( ctx, data, len, out );
+	return pubkey->decrypt ( key, data, len, out );
 }
 
 static inline __attribute__ (( always_inline )) int
-pubkey_sign ( struct pubkey_algorithm *pubkey, void *ctx,
+pubkey_sign ( struct pubkey_algorithm *pubkey, const struct asn1_cursor *key,
 	      struct digest_algorithm *digest, const void *value,
 	      void *signature ) {
-	return pubkey->sign ( ctx, digest, value, signature );
+	return pubkey->sign ( key, digest, value, signature );
 }
 
 static inline __attribute__ (( always_inline )) int
-pubkey_verify ( struct pubkey_algorithm *pubkey, void *ctx,
+pubkey_verify ( struct pubkey_algorithm *pubkey, const struct asn1_cursor *key,
 		struct digest_algorithm *digest, const void *value,
 		const void *signature, size_t signature_len ) {
-	return pubkey->verify ( ctx, digest, value, signature, signature_len );
-}
-
-static inline __attribute__ (( always_inline )) void
-pubkey_final ( struct pubkey_algorithm *pubkey, void *ctx ) {
-	pubkey->final ( ctx );
+	return pubkey->verify ( key, digest, value, signature, signature_len );
 }
 
 static inline __attribute__ (( always_inline )) int
@@ -345,15 +322,18 @@ extern void cipher_null_decrypt ( void *ctx, const void *src, void *dst,
 				  size_t len );
 extern void cipher_null_auth ( void *ctx, void *auth );
 
-extern int pubkey_null_init ( void *ctx, const struct asn1_cursor *key );
-extern size_t pubkey_null_max_len ( void *ctx );
-extern int pubkey_null_encrypt ( void *ctx, const void *plaintext,
-				 size_t plaintext_len, void *ciphertext );
-extern int pubkey_null_decrypt ( void *ctx, const void *ciphertext,
-				 size_t ciphertext_len, void *plaintext );
-extern int pubkey_null_sign ( void *ctx, struct digest_algorithm *digest,
+extern size_t pubkey_null_max_len ( const struct asn1_cursor *key );
+extern int pubkey_null_encrypt ( const struct asn1_cursor *key,
+				 const void *plaintext, size_t plaintext_len,
+				 void *ciphertext );
+extern int pubkey_null_decrypt ( const struct asn1_cursor *key,
+				 const void *ciphertext, size_t ciphertext_len,
+				 void *plaintext );
+extern int pubkey_null_sign ( const struct asn1_cursor *key,
+			      struct digest_algorithm *digest,
 			      const void *value, void *signature );
-extern int pubkey_null_verify ( void *ctx, struct digest_algorithm *digest,
+extern int pubkey_null_verify ( const struct asn1_cursor *key,
+				struct digest_algorithm *digest,
 				const void *value, const void *signature ,
 				size_t signature_len );
 
diff --git a/src/include/ipxe/rsa.h b/src/include/ipxe/rsa.h
index a1b5e0c03..e36a75edf 100644
--- a/src/include/ipxe/rsa.h
+++ b/src/include/ipxe/rsa.h
@@ -55,31 +55,6 @@ struct rsa_digestinfo_prefix {
 /** Declare an RSA digestInfo prefix */
 #define __rsa_digestinfo_prefix __table_entry ( RSA_DIGESTINFO_PREFIXES, 01 )
 
-/** An RSA context */
-struct rsa_context {
-	/** Allocated memory */
-	void *dynamic;
-	/** Modulus */
-	bigint_element_t *modulus0;
-	/** Modulus size */
-	unsigned int size;
-	/** Modulus length */
-	size_t max_len;
-	/** Exponent */
-	bigint_element_t *exponent0;
-	/** Exponent size */
-	unsigned int exponent_size;
-	/** Input buffer */
-	bigint_element_t *input0;
-	/** Output buffer */
-	bigint_element_t *output0;
-	/** Temporary working space for modular exponentiation */
-	void *tmp;
-};
-
-/** RSA context size */
-#define RSA_CTX_SIZE sizeof ( struct rsa_context )
-
 extern struct pubkey_algorithm rsa_algorithm;
 
 #endif /* _IPXE_RSA_H */
diff --git a/src/include/ipxe/tls.h b/src/include/ipxe/tls.h
index 9494eaa05..08d58689e 100644
--- a/src/include/ipxe/tls.h
+++ b/src/include/ipxe/tls.h
@@ -240,8 +240,6 @@ struct tls_cipherspec {
 	struct tls_cipher_suite *suite;
 	/** Dynamically-allocated storage */
 	void *dynamic;
-	/** Public key encryption context */
-	void *pubkey_ctx;
 	/** Bulk encryption cipher context */
 	void *cipher_ctx;
 	/** MAC secret */
@@ -402,6 +400,8 @@ struct tls_server {
 	struct x509_root *root;
 	/** Certificate chain */
 	struct x509_chain *chain;
+	/** Public key (within server certificate) */
+	struct asn1_cursor key;
 	/** Certificate validator */
 	struct interface validator;
 	/** Certificate validation pending operation */
diff --git a/src/net/tls.c b/src/net/tls.c
index ec503e43d..ded100d0e 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -856,10 +856,6 @@ tls_find_cipher_suite ( unsigned int cipher_suite ) {
 static void tls_clear_cipher ( struct tls_connection *tls __unused,
 			       struct tls_cipherspec *cipherspec ) {
 
-	if ( cipherspec->suite ) {
-		pubkey_final ( cipherspec->suite->pubkey,
-			       cipherspec->pubkey_ctx );
-	}
 	free ( cipherspec->dynamic );
 	memset ( cipherspec, 0, sizeof ( *cipherspec ) );
 	cipherspec->suite = &tls_cipher_suite_null;
@@ -876,7 +872,6 @@ static void tls_clear_cipher ( struct tls_connection *tls __unused,
 static int tls_set_cipher ( struct tls_connection *tls,
 			    struct tls_cipherspec *cipherspec,
 			    struct tls_cipher_suite *suite ) {
-	struct pubkey_algorithm *pubkey = suite->pubkey;
 	struct cipher_algorithm *cipher = suite->cipher;
 	size_t total;
 	void *dynamic;
@@ -885,8 +880,7 @@ static int tls_set_cipher ( struct tls_connection *tls,
 	tls_clear_cipher ( tls, cipherspec );
 
 	/* Allocate dynamic storage */
-	total = ( pubkey->ctxsize + cipher->ctxsize + suite->mac_len +
-		  suite->fixed_iv_len );
+	total = ( cipher->ctxsize + suite->mac_len + suite->fixed_iv_len );
 	dynamic = zalloc ( total );
 	if ( ! dynamic ) {
 		DBGC ( tls, "TLS %p could not allocate %zd bytes for crypto "
@@ -896,7 +890,6 @@ static int tls_set_cipher ( struct tls_connection *tls,
 
 	/* Assign storage */
 	cipherspec->dynamic = dynamic;
-	cipherspec->pubkey_ctx = dynamic;	dynamic += pubkey->ctxsize;
 	cipherspec->cipher_ctx = dynamic;	dynamic += cipher->ctxsize;
 	cipherspec->mac_secret = dynamic;	dynamic += suite->mac_len;
 	cipherspec->fixed_iv = dynamic;		dynamic += suite->fixed_iv_len;
@@ -1392,7 +1385,7 @@ static int tls_send_certificate ( struct tls_connection *tls ) {
 static int tls_send_client_key_exchange_pubkey ( struct tls_connection *tls ) {
 	struct tls_cipherspec *cipherspec = &tls->tx.cipherspec.pending;
 	struct pubkey_algorithm *pubkey = cipherspec->suite->pubkey;
-	size_t max_len = pubkey_max_len ( pubkey, cipherspec->pubkey_ctx );
+	size_t max_len = pubkey_max_len ( pubkey, &tls->server.key );
 	struct {
 		uint16_t version;
 		uint8_t random[46];
@@ -1419,8 +1412,8 @@ static int tls_send_client_key_exchange_pubkey ( struct tls_connection *tls ) {
 
 	/* Encrypt pre-master secret using server's public key */
 	memset ( &key_xchg, 0, sizeof ( key_xchg ) );
-	len = pubkey_encrypt ( pubkey, cipherspec->pubkey_ctx,
-			       &pre_master_secret, sizeof ( pre_master_secret ),
+	len = pubkey_encrypt ( pubkey, &tls->server.key, &pre_master_secret,
+			       sizeof ( pre_master_secret ),
 			       key_xchg.encrypted_pre_master_secret );
 	if ( len < 0 ) {
 		rc = len;
@@ -1523,7 +1516,7 @@ static int tls_verify_dh_params ( struct tls_connection *tls,
 		digest_final ( digest, ctx, hash );
 
 		/* Verify signature */
-		if ( ( rc = pubkey_verify ( pubkey, cipherspec->pubkey_ctx,
+		if ( ( rc = pubkey_verify ( pubkey, &tls->server.key,
 					    digest, hash, signature,
 					    signature_len ) ) != 0 ) {
 			DBGC ( tls, "TLS %p ServerKeyExchange failed "
@@ -1820,20 +1813,12 @@ static int tls_send_certificate_verify ( struct tls_connection *tls ) {
 	struct pubkey_algorithm *pubkey = cert->signature_algorithm->pubkey;
 	struct asn1_cursor *key = privkey_cursor ( tls->client.key );
 	uint8_t digest_out[ digest->digestsize ];
-	uint8_t ctx[ pubkey->ctxsize ];
 	struct tls_signature_hash_algorithm *sig_hash = NULL;
 	int rc;
 
 	/* Generate digest to be signed */
 	tls_verify_handshake ( tls, digest_out );
 
-	/* Initialise public-key algorithm */
-	if ( ( rc = pubkey_init ( pubkey, ctx, key ) ) != 0 ) {
-		DBGC ( tls, "TLS %p could not initialise %s client private "
-		       "key: %s\n", tls, pubkey->name, strerror ( rc ) );
-		goto err_pubkey_init;
-	}
-
 	/* TLSv1.2 and later use explicit algorithm identifiers */
 	if ( tls_version ( tls, TLS_VERSION_TLS_1_2 ) ) {
 		sig_hash = tls_signature_hash_algorithm ( pubkey, digest );
@@ -1848,7 +1833,7 @@ static int tls_send_certificate_verify ( struct tls_connection *tls ) {
 
 	/* Generate and transmit record */
 	{
-		size_t max_len = pubkey_max_len ( pubkey, ctx );
+		size_t max_len = pubkey_max_len ( pubkey, key );
 		int use_sig_hash = ( ( sig_hash == NULL ) ? 0 : 1 );
 		struct {
 			uint32_t type_length;
@@ -1860,7 +1845,7 @@ static int tls_send_certificate_verify ( struct tls_connection *tls ) {
 		int len;
 
 		/* Sign digest */
-		len = pubkey_sign ( pubkey, ctx, digest, digest_out,
+		len = pubkey_sign ( pubkey, key, digest, digest_out,
 				    certificate_verify.signature );
 		if ( len < 0 ) {
 			rc = len;
@@ -1893,8 +1878,6 @@ static int tls_send_certificate_verify ( struct tls_connection *tls ) {
 
  err_pubkey_sign:
  err_sig_hash:
-	pubkey_final ( pubkey, ctx );
- err_pubkey_init:
 	return rc;
 }
 
@@ -2312,6 +2295,7 @@ static int tls_parse_chain ( struct tls_connection *tls,
 	int rc;
 
 	/* Free any existing certificate chain */
+	memset ( &tls->server.key, 0, sizeof ( tls->server.key ) );
 	x509_chain_put ( tls->server.chain );
 	tls->server.chain = NULL;
 
@@ -2371,6 +2355,7 @@ static int tls_parse_chain ( struct tls_connection *tls,
  err_parse:
  err_overlength:
  err_underlength:
+	memset ( &tls->server.key, 0, sizeof ( tls->server.key ) );
 	x509_chain_put ( tls->server.chain );
 	tls->server.chain = NULL;
  err_alloc_chain:
@@ -3555,8 +3540,6 @@ static struct interface_descriptor tls_cipherstream_desc =
  */
 static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 	struct tls_session *session = tls->session;
-	struct tls_cipherspec *cipherspec = &tls->tx.cipherspec.pending;
-	struct pubkey_algorithm *pubkey = cipherspec->suite->pubkey;
 	struct x509_certificate *cert;
 
 	/* Mark validation as complete */
@@ -3584,13 +3567,9 @@ static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 		goto err;
 	}
 
-	/* Initialise public key algorithm */
-	if ( ( rc = pubkey_init ( pubkey, cipherspec->pubkey_ctx,
-				  &cert->subject.public_key.raw ) ) != 0 ) {
-		DBGC ( tls, "TLS %p cannot initialise public key: %s\n",
-		       tls, strerror ( rc ) );
-		goto err;
-	}
+	/* Extract the now trusted server public key */
+	memcpy ( &tls->server.key, &cert->subject.public_key.raw,
+		 sizeof ( tls->server.key ) );
 
 	/* Schedule Client Key Exchange, Change Cipher, and Finished */
 	tls->tx.pending |= ( TLS_TX_CLIENT_KEY_EXCHANGE |
diff --git a/src/tests/pubkey_test.c b/src/tests/pubkey_test.c
index 93962516a..ff318bfb7 100644
--- a/src/tests/pubkey_test.c
+++ b/src/tests/pubkey_test.c
@@ -50,77 +50,41 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 void pubkey_okx ( struct pubkey_test *test, const char *file,
 		  unsigned int line ) {
 	struct pubkey_algorithm *pubkey = test->pubkey;
-	uint8_t private_ctx[pubkey->ctxsize];
-	uint8_t public_ctx[pubkey->ctxsize];
-	size_t max_len;
-
-	/* Initialize contexts */
-	okx ( pubkey_init ( pubkey, private_ctx, &test->private ) == 0,
-	      file, line );
-	okx ( pubkey_init ( pubkey, public_ctx, &test->public ) == 0,
-	      file, line );
-	max_len = pubkey_max_len ( pubkey, private_ctx );
+	size_t max_len = pubkey_max_len ( pubkey, &test->private );
+	uint8_t encrypted[max_len];
+	uint8_t decrypted[max_len];
+	int encrypted_len;
+	int decrypted_len;
 
 	/* Test decrypting with private key to obtain known plaintext */
-	{
-		uint8_t decrypted[max_len];
-		int decrypted_len;
-
-		decrypted_len = pubkey_decrypt ( pubkey, private_ctx,
-						 test->ciphertext,
-						 test->ciphertext_len,
-						 decrypted );
-		okx ( decrypted_len == ( ( int ) test->plaintext_len ),
-		      file, line );
-		okx ( memcmp ( decrypted, test->plaintext,
-			       test->plaintext_len ) == 0, file, line );
-	}
+	decrypted_len = pubkey_decrypt ( pubkey, &test->private,
+					 test->ciphertext, test->ciphertext_len,
+					 decrypted );
+	okx ( decrypted_len == ( ( int ) test->plaintext_len ), file, line );
+	okx ( memcmp ( decrypted, test->plaintext, test->plaintext_len ) == 0,
+	      file, line );
 
 	/* Test encrypting with private key and decrypting with public key */
-	{
-		uint8_t encrypted[max_len];
-		uint8_t decrypted[max_len];
-		int encrypted_len;
-		int decrypted_len;
-
-		encrypted_len = pubkey_encrypt ( pubkey, private_ctx,
-						 test->plaintext,
-						 test->plaintext_len,
-						 encrypted );
-		okx ( encrypted_len >= 0, file, line );
-		decrypted_len = pubkey_decrypt ( pubkey, public_ctx,
-						 encrypted, encrypted_len,
-						 decrypted );
-		okx ( decrypted_len == ( ( int ) test->plaintext_len ),
-		      file, line );
-		okx ( memcmp ( decrypted, test->plaintext,
-			       test->plaintext_len ) == 0, file, line );
-	}
+	encrypted_len = pubkey_encrypt ( pubkey, &test->private,
+					 test->plaintext, test->plaintext_len,
+					 encrypted );
+	okx ( encrypted_len >= 0, file, line );
+	decrypted_len = pubkey_decrypt ( pubkey, &test->public, encrypted,
+					 encrypted_len, decrypted );
+	okx ( decrypted_len == ( ( int ) test->plaintext_len ), file, line );
+	okx ( memcmp ( decrypted, test->plaintext, test->plaintext_len ) == 0,
+	      file, line );
 
 	/* Test encrypting with public key and decrypting with private key */
-	{
-		uint8_t encrypted[max_len];
-		uint8_t decrypted[max_len];
-		int encrypted_len;
-		int decrypted_len;
-
-		encrypted_len = pubkey_encrypt ( pubkey, public_ctx,
-						 test->plaintext,
-						 test->plaintext_len,
-						 encrypted );
-		okx ( encrypted_len >= 0, file, line );
-		decrypted_len = pubkey_decrypt ( pubkey, private_ctx,
-						 encrypted, encrypted_len,
-						 decrypted );
-		okx ( decrypted_len == ( ( int ) test->plaintext_len ),
-		      file, line );
-		okx ( memcmp ( decrypted, test->plaintext,
-			       test->plaintext_len ) == 0, file, line );
-	}
-
-	/* Free contexts */
-	pubkey_final ( pubkey, public_ctx );
-	pubkey_final ( pubkey, private_ctx );
+	encrypted_len = pubkey_encrypt ( pubkey, &test->public,
+					 test->plaintext, test->plaintext_len,
+					 encrypted );
+	okx ( encrypted_len >= 0, file, line );
+	decrypted_len = pubkey_decrypt ( pubkey, &test->private, encrypted,
+					 encrypted_len, decrypted );
+	okx ( decrypted_len == ( ( int ) test->plaintext_len ), file, line );
+	okx ( memcmp ( decrypted, test->plaintext, test->plaintext_len ) == 0,
+	      file, line );
 }
 
 /**
@@ -134,18 +98,12 @@ void pubkey_sign_okx ( struct pubkey_sign_test *test, const char *file,
 		       unsigned int line ) {
 	struct pubkey_algorithm *pubkey = test->pubkey;
 	struct digest_algorithm *digest = test->digest;
-	uint8_t private_ctx[pubkey->ctxsize];
-	uint8_t public_ctx[pubkey->ctxsize];
+	size_t max_len = pubkey_max_len ( pubkey, &test->private );
+	uint8_t bad[test->signature_len];
 	uint8_t digestctx[digest->ctxsize ];
 	uint8_t digestout[digest->digestsize];
-	size_t max_len;
-
-	/* Initialize contexts */
-	okx ( pubkey_init ( pubkey, private_ctx, &test->private ) == 0,
-	      file, line );
-	okx ( pubkey_init ( pubkey, public_ctx, &test->public ) == 0,
-	      file, line );
-	max_len = pubkey_max_len ( pubkey, private_ctx );
+	uint8_t signature[max_len];
+	int signature_len;
 
 	/* Construct digest over plaintext */
 	digest_init ( digest, digestctx );
@@ -154,34 +112,20 @@ void pubkey_sign_okx ( struct pubkey_sign_test *test, const char *file,
 	digest_final ( digest, digestctx, digestout );
 
 	/* Test signing using private key */
-	{
-		uint8_t signature[max_len];
-		int signature_len;
-
-		signature_len = pubkey_sign ( pubkey, private_ctx, digest,
-					      digestout, signature );
-		okx ( signature_len == ( ( int ) test->signature_len ),
-		      file, line );
-		okx ( memcmp ( signature, test->signature,
-			       test->signature_len ) == 0, file, line );
-	}
+	signature_len = pubkey_sign ( pubkey, &test->private, digest,
+				      digestout, signature );
+	okx ( signature_len == ( ( int ) test->signature_len ), file, line );
+	okx ( memcmp ( signature, test->signature, test->signature_len ) == 0,
+	      file, line );
 
 	/* Test verification using public key */
-	okx ( pubkey_verify ( pubkey, public_ctx, digest, digestout,
+	okx ( pubkey_verify ( pubkey, &test->public, digest, digestout,
 			      test->signature, test->signature_len ) == 0,
 	      file, line );
 
 	/* Test verification failure of modified signature */
-	{
-		uint8_t bad[test->signature_len];
-
-		memcpy ( bad, test->signature, test->signature_len );
-		bad[ test->signature_len / 2 ] ^= 0x40;
-		okx ( pubkey_verify ( pubkey, public_ctx, digest, digestout,
-				      bad, sizeof ( bad ) ) != 0, file, line );
-	}
-
-	/* Free contexts */
-	pubkey_final ( pubkey, public_ctx );
-	pubkey_final ( pubkey, private_ctx );
+	memcpy ( bad, test->signature, test->signature_len );
+	bad[ test->signature_len / 2 ] ^= 0x40;
+	okx ( pubkey_verify ( pubkey, &test->public, digest, digestout,
+			      bad, sizeof ( bad ) ) != 0, file, line );
 }

From bdb5b4aef46ed34b47094652f3eefc7d0463d166 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 23 Aug 2024 12:25:36 +0100
Subject: [PATCH 179/237] [crypto] Hold CMS message as a single ASN.1 object

Reduce the number of dynamic allocations required to parse a CMS
message by retaining the ASN.1 cursor returned from image_asn1() for
the lifetime of the CMS message.  This allows embedded ASN.1 cursors
to be used for parsed objects within the message, such as embedded
signatures.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/cms.c       | 38 ++++++++++++--------------------------
 src/include/ipxe/cms.h |  6 +++---
 2 files changed, 15 insertions(+), 29 deletions(-)

diff --git a/src/crypto/cms.c b/src/crypto/cms.c
index 2e153d819..a596d22e2 100644
--- a/src/crypto/cms.c
+++ b/src/crypto/cms.c
@@ -293,26 +293,18 @@ static int cms_parse_pubkey_algorithm ( struct cms_message *cms,
 static int cms_parse_value ( struct cms_message *cms,
 			     struct cms_participant *part,
 			     const struct asn1_cursor *raw ) {
-	struct asn1_cursor cursor;
 	int rc;
 
 	/* Enter signature */
-	memcpy ( &cursor, raw, sizeof ( cursor ) );
-	if ( ( rc = asn1_enter ( &cursor, ASN1_OCTET_STRING ) ) != 0 ) {
+	memcpy ( &part->value, raw, sizeof ( part->value ) );
+	if ( ( rc = asn1_enter ( &part->value, ASN1_OCTET_STRING ) ) != 0 ) {
 		DBGC ( cms, "CMS %p/%p could not locate value:\n",
 		       cms, part );
 		DBGC_HDA ( cms, 0, raw->data, raw->len );
 		return rc;
 	}
-
-	/* Record signature */
-	part->len = cursor.len;
-	part->value = malloc ( part->len );
-	if ( ! part->value )
-		return -ENOMEM;
-	memcpy ( part->value, cursor.data, part->len );
 	DBGC ( cms, "CMS %p/%p value is:\n", cms, part );
-	DBGC_HDA ( cms, 0, part->value, part->len );
+	DBGC_HDA ( cms, 0, part->value.data, part->value.len );
 
 	return 0;
 }
@@ -456,16 +448,14 @@ static int cms_parse_signed ( struct cms_message *cms,
  * Parse CMS message from ASN.1 data
  *
  * @v cms		CMS message
- * @v raw		ASN.1 cursor
  * @ret rc		Return status code
  */
-static int cms_parse ( struct cms_message *cms,
-		       const struct asn1_cursor *raw ) {
+static int cms_parse ( struct cms_message *cms ) {
 	struct asn1_cursor cursor;
 	int rc;
 
 	/* Enter contentInfo */
-	memcpy ( &cursor, raw, sizeof ( cursor ) );
+	memcpy ( &cursor, cms->raw, sizeof ( cursor ) );
 	asn1_enter ( &cursor, ASN1_SEQUENCE );
 
 	/* Parse contentType */
@@ -497,10 +487,10 @@ static void cms_free ( struct refcnt *refcnt ) {
 	list_for_each_entry_safe ( part, tmp, &cms->participants, list ) {
 		list_del ( &part->list );
 		x509_chain_put ( part->chain );
-		free ( part->value );
 		free ( part );
 	}
 	x509_chain_put ( cms->certificates );
+	free ( cms->raw );
 	free ( cms );
 }
 
@@ -515,7 +505,6 @@ static void cms_free ( struct refcnt *refcnt ) {
  * is responsible for ultimately calling cms_put().
  */
 int cms_message ( struct image *image, struct cms_message **cms ) {
-	struct asn1_cursor *raw;
 	int next;
 	int rc;
 
@@ -529,7 +518,7 @@ int cms_message ( struct image *image, struct cms_message **cms ) {
 	INIT_LIST_HEAD ( &(*cms)->participants );
 
 	/* Get raw message data */
-	next = image_asn1 ( image, 0, &raw );
+	next = image_asn1 ( image, 0, &(*cms)->raw );
 	if ( next < 0 ) {
 		rc = next;
 		DBGC ( *cms, "CMS %p could not get raw ASN.1 data: %s\n",
@@ -538,19 +527,15 @@ int cms_message ( struct image *image, struct cms_message **cms ) {
 	}
 
 	/* Use only first message in image */
-	asn1_shrink_any ( raw );
+	asn1_shrink_any ( (*cms)->raw );
 
 	/* Parse message */
-	if ( ( rc = cms_parse ( *cms, raw ) ) != 0 )
+	if ( ( rc = cms_parse ( *cms ) ) != 0 )
 		goto err_parse;
 
-	/* Free raw message data */
-	free ( raw );
-
 	return 0;
 
  err_parse:
-	free ( raw );
  err_asn1:
 	cms_put ( *cms );
  err_alloc:
@@ -612,7 +597,8 @@ static int cms_verify_digest ( struct cms_message *cms,
 			       userptr_t data, size_t len ) {
 	struct digest_algorithm *digest = part->digest;
 	struct pubkey_algorithm *pubkey = part->pubkey;
-	struct asn1_cursor *key = &cert->subject.public_key.raw;
+	const struct asn1_cursor *key = &cert->subject.public_key.raw;
+	const struct asn1_cursor *value = &part->value;
 	uint8_t digest_out[ digest->digestsize ];
 	int rc;
 
@@ -621,7 +607,7 @@ static int cms_verify_digest ( struct cms_message *cms,
 
 	/* Verify digest */
 	if ( ( rc = pubkey_verify ( pubkey, key, digest, digest_out,
-				    part->value, part->len ) ) != 0 ) {
+				    value->data, value->len ) ) != 0 ) {
 		DBGC ( cms, "CMS %p/%p signature verification failed: %s\n",
 		       cms, part, strerror ( rc ) );
 		return rc;
diff --git a/src/include/ipxe/cms.h b/src/include/ipxe/cms.h
index 1c8a0c587..bffb06bfe 100644
--- a/src/include/ipxe/cms.h
+++ b/src/include/ipxe/cms.h
@@ -48,15 +48,15 @@ struct cms_participant {
 	struct pubkey_algorithm *pubkey;
 
 	/** Signature or key value */
-	void *value;
-	/** Length of signature or key value */
-	size_t len;
+	struct asn1_cursor value;
 };
 
 /** A CMS message */
 struct cms_message {
 	/** Reference count */
 	struct refcnt refcnt;
+	/** Raw ASN.1 data */
+	struct asn1_cursor *raw;
 	/** Message type */
 	struct cms_type *type;
 

From 4b4a362f0713ffa08c9236d66a1881d6234d7717 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 26 Aug 2024 23:36:06 +0100
Subject: [PATCH 180/237] [crypto] Allow for extraction of ASN.1 algorithm
 parameters

Some ASN.1 OID-identified algorithms require additional parameters,
such as an initialisation vector for a block cipher.  The structure of
the parameters is defined by the individual algorithm.

Extend asn1_algorithm() to allow these additional parameters to be
returned via a separate ASN.1 cursor.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/asn1.c                 | 78 +++++++++++++++++++++++++++----
 src/crypto/mishmash/oid_aes_cbc.c |  3 ++
 src/crypto/mishmash/oid_aes_gcm.c |  3 ++
 src/include/ipxe/asn1.h           | 19 +++++++-
 4 files changed, 92 insertions(+), 11 deletions(-)

diff --git a/src/crypto/asn1.c b/src/crypto/asn1.c
index 0e5b1a9dd..aa57c6a8b 100644
--- a/src/crypto/asn1.c
+++ b/src/crypto/asn1.c
@@ -32,6 +32,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <time.h>
 #include <ipxe/tables.h>
 #include <ipxe/image.h>
+#include <ipxe/crypto.h>
 #include <ipxe/asn1.h>
 
 /** @file
@@ -509,18 +510,26 @@ asn1_find_algorithm ( const struct asn1_cursor *cursor ) {
  *
  * @v cursor		ASN.1 object cursor
  * @ret algorithm	Algorithm
+ * @ret params		Algorithm parameters, or NULL
  * @ret rc		Return status code
  */
 int asn1_algorithm ( const struct asn1_cursor *cursor,
-		     struct asn1_algorithm **algorithm ) {
+		     struct asn1_algorithm **algorithm,
+		     struct asn1_cursor *params ) {
 	struct asn1_cursor contents;
 	int rc;
 
-	/* Enter signatureAlgorithm */
+	/* Enter algorithm */
 	memcpy ( &contents, cursor, sizeof ( contents ) );
 	asn1_enter ( &contents, ASN1_SEQUENCE );
 
-	/* Enter algorithm */
+	/* Get raw parameters, if applicable */
+	if ( params ) {
+		memcpy ( params, &contents, sizeof ( *params ) );
+		asn1_skip_any ( params );
+	}
+
+	/* Enter algorithm identifier */
 	if ( ( rc = asn1_enter ( &contents, ASN1_OID ) ) != 0 ) {
 		DBGC ( cursor, "ASN1 %p cannot locate algorithm OID:\n",
 		       cursor );
@@ -536,6 +545,14 @@ int asn1_algorithm ( const struct asn1_cursor *cursor,
 		return -ENOTSUP_ALGORITHM;
 	}
 
+	/* Parse parameters, if applicable */
+	if ( params && (*algorithm)->parse &&
+	     ( ( rc = (*algorithm)->parse ( *algorithm, params ) ) != 0 ) ) {
+		DBGC ( cursor, "ASN1 %p cannot parse %s parameters: %s\n",
+		       cursor, (*algorithm)->name, strerror ( rc ) );
+		return rc;
+	}
+
 	return 0;
 }
 
@@ -551,7 +568,7 @@ int asn1_pubkey_algorithm ( const struct asn1_cursor *cursor,
 	int rc;
 
 	/* Parse algorithm */
-	if ( ( rc = asn1_algorithm ( cursor, algorithm ) ) != 0 )
+	if ( ( rc = asn1_algorithm ( cursor, algorithm, NULL ) ) != 0 )
 		return rc;
 
 	/* Check algorithm has a public key */
@@ -577,7 +594,7 @@ int asn1_digest_algorithm ( const struct asn1_cursor *cursor,
 	int rc;
 
 	/* Parse algorithm */
-	if ( ( rc = asn1_algorithm ( cursor, algorithm ) ) != 0 )
+	if ( ( rc = asn1_algorithm ( cursor, algorithm, NULL ) ) != 0 )
 		return rc;
 
 	/* Check algorithm has a digest */
@@ -596,14 +613,16 @@ int asn1_digest_algorithm ( const struct asn1_cursor *cursor,
  *
  * @v cursor		ASN.1 object cursor
  * @ret algorithm	Algorithm
+ * @ret params		Algorithm parameters, or NULL
  * @ret rc		Return status code
  */
 int asn1_cipher_algorithm ( const struct asn1_cursor *cursor,
-			    struct asn1_algorithm **algorithm ) {
+			    struct asn1_algorithm **algorithm,
+			    struct asn1_cursor *params ) {
 	int rc;
 
 	/* Parse algorithm */
-	if ( ( rc = asn1_algorithm ( cursor, algorithm ) ) != 0 )
+	if ( ( rc = asn1_algorithm ( cursor, algorithm, params ) ) != 0 )
 		return rc;
 
 	/* Check algorithm has a cipher */
@@ -629,7 +648,7 @@ int asn1_signature_algorithm ( const struct asn1_cursor *cursor,
 	int rc;
 
 	/* Parse algorithm */
-	if ( ( rc = asn1_algorithm ( cursor, algorithm ) ) != 0 )
+	if ( ( rc = asn1_algorithm ( cursor, algorithm, NULL ) ) != 0 )
 		return rc;
 
 	/* Check algorithm has a public key */
@@ -664,7 +683,7 @@ int asn1_check_algorithm ( const struct asn1_cursor *cursor,
 	int rc;
 
 	/* Parse algorithm */
-	if ( ( rc = asn1_algorithm ( cursor, &actual ) ) != 0 )
+	if ( ( rc = asn1_algorithm ( cursor, &actual, NULL ) ) != 0 )
 		return rc;
 
 	/* Check algorithm matches */
@@ -677,6 +696,47 @@ int asn1_check_algorithm ( const struct asn1_cursor *cursor,
 	return 0;
 }
 
+/**
+ * Parse ASN.1 CBC cipher parameters
+ *
+ * @v algorithm		Algorithm
+ * @v param		Parameters to parse
+ * @ret rc		Return status code
+ */
+int asn1_parse_cbc ( struct asn1_algorithm *algorithm,
+		     struct asn1_cursor *params ) {
+	struct cipher_algorithm *cipher = algorithm->cipher;
+
+	/* Sanity check */
+	assert ( cipher != NULL );
+
+	/* Enter parameters */
+	asn1_enter ( params, ASN1_OCTET_STRING );
+
+	/* Check length */
+	if ( params->len != cipher->blocksize )
+		return -EINVAL;
+
+	return 0;
+}
+
+/**
+ * Parse ASN.1 GCM cipher parameters
+ *
+ * @v algorithm		Algorithm
+ * @v param		Parameters to parse
+ * @ret rc		Return status code
+ */
+int asn1_parse_gcm ( struct asn1_algorithm *algorithm __unused,
+		     struct asn1_cursor *params ) {
+
+	/* Enter parameters */
+	asn1_enter ( params, ASN1_SEQUENCE );
+
+	/* Enter nonce */
+	return asn1_enter ( params, ASN1_OCTET_STRING );
+}
+
 /**
  * Parse ASN.1 GeneralizedTime
  *
diff --git a/src/crypto/mishmash/oid_aes_cbc.c b/src/crypto/mishmash/oid_aes_cbc.c
index 0a7951c63..b5f716574 100644
--- a/src/crypto/mishmash/oid_aes_cbc.c
+++ b/src/crypto/mishmash/oid_aes_cbc.c
@@ -40,6 +40,7 @@ struct asn1_algorithm aes_128_cbc_algorithm __asn1_algorithm = {
 	.name = "aes128-cbc",
 	.cipher = &aes_cbc_algorithm,
 	.oid = ASN1_CURSOR ( oid_aes_128_cbc ),
+	.parse = asn1_parse_cbc,
 };
 
 /** "aes192-cbc" OID-identified algorithm */
@@ -47,6 +48,7 @@ struct asn1_algorithm aes_192_cbc_algorithm __asn1_algorithm = {
 	.name = "aes192-cbc",
 	.cipher = &aes_cbc_algorithm,
 	.oid = ASN1_CURSOR ( oid_aes_192_cbc ),
+	.parse = asn1_parse_cbc,
 };
 
 /** "aes256-cbc" OID-identified algorithm */
@@ -54,4 +56,5 @@ struct asn1_algorithm aes_256_cbc_algorithm __asn1_algorithm = {
 	.name = "aes256-cbc",
 	.cipher = &aes_cbc_algorithm,
 	.oid = ASN1_CURSOR ( oid_aes_256_cbc ),
+	.parse = asn1_parse_cbc,
 };
diff --git a/src/crypto/mishmash/oid_aes_gcm.c b/src/crypto/mishmash/oid_aes_gcm.c
index 0f94d5323..af1432d8e 100644
--- a/src/crypto/mishmash/oid_aes_gcm.c
+++ b/src/crypto/mishmash/oid_aes_gcm.c
@@ -40,6 +40,7 @@ struct asn1_algorithm aes_128_gcm_algorithm __asn1_algorithm = {
 	.name = "aes128-gcm",
 	.cipher = &aes_gcm_algorithm,
 	.oid = ASN1_CURSOR ( oid_aes_128_gcm ),
+	.parse = asn1_parse_gcm,
 };
 
 /** "aes192-gcm" OID-identified algorithm */
@@ -47,6 +48,7 @@ struct asn1_algorithm aes_192_gcm_algorithm __asn1_algorithm = {
 	.name = "aes192-gcm",
 	.cipher = &aes_gcm_algorithm,
 	.oid = ASN1_CURSOR ( oid_aes_192_gcm ),
+	.parse = asn1_parse_gcm,
 };
 
 /** "aes256-gcm" OID-identified algorithm */
@@ -54,4 +56,5 @@ struct asn1_algorithm aes_256_gcm_algorithm __asn1_algorithm = {
 	.name = "aes256-gcm",
 	.cipher = &aes_gcm_algorithm,
 	.oid = ASN1_CURSOR ( oid_aes_256_gcm ),
+	.parse = asn1_parse_gcm,
 };
diff --git a/src/include/ipxe/asn1.h b/src/include/ipxe/asn1.h
index 26dc47992..fac94c52e 100644
--- a/src/include/ipxe/asn1.h
+++ b/src/include/ipxe/asn1.h
@@ -363,6 +363,15 @@ struct asn1_algorithm {
 	struct cipher_algorithm *cipher;
 	/** Elliptic curve (if applicable) */
 	struct elliptic_curve *curve;
+	/**
+	 * Parse algorithm parameters (optional)
+	 *
+	 * @v algorithm		Algorithm
+	 * @v param		Parameters to parse (and potentially modify)
+	 * @ret rc		Return status code
+	 */
+	int ( * parse ) ( struct asn1_algorithm *algorithm,
+			  struct asn1_cursor *params );
 };
 
 /** ASN.1 OID-identified algorithms */
@@ -467,17 +476,23 @@ extern int asn1_integral_bit_string ( const struct asn1_cursor *cursor,
 extern int asn1_compare ( const struct asn1_cursor *cursor1,
 			  const struct asn1_cursor *cursor2 );
 extern int asn1_algorithm ( const struct asn1_cursor *cursor,
-			    struct asn1_algorithm **algorithm );
+			    struct asn1_algorithm **algorithm,
+			    struct asn1_cursor *params );
 extern int asn1_pubkey_algorithm ( const struct asn1_cursor *cursor,
 				   struct asn1_algorithm **algorithm );
 extern int asn1_digest_algorithm ( const struct asn1_cursor *cursor,
 				   struct asn1_algorithm **algorithm );
 extern int asn1_cipher_algorithm ( const struct asn1_cursor *cursor,
-				   struct asn1_algorithm **algorithm );
+				   struct asn1_algorithm **algorithm,
+				   struct asn1_cursor *params );
 extern int asn1_signature_algorithm ( const struct asn1_cursor *cursor,
 				      struct asn1_algorithm **algorithm );
 extern int asn1_check_algorithm ( const struct asn1_cursor *cursor,
 				  struct asn1_algorithm *expected );
+extern int asn1_parse_cbc ( struct asn1_algorithm *algorithm,
+			    struct asn1_cursor *params );
+extern int asn1_parse_gcm ( struct asn1_algorithm *algorithm,
+			    struct asn1_cursor *params );
 extern int asn1_generalized_time ( const struct asn1_cursor *cursor,
 				   time_t *time );
 extern int asn1_grow ( struct asn1_builder *builder, size_t extra );

From b053ba19884415794504c1bcffa910f0295598f0 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 26 Aug 2024 15:15:09 +0100
Subject: [PATCH 181/237] [test] Update CMS self-test terminology

Generalise CMS self-test data structure and macro names to refer to
"messages" rather than "signatures", in preparation for adding image
decryption tests.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/tests/cms_test.c | 117 +++++++++++++++++++++----------------------
 1 file changed, 58 insertions(+), 59 deletions(-)

diff --git a/src/tests/cms_test.c b/src/tests/cms_test.c
index 86f9bb98f..2480263e1 100644
--- a/src/tests/cms_test.c
+++ b/src/tests/cms_test.c
@@ -45,15 +45,15 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** Fingerprint algorithm used for X.509 test certificates */
 #define cms_test_algorithm sha256_algorithm
 
-/** CMS test code blob */
-struct cms_test_code {
-	/** Code image */
+/** Test image */
+struct cms_test_image {
+	/** Image */
 	struct image image;
 };
 
-/** CMS test signature */
-struct cms_test_signature {
-	/** Signature image */
+/** Test CMS message */
+struct cms_test_message {
+	/** Message image */
 	struct image image;
 	/** Parsed message */
 	struct cms_message *cms;
@@ -65,23 +65,22 @@ struct cms_test_signature {
 /** Define inline fingerprint data */
 #define FINGERPRINT(...) { __VA_ARGS__ }
 
-/** Define a test code blob */
-#define SIGNED_CODE( NAME, DATA )					\
+/** Define a test image */
+#define IMAGE( NAME, DATA )						\
 	static const uint8_t NAME ## _data[] = DATA;			\
-	static struct cms_test_code NAME = {				\
+	static struct cms_test_image NAME = {				\
 		.image = {						\
 			.refcnt = REF_INIT ( ref_no_free ),		\
 			.name = #NAME,					\
-			.type = &der_image_type,			\
 			.data = ( userptr_t ) ( NAME ## _data ),	\
 			.len = sizeof ( NAME ## _data ),		\
 		},							\
 	}
 
-/** Define a test signature */
-#define SIGNATURE( NAME, DATA )						\
+/** Define a test message */
+#define MESSAGE( NAME, DATA )						\
 	static const uint8_t NAME ## _data[] = DATA;			\
-	static struct cms_test_signature NAME = {			\
+	static struct cms_test_message NAME = {			\
 		.image = {						\
 			.refcnt = REF_INIT ( ref_no_free ),		\
 			.name = #NAME,					\
@@ -92,7 +91,7 @@ struct cms_test_signature {
 	}
 
 /** Code that has been signed */
-SIGNED_CODE ( test_code,
+IMAGE ( test_code,
 	DATA ( 0x23, 0x21, 0x69, 0x70, 0x78, 0x65, 0x0a, 0x0a, 0x65, 0x63,
 	       0x68, 0x6f, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73,
 	       0x20, 0x61, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x20,
@@ -104,7 +103,7 @@ SIGNED_CODE ( test_code,
 	       0x65, 0x6c, 0x6c, 0x0a ) );
 
 /** Code that has not been signed */
-SIGNED_CODE ( bad_code,
+IMAGE ( bad_code,
 	DATA ( 0x23, 0x21, 0x69, 0x70, 0x78, 0x65, 0x0a, 0x0a, 0x65, 0x63,
 	       0x68, 0x6f, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73,
 	       0x20, 0x61, 0x20, 0x6d, 0x61, 0x6c, 0x69, 0x63, 0x69, 0x6f,
@@ -115,7 +114,7 @@ SIGNED_CODE ( bad_code,
 	       0x68, 0x65, 0x6c, 0x6c, 0x0a ) );
 
 /** Valid signature */
-SIGNATURE ( codesigned_sig,
+MESSAGE ( codesigned_sig,
 	DATA ( 0x30, 0x82, 0x0c, 0x41, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86,
 	       0xf7, 0x0d, 0x01, 0x07, 0x02, 0xa0, 0x82, 0x0c, 0x32, 0x30,
 	       0x82, 0x0c, 0x2e, 0x02, 0x01, 0x01, 0x31, 0x09, 0x30, 0x07,
@@ -433,7 +432,7 @@ SIGNATURE ( codesigned_sig,
 	       0xbf ) );
 
 /** Signature with a broken certificate chain */
-SIGNATURE ( brokenchain_sig,
+MESSAGE ( brokenchain_sig,
 	DATA ( 0x30, 0x82, 0x09, 0x8a, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86,
 	       0xf7, 0x0d, 0x01, 0x07, 0x02, 0xa0, 0x82, 0x09, 0x7b, 0x30,
 	       0x82, 0x09, 0x77, 0x02, 0x01, 0x01, 0x31, 0x09, 0x30, 0x07,
@@ -681,7 +680,7 @@ SIGNATURE ( brokenchain_sig,
 	       0xf9, 0x71, 0x64, 0x03, 0x05, 0xbf ) );
 
 /** Signature generated with a non-code-signing certificate */
-SIGNATURE ( genericsigned_sig,
+MESSAGE ( genericsigned_sig,
 	DATA ( 0x30, 0x82, 0x0c, 0x2f, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86,
 	       0xf7, 0x0d, 0x01, 0x07, 0x02, 0xa0, 0x82, 0x0c, 0x20, 0x30,
 	       0x82, 0x0c, 0x1c, 0x02, 0x01, 0x01, 0x31, 0x09, 0x30, 0x07,
@@ -997,7 +996,7 @@ SIGNATURE ( genericsigned_sig,
 	       0x7e, 0x7c, 0x99 ) );
 
 /** Signature generated with a non-signing certificate */
-SIGNATURE ( nonsigned_sig,
+MESSAGE ( nonsigned_sig,
 	DATA ( 0x30, 0x82, 0x0c, 0x12, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86,
 	       0xf7, 0x0d, 0x01, 0x07, 0x02, 0xa0, 0x82, 0x0c, 0x03, 0x30,
 	       0x82, 0x0b, 0xff, 0x02, 0x01, 0x01, 0x31, 0x09, 0x30, 0x07,
@@ -1352,33 +1351,33 @@ static time_t test_time = 1332374737ULL; /* Thu Mar 22 00:05:37 2012 */
 static time_t test_expired = 1375573111ULL; /* Sat Aug  3 23:38:31 2013 */
 
 /**
- * Report signature parsing test result
+ * Report message parsing test result
  *
- * @v sgn		Test signature
+ * @v msg		Test message
  * @v file		Test code file
  * @v line		Test code line
  */
-static void cms_signature_okx ( struct cms_test_signature *sgn,
-				const char *file, unsigned int line ) {
-	const void *data = ( ( void * ) sgn->image.data );
+static void cms_message_okx ( struct cms_test_message *msg,
+			      const char *file, unsigned int line ) {
+	const void *data = ( ( void * ) msg->image.data );
 
 	/* Fix up image data pointer */
-	sgn->image.data = virt_to_user ( data );
+	msg->image.data = virt_to_user ( data );
 
-	/* Check ability to parse signature */
-	okx ( cms_message ( &sgn->image, &sgn->cms ) == 0, file, line );
+	/* Check ability to parse message */
+	okx ( cms_message ( &msg->image, &msg->cms ) == 0, file, line );
 
 	/* Reset image data pointer */
-	sgn->image.data = ( ( userptr_t ) data );
+	msg->image.data = ( ( userptr_t ) data );
 }
-#define cms_signature_ok( sgn ) \
-	cms_signature_okx ( sgn, __FILE__, __LINE__ )
+#define cms_message_ok( msg ) \
+	cms_message_okx ( msg, __FILE__, __LINE__ )
 
 /**
  * Report signature verification test result
  *
- * @v sgn		Test signature
- * @v code		Test signed code
+ * @v msg		Test signature message
+ * @v img		Test signed image
  * @v name		Test verification name
  * @v time		Test verification time
  * @v store		Test certificate store
@@ -1386,36 +1385,36 @@ static void cms_signature_okx ( struct cms_test_signature *sgn,
  * @v file		Test code file
  * @v line		Test code line
  */
-static void cms_verify_okx ( struct cms_test_signature *sgn,
-			     struct cms_test_code *code, const char *name,
+static void cms_verify_okx ( struct cms_test_message *msg,
+			     struct cms_test_image *img, const char *name,
 			     time_t time, struct x509_chain *store,
 			     struct x509_root *root, const char *file,
 			     unsigned int line ) {
-	const void *data = ( ( void * ) code->image.data );
+	const void *data = ( ( void * ) img->image.data );
 
 	/* Fix up image data pointer */
-	code->image.data = virt_to_user ( data );
+	img->image.data = virt_to_user ( data );
 
 	/* Invalidate any certificates from previous tests */
-	x509_invalidate_chain ( sgn->cms->certificates );
+	x509_invalidate_chain ( msg->cms->certificates );
 
 	/* Check ability to verify signature */
-	okx ( cms_verify ( sgn->cms, &code->image, name, time, store,
+	okx ( cms_verify ( msg->cms, &img->image, name, time, store,
 			   root ) == 0, file, line );
-	okx ( code->image.flags & IMAGE_TRUSTED, file, line );
+	okx ( img->image.flags & IMAGE_TRUSTED, file, line );
 
 	/* Reset image data pointer */
-	code->image.data = ( ( userptr_t ) data );
+	img->image.data = ( ( userptr_t ) data );
 }
-#define cms_verify_ok( sgn, code, name, time, store, root )		\
-	cms_verify_okx ( sgn, code, name, time, store, root,		\
+#define cms_verify_ok( msg, img, name, time, store, root )		\
+	cms_verify_okx ( msg, img, name, time, store, root,		\
 			 __FILE__, __LINE__ )
 
 /**
  * Report signature verification failure test result
  *
- * @v sgn		Test signature
- * @v code		Test signed code
+ * @v msg		Test signature message
+ * @v img		Test signed image
  * @v name		Test verification name
  * @v time		Test verification time
  * @v store		Test certificate store
@@ -1423,29 +1422,29 @@ static void cms_verify_okx ( struct cms_test_signature *sgn,
  * @v file		Test code file
  * @v line		Test code line
  */
-static void cms_verify_fail_okx ( struct cms_test_signature *sgn,
-				  struct cms_test_code *code, const char *name,
+static void cms_verify_fail_okx ( struct cms_test_message *msg,
+				  struct cms_test_image *img, const char *name,
 				  time_t time, struct x509_chain *store,
 				  struct x509_root *root, const char *file,
 				  unsigned int line ) {
-	const void *data = ( ( void * ) code->image.data );
+	const void *data = ( ( void * ) img->image.data );
 
 	/* Fix up image data pointer */
-	code->image.data = virt_to_user ( data );
+	img->image.data = virt_to_user ( data );
 
 	/* Invalidate any certificates from previous tests */
-	x509_invalidate_chain ( sgn->cms->certificates );
+	x509_invalidate_chain ( msg->cms->certificates );
 
 	/* Check inability to verify signature */
-	okx ( cms_verify ( sgn->cms, &code->image, name, time, store,
+	okx ( cms_verify ( msg->cms, &img->image, name, time, store,
 			   root ) != 0, file, line );
-	okx ( ! ( code->image.flags & IMAGE_TRUSTED ), file, line );
+	okx ( ! ( img->image.flags & IMAGE_TRUSTED ), file, line );
 
 	/* Reset image data pointer */
-	code->image.data = ( ( userptr_t ) data );
+	img->image.data = ( ( userptr_t ) data );
 }
-#define cms_verify_fail_ok( sgn, code, name, time, store, root )	\
-	cms_verify_fail_okx ( sgn, code, name, time, store, root,	\
+#define cms_verify_fail_ok( msg, img, name, time, store, root )	\
+	cms_verify_fail_okx ( msg, img, name, time, store, root,	\
 			      __FILE__, __LINE__ )
 
 /**
@@ -1454,11 +1453,11 @@ static void cms_verify_fail_okx ( struct cms_test_signature *sgn,
  */
 static void cms_test_exec ( void ) {
 
-	/* Check that all signatures can be parsed */
-	cms_signature_ok ( &codesigned_sig );
-	cms_signature_ok ( &brokenchain_sig );
-	cms_signature_ok ( &genericsigned_sig );
-	cms_signature_ok ( &nonsigned_sig );
+	/* Check that all messages can be parsed */
+	cms_message_ok ( &codesigned_sig );
+	cms_message_ok ( &brokenchain_sig );
+	cms_message_ok ( &genericsigned_sig );
+	cms_message_ok ( &nonsigned_sig );
 
 	/* Check good signature */
 	cms_verify_ok ( &codesigned_sig, &test_code, "codesign.test.ipxe.org",

From 748cab7745186ec6c770fb4d47b0e8c9f213e6df Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 28 Aug 2024 16:09:55 +0100
Subject: [PATCH 182/237] [crypto] Add cmsdetach script for detaching encrypted
 data from CMS messages

The openssl toolchain does not currently seem to support creating CMS
envelopedData or authEnvelopedData messages with detached encrypted
data.

Add a standalone tool "cmsdetach" that can be used to detach the
encrypted data from a CMS message.  For example:

  openssl cms -encrypt -binary -aes-256-gcm -recip client.crt \
              -in bootfile -outform DER -out bootfile.cms

  cmsdetach bootfile.cms --data bootfile.dat --envelope bootfile.env

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 contrib/crypto/cmsdetach | 80 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100755 contrib/crypto/cmsdetach

diff --git a/contrib/crypto/cmsdetach b/contrib/crypto/cmsdetach
new file mode 100755
index 000000000..007500996
--- /dev/null
+++ b/contrib/crypto/cmsdetach
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+"""Detach CMS encrypted data.
+
+Detach encrypted data from a CMS envelopedData or authEnvelopedData
+message into a separate file.
+"""
+
+import argparse
+
+import asn1
+
+# Parse command-line arguments
+#
+parser = argparse.ArgumentParser(
+    description=__doc__,
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+)
+parser.add_argument("-d", "--data", metavar="FILE",
+                    help="Write detached data (without envelope) to FILE")
+parser.add_argument("-e", "--envelope", metavar="FILE",
+                    help="Write envelope (without data) to FILE")
+parser.add_argument("-o", "--overwrite", action="store_true",
+                    help="Overwrite output files")
+parser.add_argument("file", help="Input envelope file")
+args = parser.parse_args()
+if args.data is None and args.envelope is None:
+    parser.error("at least one of --data and --envelope is required")
+outmode = "wb" if args.overwrite else "xb"
+
+# Create decoder
+#
+decoder = asn1.Decoder()
+with open(args.file, mode="rb") as fh:
+    decoder.start(fh.read())
+
+# Create encoder
+#
+encoder = asn1.Encoder()
+encoder.start()
+
+# Detach encrypted data
+#
+data = None
+datastack = [
+    asn1.Numbers.Sequence, 0, asn1.Numbers.Sequence, asn1.Numbers.Sequence
+]
+stack = []
+while stack or not decoder.eof():
+    if decoder.eof():
+        encoder.leave()
+        decoder.leave()
+        stack.pop()
+    else:
+        tag = decoder.peek()
+        if tag.typ == asn1.Types.Constructed:
+            encoder.enter(nr=tag.nr, cls=tag.cls)
+            decoder.enter()
+            stack.append(tag.nr)
+        else:
+            (tag, value) = decoder.read()
+            if stack == datastack and tag.nr == 0:
+                data = value
+            else:
+                encoder.write(value, nr=tag.nr, cls=tag.cls)
+envelope = encoder.output()
+if data is None:
+    parser.error("Input file does not contain any encrypted data")
+
+# Write envelope (without data), if applicable
+#
+if args.envelope:
+    with open(args.envelope, mode=outmode) as fh:
+        fh.write(envelope)
+
+# Write data (without envelope), if applicable
+#
+if args.data:
+    with open(args.data, mode=outmode) as fh:
+        fh.write(data)

From 49404bfea99f68f4c364ea30a9ad3ea6ffb7e5f6 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 29 Aug 2024 13:08:27 +0100
Subject: [PATCH 183/237] [image] Split image_strip_suffix() out from
 image_extract()

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/archive.c       |  7 ++-----
 src/core/image.c         | 19 +++++++++++++++++++
 src/include/ipxe/image.h |  1 +
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/core/archive.c b/src/core/archive.c
index bb62c7e47..3d3f00cc6 100644
--- a/src/core/archive.c
+++ b/src/core/archive.c
@@ -43,7 +43,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 int image_extract ( struct image *image, const char *name,
 		    struct image **extracted ) {
-	char *dot;
 	int rc;
 
 	/* Check that this image can be used to extract an archive image */
@@ -66,10 +65,8 @@ int image_extract ( struct image *image, const char *name,
 	}
 
 	/* Strip any archive or compression suffix from implicit name */
-	if ( ( ! name ) && ( (*extracted)->name ) &&
-	     ( ( dot = strrchr ( (*extracted)->name, '.' ) ) != NULL ) ) {
-		*dot = '\0';
-	}
+	if ( ! name )
+		image_strip_suffix ( *extracted );
 
 	/* Try extracting archive image */
 	if ( ( rc = image->type->extract ( image, *extracted ) ) != 0 ) {
diff --git a/src/core/image.c b/src/core/image.c
index bf0e4f756..c69c05c93 100644
--- a/src/core/image.c
+++ b/src/core/image.c
@@ -172,6 +172,25 @@ int image_set_name ( struct image *image, const char *name ) {
 	return 0;
 }
 
+/**
+ * Strip dot suffix from image name, if present
+ *
+ * @v image		Image
+ * @ret sep		Position of old dot separator, or NULL
+ */
+char * image_strip_suffix ( struct image *image ) {
+	char *dot;
+
+	/* Locate and strip suffix, if present */
+	if ( image->name &&
+	     ( ( dot = strrchr ( image->name, '.' ) ) != NULL ) ) {
+		*dot = '\0';
+		return dot;
+	}
+
+	return NULL;
+}
+
 /**
  * Set image command line
  *
diff --git a/src/include/ipxe/image.h b/src/include/ipxe/image.h
index bfbf23687..ca383dc7b 100644
--- a/src/include/ipxe/image.h
+++ b/src/include/ipxe/image.h
@@ -188,6 +188,7 @@ static inline struct image * first_image ( void ) {
 extern struct image * alloc_image ( struct uri *uri );
 extern int image_set_uri ( struct image *image, struct uri *uri );
 extern int image_set_name ( struct image *image, const char *name );
+extern char * image_strip_suffix ( struct image *image );
 extern int image_set_cmdline ( struct image *image, const char *cmdline );
 extern int image_set_len ( struct image *image, size_t len );
 extern int image_set_data ( struct image *image, userptr_t data, size_t len );

From 486b15b3c11692af4325cd7d0220cdb72ec27586 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 23 Aug 2024 12:28:21 +0100
Subject: [PATCH 184/237] [crypto] Support decryption of images via CMS
 envelopes

Add support for decrypting images containing detached encrypted data
using a cipher key obtained from a separate CMS envelope image (in DER
or PEM format).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/cms.c        | 511 ++++++++++++++++++++++++++++++++++++++--
 src/include/ipxe/asn1.h |  13 +
 src/include/ipxe/cms.h  |  22 ++
 3 files changed, 529 insertions(+), 17 deletions(-)

diff --git a/src/crypto/cms.c b/src/crypto/cms.c
index a596d22e2..3a5debfcf 100644
--- a/src/crypto/cms.c
+++ b/src/crypto/cms.c
@@ -40,6 +40,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/image.h>
 #include <ipxe/malloc.h>
 #include <ipxe/uaccess.h>
+#include <ipxe/privkey.h>
 #include <ipxe/cms.h>
 
 /* Disambiguate the various error causes */
@@ -59,17 +60,48 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	__einfo_error ( EINFO_EACCES_NO_SIGNATURES )
 #define EINFO_EACCES_NO_SIGNATURES \
 	__einfo_uniqify ( EINFO_EACCES, 0x05, "No signatures present" )
+#define EACCES_NO_RECIPIENTS \
+	__einfo_error ( EINFO_EACCES_NO_RECIPIENTS )
+#define EINFO_EACCES_NO_RECIPIENTS \
+	__einfo_uniqify ( EINFO_EACCES, 0x06, "No usable recipients" )
+#define EACCES_LEN \
+	__einfo_error ( EINFO_EACCES_LEN )
+#define EINFO_EACCES_LEN \
+	__einfo_uniqify ( EINFO_EACCES, 0x07, "Bad file length" )
+#define EACCES_PAD \
+	__einfo_error ( EINFO_EACCES_PAD )
+#define EINFO_EACCES_PAD \
+	__einfo_uniqify ( EINFO_EACCES, 0x08, "Bad block padding" )
+#define EACCES_MAC \
+	__einfo_error ( EINFO_EACCES_MAC )
+#define EINFO_EACCES_MAC \
+	__einfo_uniqify ( EINFO_EACCES, 0x09, "Invalid MAC" )
 #define ENOTSUP_TYPE \
 	__einfo_error ( EINFO_ENOTSUP_TYPE )
 #define EINFO_ENOTSUP_TYPE \
 	__einfo_uniqify ( EINFO_ENOTSUP, 0x01, "Unrecognised message type" )
 
+/** Buffer size for decryption
+ *
+ * Must be at least 256 to allow block padding to be removed if
+ * needed.
+ */
+#define CMS_DECRYPT_BLKSZ 2048
+
 static int cms_parse_signed ( struct cms_message *cms,
 			      const struct asn1_cursor *raw );
+static int cms_parse_enveloped ( struct cms_message *cms,
+				 const struct asn1_cursor *raw );
 
 /** "id-signedData" object identifier */
 static uint8_t oid_signeddata[] = { ASN1_OID_SIGNEDDATA };
 
+/** "id-envelopedData" object identifier */
+static uint8_t oid_envelopeddata[] = { ASN1_OID_ENVELOPEDDATA };
+
+/** "id-authEnvelopedData" object identifier */
+static uint8_t oid_authenvelopeddata[] = { ASN1_OID_AUTHENVELOPEDDATA };
+
 /** CMS message types */
 static struct cms_type cms_types[] = {
 	{
@@ -77,6 +109,16 @@ static struct cms_type cms_types[] = {
 		.oid = ASN1_CURSOR ( oid_signeddata ),
 		.parse = cms_parse_signed,
 	},
+	{
+		.name = "enveloped",
+		.oid = ASN1_CURSOR ( oid_envelopeddata ),
+		.parse = cms_parse_enveloped,
+	},
+	{
+		.name = "authEnveloped",
+		.oid = ASN1_CURSOR ( oid_authenvelopeddata ),
+		.parse = cms_parse_enveloped,
+	}
 };
 
 /**
@@ -201,7 +243,7 @@ static int cms_parse_identifier ( struct cms_message *cms,
 	if ( ! cert ) {
 		DBGC ( cms, "CMS %p/%p could not identify certificate\n",
 		       cms, part );
-		return -ENOENT;
+		return ( cms_is_signature ( cms ) ? -ENOENT : 0 );
 	}
 
 	/* Append certificate to chain */
@@ -282,6 +324,34 @@ static int cms_parse_pubkey_algorithm ( struct cms_message *cms,
 	return 0;
 }
 
+/**
+ * Parse CMS message cipher algorithm
+ *
+ * @v cms		CMS message
+ * @v raw		ASN.1 cursor
+ * @ret rc		Return status code
+ */
+static int cms_parse_cipher_algorithm ( struct cms_message *cms,
+					const struct asn1_cursor *raw ) {
+	struct asn1_algorithm *algorithm;
+	int rc;
+
+	/* Identify algorithm */
+	if ( ( rc = asn1_cipher_algorithm ( raw, &algorithm,
+					    &cms->iv ) ) != 0 ) {
+		DBGC ( cms, "CMS %p could not identify cipher algorithm: %s\n",
+		       cms, strerror ( rc ) );
+		DBGC_HDA ( cms, 0, raw->data, raw->len );
+		return rc;
+	}
+
+	/* Record cipher */
+	cms->cipher = algorithm->cipher;
+	DBGC ( cms, "CMS %p cipher algorithm is %s\n", cms, algorithm->name );
+
+	return 0;
+}
+
 /**
  * Parse CMS message signature or key value
  *
@@ -295,7 +365,7 @@ static int cms_parse_value ( struct cms_message *cms,
 			     const struct asn1_cursor *raw ) {
 	int rc;
 
-	/* Enter signature */
+	/* Enter signature or encryptedKey */
 	memcpy ( &part->value, raw, sizeof ( part->value ) );
 	if ( ( rc = asn1_enter ( &part->value, ASN1_OCTET_STRING ) ) != 0 ) {
 		DBGC ( cms, "CMS %p/%p could not locate value:\n",
@@ -323,33 +393,37 @@ static int cms_parse_participant ( struct cms_message *cms,
 	struct asn1_cursor cursor;
 	int rc;
 
-	/* Enter signerInfo */
+	/* Enter signerInfo or ktri */
 	memcpy ( &cursor, raw, sizeof ( cursor ) );
 	asn1_enter ( &cursor, ASN1_SEQUENCE );
 
 	/* Skip version */
 	asn1_skip ( &cursor, ASN1_INTEGER );
 
-	/* Parse sid */
+	/* Parse sid or rid */
 	if ( ( rc = cms_parse_identifier ( cms, part, &cursor ) ) != 0 )
 		return rc;
 	asn1_skip_any ( &cursor );
 
-	/* Parse digestAlgorithm */
-	if ( ( rc = cms_parse_digest_algorithm ( cms, part,
-						 &cursor ) ) != 0 )
-		return rc;
-	asn1_skip_any ( &cursor );
+	/* Parse signature-only objects */
+	if ( cms_is_signature ( cms ) ) {
 
-	/* Skip signedAttrs, if present */
-	asn1_skip_if_exists ( &cursor, ASN1_EXPLICIT_TAG ( 0 ) );
+		/* Parse digestAlgorithm */
+		if ( ( rc = cms_parse_digest_algorithm ( cms, part,
+							 &cursor ) ) != 0 )
+			return rc;
+		asn1_skip_any ( &cursor );
 
-	/* Parse signatureAlgorithm */
+		/* Skip signedAttrs, if present */
+		asn1_skip_if_exists ( &cursor, ASN1_EXPLICIT_TAG ( 0 ) );
+	}
+
+	/* Parse signatureAlgorithm or contentEncryptionAlgorithm */
 	if ( ( rc = cms_parse_pubkey_algorithm ( cms, part, &cursor ) ) != 0 )
 		return rc;
 	asn1_skip_any ( &cursor );
 
-	/* Parse signature */
+	/* Parse signature or encryptedKey */
 	if ( ( rc = cms_parse_value ( cms, part, &cursor ) ) != 0 )
 		return rc;
 
@@ -369,12 +443,13 @@ static int cms_parse_participants ( struct cms_message *cms,
 	struct cms_participant *part;
 	int rc;
 
-	/* Enter signerInfos */
+	/* Enter signerInfos or recipientInfos */
 	memcpy ( &cursor, raw, sizeof ( cursor ) );
 	asn1_enter ( &cursor, ASN1_SET );
 
-	/* Add each signerInfo.  Errors are handled by ensuring that
-	 * cms_put() will always be able to free any allocated memory.
+	/* Add each signerInfo or recipientInfo.  Errors are handled
+	 * by ensuring that cms_put() will always be able to free any
+	 * allocated memory.
 	 */
 	while ( cursor.len ) {
 
@@ -389,7 +464,7 @@ static int cms_parse_participants ( struct cms_message *cms,
 		if ( ! part->chain )
 			return -ENOMEM;
 
-		/* Parse signerInfo */
+		/* Parse signerInfo or recipientInfo */
 		if ( ( rc = cms_parse_participant ( cms, part,
 						    &cursor ) ) != 0 )
 			return rc;
@@ -399,6 +474,57 @@ static int cms_parse_participants ( struct cms_message *cms,
 	return 0;
 }
 
+/**
+ * Parse CMS message encrypted content information
+ *
+ * @v cms		CMS message
+ * @v raw		ASN.1 cursor
+ * @ret rc		Return status code
+ */
+static int cms_parse_encrypted ( struct cms_message *cms,
+				 const struct asn1_cursor *raw ) {
+	struct asn1_cursor cursor;
+	int rc;
+
+	/* Enter encryptedContentInfo */
+	memcpy ( &cursor, raw, sizeof ( cursor ) );
+	asn1_enter ( &cursor, ASN1_SEQUENCE );
+
+	/* Skip contentType */
+	asn1_skip ( &cursor, ASN1_OID );
+
+	/* Parse contentEncryptionAlgorithm */
+	if ( ( rc = cms_parse_cipher_algorithm ( cms, &cursor ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Parse CMS message MAC
+ *
+ * @v cms		CMS message
+ * @v raw		ASN.1 cursor
+ * @ret rc		Return status code
+ */
+static int cms_parse_mac ( struct cms_message *cms,
+			   const struct asn1_cursor *raw ) {
+	int rc;
+
+	/* Enter mac */
+	memcpy ( &cms->mac, raw, sizeof ( cms->mac ) );
+	if ( ( rc = asn1_enter ( &cms->mac, ASN1_OCTET_STRING ) ) != 0 ) {
+		DBGC ( cms, "CMS %p could not locate mac: %s\n",
+		       cms, strerror ( rc ) );
+		DBGC_HDA ( cms, 0, raw->data, raw->len );
+		return rc;
+	}
+	DBGC ( cms, "CMS %p mac is:\n", cms );
+	DBGC_HDA ( cms, 0, cms->mac.data, cms->mac.len );
+
+	return 0;
+}
+
 /**
  * Parse CMS signed data
  *
@@ -444,6 +570,50 @@ static int cms_parse_signed ( struct cms_message *cms,
 	return 0;
 }
 
+/**
+ * Parse CMS enveloped data
+ *
+ * @v cms		CMS message
+ * @v raw		ASN.1 cursor
+ * @ret rc		Return status code
+ */
+static int cms_parse_enveloped ( struct cms_message *cms,
+				 const struct asn1_cursor *raw ) {
+	struct asn1_cursor cursor;
+	int rc;
+
+	/* Enter envelopedData or authEnvelopedData */
+	memcpy ( &cursor, raw, sizeof ( cursor ) );
+	asn1_enter ( &cursor, ASN1_SEQUENCE );
+
+	/* Skip version */
+	asn1_skip ( &cursor, ASN1_INTEGER );
+
+	/* Skip originatorInfo, if present */
+	asn1_skip_if_exists ( &cursor, ASN1_IMPLICIT_TAG ( 0 ) );
+
+	/* Parse recipientInfos */
+	if ( ( rc = cms_parse_participants ( cms, &cursor ) ) != 0 )
+		return rc;
+	asn1_skip_any ( &cursor );
+
+	/* Parse encryptedContentInfo or authEncryptedContentInfo */
+	if ( ( rc = cms_parse_encrypted ( cms, &cursor ) ) != 0 )
+		return rc;
+	asn1_skip_any ( &cursor );
+	assert ( cms->cipher != NULL );
+
+	/* Skip unprotectedAttrs or authAttrs, if present */
+	asn1_skip_if_exists ( &cursor, ASN1_IMPLICIT_TAG ( 1 ) );
+
+	/* Parse mac, if present */
+	if ( ( cms->cipher->authsize != 0 ) &&
+	     ( ( rc = cms_parse_mac ( cms, &cursor ) ) != 0 ) )
+		return rc;
+
+	return 0;
+}
+
 /**
  * Parse CMS message from ASN.1 data
  *
@@ -516,6 +686,7 @@ int cms_message ( struct image *image, struct cms_message **cms ) {
 	}
 	ref_init ( &(*cms)->refcnt, cms_free );
 	INIT_LIST_HEAD ( &(*cms)->participants );
+	(*cms)->cipher = &cipher_null;
 
 	/* Get raw message data */
 	next = image_asn1 ( image, 0, &(*cms)->raw );
@@ -724,3 +895,309 @@ int cms_verify ( struct cms_message *cms, struct image *image,
 
 	return 0;
 }
+
+/**
+ * Identify CMS recipient corresponding to private key
+ *
+ * @v cms		CMS message
+ * @v private_key	Private key
+ * @ret part		Participant information, or NULL if not found
+ */
+static struct cms_participant *
+cms_recipient ( struct cms_message *cms, struct private_key *private_key ) {
+	struct cms_participant *part;
+	struct x509_certificate *cert;
+
+	/* Identify certificate (if any) for which we have a private key */
+	cert = x509_find_key ( NULL, private_key );
+	if ( ! cert )
+		return NULL;
+
+	/* Identify corresponding recipient, if any */
+	list_for_each_entry ( part, &cms->participants, list ) {
+		if ( cert == x509_first ( part->chain ) )
+			return part;
+	}
+
+	return NULL;
+}
+
+/**
+ * Set CMS cipher key
+ *
+ * @v cms		CMS message
+ * @v part		Participant information
+ * @v private_key	Private key
+ * @v ctx		Cipher context
+ * @ret rc		Return status code
+ */
+static int cms_cipher_key ( struct cms_message *cms,
+			    struct cms_participant *part,
+			    struct private_key *private_key, void *ctx ) {
+	struct cipher_algorithm *cipher = cms->cipher;
+	struct pubkey_algorithm *pubkey = part->pubkey;
+	const struct asn1_cursor *key = privkey_cursor ( private_key );
+	const struct asn1_cursor *value = &part->value;
+	size_t max_len = pubkey_max_len ( pubkey, key );
+	uint8_t cipher_key[max_len];
+	int len;
+	int rc;
+
+	/* Decrypt cipher key */
+	len = pubkey_decrypt ( pubkey, key, value->data, value->len,
+			       cipher_key );
+	if ( len < 0 ) {
+		rc = len;
+		DBGC ( cms, "CMS %p/%p could not decrypt cipher key: %s\n",
+		       cms, part, strerror ( rc ) );
+		DBGC_HDA ( cms, 0, value->data, value->len );
+		return rc;
+	}
+	DBGC ( cms, "CMS %p/%p cipher key:\n", cms, part );
+	DBGC_HDA ( cms, 0, cipher_key, len );
+
+	/* Set cipher key */
+	if ( ( rc = cipher_setkey ( cipher, ctx, cipher_key, len ) ) != 0 ) {
+		DBGC ( cms, "CMS %p could not set cipher key: %s\n",
+		       cms, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Set cipher initialization vector */
+	cipher_setiv ( cipher, ctx, cms->iv.data, cms->iv.len );
+	if ( cms->iv.len ) {
+		DBGC ( cms, "CMS %p cipher IV:\n", cms );
+		DBGC_HDA ( cms, 0, cms->iv.data, cms->iv.len );
+	}
+
+	return 0;
+}
+
+/**
+ * Initialise cipher for CMS decryption
+ *
+ * @v cms		CMS message
+ * @v private_key	Private key
+ * @v ctx		Cipher context
+ * @ret rc		Return status code
+ */
+static int cms_cipher ( struct cms_message *cms,
+			struct private_key *private_key, void *ctx ) {
+	struct cms_participant *part;
+	int rc;
+
+	/* Identify a usable recipient */
+	part = cms_recipient ( cms, private_key );
+	if ( ! part ) {
+		DBGC ( cms, "CMS %p had no usable recipients\n", cms );
+		return -EACCES_NO_RECIPIENTS;
+	}
+
+	/* Decrypt and set cipher key */
+	if ( ( rc = cms_cipher_key ( cms, part, private_key, ctx ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Check CMS padding
+ *
+ * @v cms		CMS message
+ * @v data		Final block
+ * @v len		Final block length
+ * @ret len		Padding length, or negative error
+ */
+static int cms_verify_padding ( struct cms_message *cms, const void *data,
+				size_t len ) {
+	struct cipher_algorithm *cipher = cms->cipher;
+	const uint8_t *pad;
+	size_t pad_len;
+	unsigned int i;
+
+	/* Non-block ciphers do not use padding */
+	if ( ! is_block_cipher ( cipher ) )
+		return 0;
+
+	/* Block padding can never produce an empty file */
+	if ( len == 0 ) {
+		DBGC ( cms, "CMS %p invalid empty padding\n", cms );
+		return -EACCES_PAD;
+	}
+
+	/* Sanity check */
+	assert ( len >= cipher->blocksize );
+
+	/* Extract and verify padding */
+	pad = ( data + len - 1 );
+	pad_len = *pad;
+	if ( ( pad_len == 0 ) || ( pad_len > len ) ) {
+		DBGC ( cms, "CMS %p invalid padding length %zd\n",
+		       cms, pad_len );
+		return -EACCES_PAD;
+	}
+	for ( i = 0 ; i < pad_len ; i++ ) {
+		if ( *(pad--) != pad_len ) {
+			DBGC ( cms, "CMS %p invalid padding\n", cms );
+			DBGC_HDA ( cms, 0, ( data + len - pad_len ), pad_len );
+			return -EACCES_PAD;
+		}
+	}
+
+	return pad_len;
+}
+
+/**
+ * Decrypt CMS message
+ *
+ * @v cms		CMS message
+ * @v image		Image to decrypt
+ * @v name		Decrypted image name, or NULL to use default
+ * @v private_key	Private key
+ * @ret rc		Return status code
+ */
+int cms_decrypt ( struct cms_message *cms, struct image *image,
+		  const char *name, struct private_key *private_key ) {
+	struct cipher_algorithm *cipher = cms->cipher;
+	const unsigned int original_flags = image->flags;
+	size_t offset;
+	size_t remaining;
+	size_t frag_len;
+	int pad_len;
+	void *tmp;
+	void *ctx;
+	void *ctxdup;
+	void *auth;
+	int rc;
+
+	/* Sanity checks */
+	if ( ! cipher ) {
+		rc = -ENOTTY;
+		goto err_no_cipher;
+	}
+
+	/* Check block size */
+	if ( ( image->len & ( cipher->blocksize - 1 ) ) != 0 ) {
+		DBGC ( cms, "CMS %p invalid length %zd\n", cms, image->len );
+		rc = -EACCES_LEN;
+		goto err_blocksize;
+	}
+
+	/* Allocate temporary working space */
+	tmp = malloc ( CMS_DECRYPT_BLKSZ + ( 2 * cipher->ctxsize ) +
+		       cipher->authsize );
+	if ( ! tmp ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	ctx = ( tmp + CMS_DECRYPT_BLKSZ );
+	ctxdup = ( ctx + cipher->ctxsize );
+	auth = ( ctxdup + cipher->ctxsize );
+
+	/* Initialise cipher */
+	if ( ( rc = cms_cipher ( cms, private_key, ctx ) ) != 0 )
+		goto err_cipher;
+
+	/* Duplicate cipher context for potential reencryption on error */
+	memcpy ( ctxdup, ctx, cipher->ctxsize );
+
+	/* Temporarily unregister image */
+	image_get ( image );
+	unregister_image ( image );
+
+	/* Clear trusted flag before modifying image */
+	image_untrust ( image );
+
+	/* Decrypt one block at a time */
+	offset = 0;
+	remaining = image->len;
+	frag_len = 0;
+	while ( remaining ) {
+
+		/* Calculate fragment length */
+		frag_len = remaining;
+		if ( frag_len > CMS_DECRYPT_BLKSZ )
+			frag_len = CMS_DECRYPT_BLKSZ;
+
+		/* Decrypt fragment */
+		copy_from_user ( tmp, image->data, offset, frag_len );
+		cipher_decrypt ( cipher, ctx, tmp, tmp, frag_len );
+
+		/* Overwrite all but the final fragment */
+		if ( remaining > frag_len )
+			copy_to_user ( image->data, offset, tmp, frag_len );
+
+		/* Move to next block */
+		remaining -= frag_len;
+		offset += frag_len;
+	}
+
+	/* Check authentication tag, if applicable */
+	cipher_auth ( cipher, ctx, auth );
+	if ( ( cms->mac.len != cipher->authsize ) ||
+	     ( memcmp ( cms->mac.data, auth, cipher->authsize ) != 0 ) ) {
+		DBGC ( cms, "CMS %p invalid authentication tag\n", cms );
+		DBGC_HDA ( cms, 0, auth, cipher->authsize );
+		rc = -EACCES_MAC;
+		goto err_auth;
+	}
+
+	/* Check block padding, if applicable */
+	if ( ( pad_len = cms_verify_padding ( cms, tmp, frag_len ) ) < 0 ) {
+		rc = pad_len;
+		goto err_pad;
+	}
+
+	/* Update image name. Do this as the last possible failure, so
+	 * that we do not have to include any error-handling code path
+	 * to restore the original image name (which may itself fail).
+	 */
+	if ( name ) {
+		if ( ( rc = image_set_name ( image, name ) ) != 0 )
+			goto err_set_name;
+	} else {
+		image_strip_suffix ( image );
+	}
+
+	/* Overwrite final fragment and strip block padding.  Do this
+	 * only once no further failure paths exist, so that we do not
+	 * have to include include any error-handling code path to
+	 * reconstruct the block padding.
+	 */
+	copy_to_user ( image->data, ( offset - frag_len ), tmp, frag_len );
+	image->len -= pad_len;
+
+	/* Clear image type and re-register image */
+	image->type = NULL;
+	register_image ( image );
+	image_put ( image );
+
+	/* Free temporary working space */
+	free ( tmp );
+
+	return 0;
+
+ err_set_name:
+ err_pad:
+ err_auth:
+	/* Reencrypt all overwritten fragments.  This can be done
+	 * since we have deliberately not overwritten the final
+	 * fragment containing the potentially invalid (and therefore
+	 * unreproducible) block padding.
+	 */
+	remaining = ( offset - frag_len );
+	for ( offset = 0 ; offset < remaining ; offset += CMS_DECRYPT_BLKSZ ) {
+		copy_from_user ( tmp, image->data, offset, CMS_DECRYPT_BLKSZ );
+		cipher_encrypt ( cipher, ctxdup, tmp, tmp, CMS_DECRYPT_BLKSZ );
+		copy_to_user ( image->data, offset, tmp, CMS_DECRYPT_BLKSZ );
+	}
+	image->flags = original_flags;
+	register_image ( image ); /* Cannot fail on re-registration */
+	image_put ( image );
+ err_cipher:
+	free ( tmp );
+ err_alloc:
+ err_blocksize:
+ err_no_cipher:
+	return rc;
+}
diff --git a/src/include/ipxe/asn1.h b/src/include/ipxe/asn1.h
index fac94c52e..752b423b9 100644
--- a/src/include/ipxe/asn1.h
+++ b/src/include/ipxe/asn1.h
@@ -309,6 +309,19 @@ struct asn1_builder_header {
 	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
 	ASN1_OID_SINGLE ( 7 ), ASN1_OID_SINGLE ( 2 )
 
+/** ASN.1 OID for id-envelopedData (1.2.840.113549.1.7.3) */
+#define ASN1_OID_ENVELOPEDDATA					\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 7 ), ASN1_OID_SINGLE ( 3 )
+
+/** ASN.1 OID for id-authEnvelopedData (1.2.840.113549.1.9.16.1.23) */
+#define ASN1_OID_AUTHENVELOPEDDATA					\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 9 ), ASN1_OID_SINGLE ( 16 ),		\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 23 )
+
 /** ASN.1 OID for id-pe-authorityInfoAccess (1.3.6.1.5.5.7.1.1) */
 #define ASN1_OID_AUTHORITYINFOACCESS				\
 	ASN1_OID_INITIAL ( 1, 3 ), ASN1_OID_SINGLE ( 6 ),	\
diff --git a/src/include/ipxe/cms.h b/src/include/ipxe/cms.h
index bffb06bfe..6d4a78d48 100644
--- a/src/include/ipxe/cms.h
+++ b/src/include/ipxe/cms.h
@@ -64,6 +64,13 @@ struct cms_message {
 	struct x509_chain *certificates;
 	/** List of participant information blocks */
 	struct list_head participants;
+
+	/** Cipher algorithm */
+	struct cipher_algorithm *cipher;
+	/** Cipher initialization vector */
+	struct asn1_cursor iv;
+	/** Cipher authentication tag */
+	struct asn1_cursor mac;
 };
 
 /**
@@ -101,9 +108,24 @@ cms_is_signature ( struct cms_message *cms ) {
 	return ( cms->certificates != NULL );
 }
 
+/**
+ * Check if CMS message is an encrypted message
+ *
+ * @v cms		CMS message
+ * @ret is_encrypted	Message is an encrypted message
+ */
+static inline __attribute__ (( always_inline )) int
+cms_is_encrypted ( struct cms_message *cms ) {
+
+	/* CMS encrypted messages have a cipher algorithm */
+	return ( cms->cipher != NULL );
+}
+
 extern int cms_message ( struct image *image, struct cms_message **cms );
 extern int cms_verify ( struct cms_message *cms, struct image *image,
 			const char *name, time_t time, struct x509_chain *store,
 			struct x509_root *root );
+extern int cms_decrypt ( struct cms_message *cms, struct image *image,
+			 const char *name, struct private_key *private_key );
 
 #endif /* _IPXE_CMS_H */

From 72316b820d4bdbf3d75a0ae7e13f1c3bc8e6ac29 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 29 Aug 2024 14:00:34 +0100
Subject: [PATCH 185/237] [image] Add the "imgdecrypt" command

Add the "imgdecrypt" command that can be used to decrypt a detached
encrypted data image using a cipher key obtained from a separate CMS
envelope image.  For example:

  # Create non-detached encrypted CMS messages
  #
  openssl cms -encrypt -binary -aes-256-gcm -recip client.crt \
              -in vmlinuz -outform DER -out vmlinuz.cms
  openssl cms -encrypt -binary -aes-256-gcm -recip client.crt \
              -in initrd.img -outform DER -out initrd.img.cms

  # Detach data from envelopes (using iPXE's contrib/crypto/cmsdetach)
  #
  cmsdetach vmlinuz.cms -d vmlinuz.dat -e vmlinuz.env
  cmsdetach initrd.img.cms -d initrd.img.dat -e initrd.img.env

and then within iPXE:

  #!ipxe
  imgfetch http://192.168.0.1/vmlinuz.dat
  imgfetch http://192.168.0.1/initrd.img.dat
  imgdecrypt vmlinuz.dat    http://192.168.0.1/vmlinuz.env
  imgdecrypt initrd.img.dat http://192.168.0.1/initrd.img.env
  boot vmlinuz

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config.c                |   3 +
 src/config/general.h               |   1 +
 src/hci/commands/image_crypt_cmd.c | 124 +++++++++++++++++++++++++++++
 src/include/usr/imgcrypt.h         |  17 ++++
 src/usr/imgcrypt.c                 |  75 +++++++++++++++++
 5 files changed, 220 insertions(+)
 create mode 100644 src/hci/commands/image_crypt_cmd.c
 create mode 100644 src/include/usr/imgcrypt.h
 create mode 100644 src/usr/imgcrypt.c

diff --git a/src/config/config.c b/src/config/config.c
index 6bcd3c1ca..f532c1d2d 100644
--- a/src/config/config.c
+++ b/src/config/config.c
@@ -299,6 +299,9 @@ REQUIRE_OBJECT ( image_mem_cmd );
 #ifdef SHIM_CMD
 REQUIRE_OBJECT ( shim_cmd );
 #endif
+#ifdef IMAGE_CRYPT_CMD
+REQUIRE_OBJECT ( image_crypt_cmd );
+#endif
 
 /*
  * Drag in miscellaneous objects
diff --git a/src/config/general.h b/src/config/general.h
index f936e874c..b82e5602c 100644
--- a/src/config/general.h
+++ b/src/config/general.h
@@ -158,6 +158,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 //#define REBOOT_CMD		/* Reboot command */
 //#define POWEROFF_CMD		/* Power off command */
 //#define IMAGE_TRUST_CMD	/* Image trust management commands */
+//#define IMAGE_CRYPT_CMD	/* Image encryption management commands */
 //#define PCI_CMD		/* PCI commands */
 //#define PARAM_CMD		/* Request parameter commands */
 //#define NEIGHBOUR_CMD		/* Neighbour management commands */
diff --git a/src/hci/commands/image_crypt_cmd.c b/src/hci/commands/image_crypt_cmd.c
new file mode 100644
index 000000000..26e9d79f8
--- /dev/null
+++ b/src/hci/commands/image_crypt_cmd.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <stdio.h>
+#include <getopt.h>
+#include <ipxe/image.h>
+#include <ipxe/command.h>
+#include <ipxe/parseopt.h>
+#include <usr/imgmgmt.h>
+#include <usr/imgcrypt.h>
+
+/** @file
+ *
+ * Image encryption management commands
+ *
+ */
+
+/** "imgdecrypt" options */
+struct imgdecrypt_options {
+	/** Decrypted image name */
+	char *name;
+	/** Keep envelope after decryption */
+	int keep;
+	/** Download timeout */
+	unsigned long timeout;
+};
+
+/** "imgdecrypt" option list */
+static struct option_descriptor imgdecrypt_opts[] = {
+	OPTION_DESC ( "name", 'n', required_argument,
+		      struct imgdecrypt_options, name, parse_string ),
+	OPTION_DESC ( "keep", 'k', no_argument,
+		      struct imgdecrypt_options, keep, parse_flag ),
+	OPTION_DESC ( "timeout", 't', required_argument,
+		      struct imgdecrypt_options, timeout, parse_timeout),
+};
+
+/** "imgdecrypt" command descriptor */
+static struct command_descriptor imgdecrypt_cmd =
+	COMMAND_DESC ( struct imgdecrypt_options, imgdecrypt_opts, 2, 2,
+		       "<uri|image> <envelope uri|image>" );
+
+/**
+ * The "imgdecrypt" command
+ *
+ * @v argc		Argument count
+ * @v argv		Argument list
+ * @ret rc		Return status code
+ */
+static int imgdecrypt_exec ( int argc, char **argv ) {
+	struct imgdecrypt_options opts;
+	const char *image_name_uri;
+	const char *envelope_name_uri;
+	struct image *image;
+	struct image *envelope;
+	int rc;
+
+	/* Parse options */
+	if ( ( rc = parse_options ( argc, argv, &imgdecrypt_cmd, &opts ) ) != 0)
+		return rc;
+
+	/* Parse image name/URI string */
+	image_name_uri = argv[optind];
+
+	/* Parse envelope name/URI string */
+	envelope_name_uri = argv[ optind + 1 ];
+
+	/* Acquire the image */
+	if ( ( rc = imgacquire ( image_name_uri, opts.timeout, &image ) ) != 0 )
+		goto err_acquire_image;
+
+	/* Acquire the envelope image */
+	if ( ( rc = imgacquire ( envelope_name_uri, opts.timeout,
+				 &envelope ) ) != 0 )
+		goto err_acquire_envelope;
+
+	/* Decrypt image */
+	if ( ( rc = imgdecrypt ( image, envelope, opts.name ) ) != 0 ) {
+		printf ( "Could not decrypt: %s\n", strerror ( rc ) );
+		goto err_decrypt;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_decrypt:
+	/* Discard envelope unless --keep was specified */
+	if ( ! opts.keep )
+		unregister_image ( envelope );
+ err_acquire_envelope:
+ err_acquire_image:
+	return rc;
+}
+
+/** Image encryption management commands */
+struct command image_crypt_commands[] __command = {
+	{
+		.name = "imgdecrypt",
+		.exec = imgdecrypt_exec,
+	},
+};
diff --git a/src/include/usr/imgcrypt.h b/src/include/usr/imgcrypt.h
new file mode 100644
index 000000000..c9408c75f
--- /dev/null
+++ b/src/include/usr/imgcrypt.h
@@ -0,0 +1,17 @@
+#ifndef _USR_IMGCRYPT_H
+#define _USR_IMGCRYPT_H
+
+/** @file
+ *
+ * Image encryption management
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/image.h>
+
+extern int imgdecrypt ( struct image *image, struct image *envelope,
+			const char *name );
+
+#endif /* _USR_IMGCRYPT_H */
diff --git a/src/usr/imgcrypt.c b/src/usr/imgcrypt.c
new file mode 100644
index 000000000..1b649f197
--- /dev/null
+++ b/src/usr/imgcrypt.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <syslog.h>
+#include <ipxe/image.h>
+#include <ipxe/cms.h>
+#include <ipxe/privkey.h>
+#include <usr/imgcrypt.h>
+
+/** @file
+ *
+ * Image encryption management
+ *
+ */
+
+/**
+ * Decrypt image using downloaded envelope
+ *
+ * @v image		Image to decrypt
+ * @v envelope		Image containing decryption key
+ * @v name		Decrypted image name (or NULL to use default)
+ * @ret rc		Return status code
+ */
+int imgdecrypt ( struct image *image, struct image *envelope,
+		 const char *name ) {
+	struct cms_message *cms;
+	int rc;
+
+	/* Parse envelope */
+	if ( ( rc = cms_message ( envelope, &cms ) ) != 0 )
+		goto err_parse;
+
+	/* Decrypt image */
+	if ( ( rc = cms_decrypt ( cms, image, name, &private_key ) ) != 0 )
+		goto err_decrypt;
+
+	/* Drop reference to message */
+	cms_put ( cms );
+	cms = NULL;
+
+	/* Record decryption */
+	syslog ( LOG_NOTICE, "Image \"%s\" decrypted OK\n", image->name );
+
+	return 0;
+
+ err_decrypt:
+	cms_put ( cms );
+ err_parse:
+	syslog ( LOG_ERR, "Image \"%s\" decryption failed: %s\n",
+		 image->name, strerror ( rc ) );
+	return rc;
+}

From 5e69cf08d7924aedc678741253773af440142bb8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 29 Aug 2024 23:31:10 +0100
Subject: [PATCH 186/237] [crypto] Allow cms_decrypt() to be called on
 unregistered images

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/cms.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/crypto/cms.c b/src/crypto/cms.c
index 3a5debfcf..6c8217c0f 100644
--- a/src/crypto/cms.c
+++ b/src/crypto/cms.c
@@ -1101,13 +1101,15 @@ int cms_decrypt ( struct cms_message *cms, struct image *image,
 	/* Duplicate cipher context for potential reencryption on error */
 	memcpy ( ctxdup, ctx, cipher->ctxsize );
 
-	/* Temporarily unregister image */
-	image_get ( image );
-	unregister_image ( image );
-
 	/* Clear trusted flag before modifying image */
 	image_untrust ( image );
 
+	/* Temporarily unregister image, if applicable */
+	if ( original_flags & IMAGE_REGISTERED ) {
+		image_get ( image );
+		unregister_image ( image );
+	}
+
 	/* Decrypt one block at a time */
 	offset = 0;
 	remaining = image->len;
@@ -1167,10 +1169,12 @@ int cms_decrypt ( struct cms_message *cms, struct image *image,
 	copy_to_user ( image->data, ( offset - frag_len ), tmp, frag_len );
 	image->len -= pad_len;
 
-	/* Clear image type and re-register image */
+	/* Clear image type and re-register image, if applicable */
 	image->type = NULL;
-	register_image ( image );
-	image_put ( image );
+	if ( original_flags & IMAGE_REGISTERED ) {
+		register_image ( image );
+		image_put ( image );
+	}
 
 	/* Free temporary working space */
 	free ( tmp );
@@ -1191,9 +1195,11 @@ int cms_decrypt ( struct cms_message *cms, struct image *image,
 		cipher_encrypt ( cipher, ctxdup, tmp, tmp, CMS_DECRYPT_BLKSZ );
 		copy_to_user ( image->data, offset, tmp, CMS_DECRYPT_BLKSZ );
 	}
+	if ( original_flags & IMAGE_REGISTERED ) {
+		register_image ( image ); /* Cannot fail on re-registration */
+		image_put ( image );
+	}
 	image->flags = original_flags;
-	register_image ( image ); /* Cannot fail on re-registration */
-	image_put ( image );
  err_cipher:
 	free ( tmp );
  err_alloc:

From 301644ab480ab9787c617e1a94e19ca5c2774072 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 29 Aug 2024 23:31:40 +0100
Subject: [PATCH 187/237] [test] Add CMS decryption self-tests

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/tests/cms_test.c | 355 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 353 insertions(+), 2 deletions(-)

diff --git a/src/tests/cms_test.c b/src/tests/cms_test.c
index 2480263e1..fc4f6bd19 100644
--- a/src/tests/cms_test.c
+++ b/src/tests/cms_test.c
@@ -40,6 +40,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/uaccess.h>
 #include <ipxe/der.h>
 #include <ipxe/cms.h>
+#include <ipxe/privkey.h>
 #include <ipxe/test.h>
 
 /** Fingerprint algorithm used for X.509 test certificates */
@@ -59,6 +60,18 @@ struct cms_test_message {
 	struct cms_message *cms;
 };
 
+/** Test CMS key pair */
+struct cms_test_keypair {
+	/** Private key */
+	struct private_key privkey;
+	/** Certificate data */
+	const void *data;
+	/** Length of certificate data */
+	size_t len;
+	/** Parsed certificate */
+	struct x509_certificate *cert;
+};
+
 /** Define inline data */
 #define DATA(...) { __VA_ARGS__ }
 
@@ -67,7 +80,7 @@ struct cms_test_message {
 
 /** Define a test image */
 #define IMAGE( NAME, DATA )						\
-	static const uint8_t NAME ## _data[] = DATA;			\
+	static uint8_t NAME ## _data[] = DATA;				\
 	static struct cms_test_image NAME = {				\
 		.image = {						\
 			.refcnt = REF_INIT ( ref_no_free ),		\
@@ -80,7 +93,7 @@ struct cms_test_message {
 /** Define a test message */
 #define MESSAGE( NAME, DATA )						\
 	static const uint8_t NAME ## _data[] = DATA;			\
-	static struct cms_test_message NAME = {			\
+	static struct cms_test_message NAME = {				\
 		.image = {						\
 			.refcnt = REF_INIT ( ref_no_free ),		\
 			.name = #NAME,					\
@@ -90,6 +103,22 @@ struct cms_test_message {
 		},							\
 	}
 
+/** Define a test key pair */
+#define KEYPAIR( NAME, PRIVKEY, CERT )					\
+	static uint8_t NAME ## _privkey[] = PRIVKEY;			\
+	static const uint8_t NAME ## _cert[] = CERT;			\
+	static struct cms_test_keypair NAME = {				\
+		.privkey = {						\
+			.refcnt = REF_INIT ( ref_no_free ),		\
+			.builder = {					\
+				.data = NAME ## _privkey,		\
+				.len = sizeof ( NAME ## _privkey ),	\
+			},						\
+		},							\
+		.data = NAME ## _cert,					\
+		.len = sizeof ( NAME ## _cert ),			\
+	}
+
 /** Code that has been signed */
 IMAGE ( test_code,
 	DATA ( 0x23, 0x21, 0x69, 0x70, 0x78, 0x65, 0x0a, 0x0a, 0x65, 0x63,
@@ -113,6 +142,123 @@ IMAGE ( bad_code,
 	       0x6f, 0x75, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x21, 0x0a, 0x73,
 	       0x68, 0x65, 0x6c, 0x6c, 0x0a ) );
 
+/** Plaintext of encrypted code */
+IMAGE ( hidden_code,
+	DATA ( 0x23, 0x21, 0x69, 0x70, 0x78, 0x65, 0x0a, 0x0a, 0x65, 0x63,
+	       0x68, 0x6f, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73,
+	       0x20, 0x61, 0x20, 0x64, 0x65, 0x63, 0x72, 0x79, 0x70, 0x74,
+	       0x65, 0x64, 0x20, 0x69, 0x50, 0x58, 0x45, 0x20, 0x73, 0x63,
+	       0x72, 0x69, 0x70, 0x74, 0x2e, 0x20, 0x20, 0x44, 0x6f, 0x20,
+	       0x61, 0x6e, 0x79, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x20, 0x79,
+	       0x6f, 0x75, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x21, 0x0a, 0x73,
+	       0x68, 0x65, 0x6c, 0x6c, 0x0a ) );
+
+/** Code encrypted with AES-256-CBC */
+IMAGE ( hidden_code_cbc_dat,
+	DATA ( 0xaa, 0x63, 0x9f, 0x12, 0xeb, 0x1e, 0xdd, 0x9b, 0xb6, 0x4d,
+	       0x81, 0xd5, 0xba, 0x2d, 0x86, 0x7a, 0x1c, 0x39, 0x10, 0x60,
+	       0x43, 0xac, 0x1b, 0x4e, 0x43, 0xb7, 0x50, 0x5a, 0x6d, 0x7a,
+	       0x4b, 0xd8, 0x65, 0x3c, 0x3e, 0xbd, 0x40, 0x9e, 0xb2, 0xe1,
+	       0x7d, 0x80, 0xf8, 0x22, 0x50, 0xf7, 0x32, 0x3a, 0x43, 0xf9,
+	       0xdf, 0xa6, 0xab, 0xa4, 0xb3, 0xdd, 0x27, 0x88, 0xd9, 0xb0,
+	       0x99, 0xb5, 0x7a, 0x89, 0x6c, 0xb7, 0x63, 0x45, 0x42, 0x7d,
+	       0xbe, 0x43, 0xab, 0x7e, 0x6c, 0x0c, 0xd5, 0xba, 0x9e, 0x37 ) );
+
+/** Envelope for code encrypted with AES-256-CBC */
+MESSAGE ( hidden_code_cbc_env,
+	DATA ( 0x30, 0x82, 0x01, 0x70, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86,
+	       0xf7, 0x0d, 0x01, 0x07, 0x03, 0xa0, 0x82, 0x01, 0x61, 0x30,
+	       0x82, 0x01, 0x5d, 0x02, 0x01, 0x00, 0x31, 0x82, 0x01, 0x2a,
+	       0x30, 0x82, 0x01, 0x26, 0x02, 0x01, 0x00, 0x30, 0x81, 0x8e,
+	       0x30, 0x81, 0x88, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55,
+	       0x04, 0x06, 0x13, 0x02, 0x47, 0x42, 0x31, 0x17, 0x30, 0x15,
+	       0x06, 0x03, 0x55, 0x04, 0x08, 0x0c, 0x0e, 0x43, 0x61, 0x6d,
+	       0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x73, 0x68, 0x69, 0x72,
+	       0x65, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x07,
+	       0x0c, 0x09, 0x43, 0x61, 0x6d, 0x62, 0x72, 0x69, 0x64, 0x67,
+	       0x65, 0x31, 0x18, 0x30, 0x16, 0x06, 0x03, 0x55, 0x04, 0x0a,
+	       0x0c, 0x0f, 0x46, 0x65, 0x6e, 0x20, 0x53, 0x79, 0x73, 0x74,
+	       0x65, 0x6d, 0x73, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x11, 0x30,
+	       0x0f, 0x06, 0x03, 0x55, 0x04, 0x0b, 0x0c, 0x08, 0x69, 0x70,
+	       0x78, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x31, 0x1f, 0x30, 0x1d,
+	       0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, 0x16, 0x69, 0x50, 0x58,
+	       0x45, 0x20, 0x73, 0x65, 0x6c, 0x66, 0x2d, 0x74, 0x65, 0x73,
+	       0x74, 0x20, 0x6c, 0x65, 0x61, 0x66, 0x20, 0x43, 0x41, 0x02,
+	       0x01, 0x08, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86,
+	       0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, 0x04, 0x81, 0x80,
+	       0x10, 0x26, 0x94, 0x97, 0xe5, 0x84, 0x53, 0xf9, 0x03, 0xc4,
+	       0xf6, 0xc1, 0x55, 0xf5, 0x00, 0x23, 0xb9, 0x80, 0x20, 0x85,
+	       0xd2, 0xf2, 0xc4, 0x61, 0xdb, 0x73, 0x8d, 0xe1, 0xa1, 0x73,
+	       0x7d, 0x31, 0x12, 0xb7, 0xac, 0xa4, 0xe0, 0x40, 0x10, 0xcf,
+	       0xd5, 0x55, 0x70, 0x75, 0x9f, 0x7b, 0x61, 0xd3, 0x9e, 0xc6,
+	       0x58, 0x78, 0x05, 0x66, 0xc1, 0x86, 0x2f, 0x00, 0xcd, 0xe9,
+	       0x32, 0x63, 0x7c, 0x95, 0xd7, 0x75, 0x46, 0xde, 0x76, 0x7d,
+	       0x49, 0x64, 0x86, 0xd6, 0xeb, 0x0f, 0x1c, 0x01, 0x20, 0xb9,
+	       0xb9, 0x42, 0xc4, 0x20, 0x1f, 0x3f, 0xae, 0x5f, 0xb9, 0xd0,
+	       0xb6, 0x49, 0xcc, 0x4c, 0xd9, 0xcb, 0x36, 0xa4, 0x2f, 0xb1,
+	       0x97, 0x40, 0xf3, 0xe7, 0x19, 0x88, 0x93, 0x58, 0x61, 0x31,
+	       0xac, 0x9a, 0x93, 0x6e, 0x79, 0x31, 0x3a, 0x79, 0xa4, 0x20,
+	       0x38, 0x17, 0x92, 0x40, 0x7c, 0x98, 0xea, 0x86, 0x30, 0x2a,
+	       0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x07,
+	       0x01, 0x30, 0x1d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65,
+	       0x03, 0x04, 0x01, 0x2a, 0x04, 0x10, 0xd8, 0x80, 0xc5, 0xfa,
+	       0xc3, 0x25, 0xa0, 0x22, 0xb5, 0x6c, 0x0c, 0x27, 0x2c, 0xe9,
+	       0xba, 0xcf ) );
+
+/** Code encrypted with AES-256-GCM (no block padding) */
+IMAGE ( hidden_code_gcm_dat,
+	DATA ( 0x0c, 0x96, 0xa6, 0x54, 0x9a, 0xc2, 0x24, 0x89, 0x15, 0x00,
+	       0x90, 0xe1, 0x35, 0xca, 0x4a, 0x84, 0x8e, 0x0b, 0xc3, 0x5e,
+	       0xc0, 0x61, 0x61, 0xbd, 0x2e, 0x69, 0x84, 0x7a, 0x2f, 0xf6,
+	       0xbe, 0x39, 0x04, 0x0a, 0x8d, 0x91, 0x6b, 0xaf, 0x63, 0xd4,
+	       0x03, 0xf1, 0x72, 0x38, 0xee, 0x27, 0xd6, 0x5a, 0xae, 0x15,
+	       0xd5, 0xec, 0xb6, 0xb6, 0x4f, 0x6f, 0xf6, 0x76, 0x22, 0x74,
+	       0xca, 0x72, 0x0b, 0xfa, 0x6a, 0x0e, 0x4a, 0x3e, 0x8c, 0x60,
+	       0x78, 0x24, 0x48, 0x58, 0xdd ) );
+
+/** Envelope for code encrypted with AES-256-GCM (no block padding) */
+MESSAGE ( hidden_code_gcm_env,
+	DATA ( 0x30, 0x82, 0x01, 0x85, 0x06, 0x0b, 0x2a, 0x86, 0x48, 0x86,
+	       0xf7, 0x0d, 0x01, 0x09, 0x10, 0x01, 0x17, 0xa0, 0x82, 0x01,
+	       0x74, 0x30, 0x82, 0x01, 0x70, 0x02, 0x01, 0x00, 0x31, 0x82,
+	       0x01, 0x2a, 0x30, 0x82, 0x01, 0x26, 0x02, 0x01, 0x00, 0x30,
+	       0x81, 0x8e, 0x30, 0x81, 0x88, 0x31, 0x0b, 0x30, 0x09, 0x06,
+	       0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x47, 0x42, 0x31, 0x17,
+	       0x30, 0x15, 0x06, 0x03, 0x55, 0x04, 0x08, 0x0c, 0x0e, 0x43,
+	       0x61, 0x6d, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x73, 0x68,
+	       0x69, 0x72, 0x65, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55,
+	       0x04, 0x07, 0x0c, 0x09, 0x43, 0x61, 0x6d, 0x62, 0x72, 0x69,
+	       0x64, 0x67, 0x65, 0x31, 0x18, 0x30, 0x16, 0x06, 0x03, 0x55,
+	       0x04, 0x0a, 0x0c, 0x0f, 0x46, 0x65, 0x6e, 0x20, 0x53, 0x79,
+	       0x73, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x4c, 0x74, 0x64, 0x31,
+	       0x11, 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0b, 0x0c, 0x08,
+	       0x69, 0x70, 0x78, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x31, 0x1f,
+	       0x30, 0x1d, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, 0x16, 0x69,
+	       0x50, 0x58, 0x45, 0x20, 0x73, 0x65, 0x6c, 0x66, 0x2d, 0x74,
+	       0x65, 0x73, 0x74, 0x20, 0x6c, 0x65, 0x61, 0x66, 0x20, 0x43,
+	       0x41, 0x02, 0x01, 0x08, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86,
+	       0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, 0x04,
+	       0x81, 0x80, 0x7e, 0xbc, 0xba, 0xee, 0xfa, 0x50, 0x46, 0xee,
+	       0xed, 0xf1, 0x54, 0xe1, 0x46, 0xeb, 0x57, 0x3e, 0x76, 0xd7,
+	       0x8f, 0xe3, 0x26, 0x42, 0x3d, 0x28, 0xf9, 0xdc, 0x20, 0xe6,
+	       0x27, 0x3b, 0x89, 0xcb, 0xab, 0xd5, 0xad, 0xc2, 0xf0, 0x4f,
+	       0xa8, 0xb9, 0x77, 0x5b, 0x6c, 0xe6, 0x34, 0x22, 0x73, 0x5b,
+	       0xa4, 0x8e, 0x1c, 0xc2, 0xf8, 0x50, 0xef, 0xe5, 0xcf, 0x80,
+	       0x16, 0x79, 0x6b, 0x0f, 0xa7, 0xfd, 0xdc, 0x60, 0x9c, 0x94,
+	       0x60, 0xa6, 0x12, 0x5a, 0xfb, 0xc2, 0xc8, 0x4c, 0x64, 0x83,
+	       0x99, 0x73, 0xfc, 0xa1, 0xf8, 0xa5, 0x82, 0x75, 0xba, 0x53,
+	       0xeb, 0xc8, 0x94, 0x0e, 0x29, 0x23, 0x9a, 0x2b, 0xa6, 0x63,
+	       0x2d, 0x5b, 0x9f, 0x38, 0x70, 0x21, 0xe8, 0xe9, 0xa6, 0xcf,
+	       0xf2, 0xbe, 0x66, 0xf4, 0xcc, 0x91, 0x01, 0x4f, 0x04, 0x00,
+	       0x4a, 0x55, 0xc9, 0x42, 0x76, 0xd4, 0x3f, 0x65, 0x0c, 0x76,
+	       0x30, 0x2b, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d,
+	       0x01, 0x07, 0x01, 0x30, 0x1e, 0x06, 0x09, 0x60, 0x86, 0x48,
+	       0x01, 0x65, 0x03, 0x04, 0x01, 0x2e, 0x30, 0x11, 0x04, 0x0c,
+	       0x72, 0xb5, 0x14, 0xfb, 0x05, 0x78, 0x70, 0x1c, 0xf4, 0x4b,
+	       0xba, 0xbf, 0x02, 0x01, 0x10, 0x04, 0x10, 0x1e, 0x44, 0xda,
+	       0x5c, 0x75, 0x7e, 0x4f, 0xa7, 0xe7, 0xd9, 0x98, 0x80, 0x9c,
+	       0x25, 0x6a, 0xfb ) );
+
 /** Valid signature */
 MESSAGE ( codesigned_sig,
 	DATA ( 0x30, 0x82, 0x0c, 0x41, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86,
@@ -1308,6 +1454,138 @@ MESSAGE ( nonsigned_sig,
 	       0x5d, 0x70, 0x47, 0x54, 0xbc, 0x15, 0xad, 0x9c, 0xe8, 0x90,
 	       0x52, 0x3e, 0x49, 0x86 ) );
 
+/** Client certificate and private key */
+KEYPAIR ( client_keypair,
+	DATA ( 0x30, 0x82, 0x02, 0x77, 0x02, 0x01, 0x00, 0x30, 0x0d, 0x06,
+	       0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01,
+	       0x05, 0x00, 0x04, 0x82, 0x02, 0x61, 0x30, 0x82, 0x02, 0x5d,
+	       0x02, 0x01, 0x00, 0x02, 0x81, 0x81, 0x00, 0xb7, 0x9f, 0xb5,
+	       0x90, 0xfa, 0x47, 0x25, 0xee, 0x3b, 0x04, 0x02, 0x4f, 0xd4,
+	       0x1d, 0x62, 0x46, 0x9d, 0xce, 0x79, 0x3a, 0x80, 0x3a, 0xc4,
+	       0x06, 0x6a, 0x67, 0xd4, 0x3a, 0x61, 0x71, 0xcd, 0xb3, 0xcc,
+	       0x1b, 0xfc, 0x2f, 0x17, 0xa8, 0xf2, 0x26, 0x9e, 0x54, 0xd3,
+	       0x49, 0x81, 0x22, 0xa9, 0x72, 0x4c, 0xe8, 0x92, 0xb7, 0x1e,
+	       0x44, 0x8f, 0xa9, 0x4d, 0x83, 0x0b, 0x89, 0x2a, 0xc7, 0xb3,
+	       0xad, 0x54, 0x32, 0x76, 0x62, 0x1e, 0xe5, 0xbe, 0xc1, 0xa8,
+	       0x7a, 0x2b, 0xf0, 0xa9, 0x9a, 0xfb, 0xb8, 0x18, 0x0c, 0x30,
+	       0x5a, 0x13, 0x9c, 0x21, 0x26, 0x96, 0x4f, 0x39, 0x98, 0x64,
+	       0x41, 0x3b, 0x94, 0xc8, 0xe3, 0xb7, 0x8c, 0x33, 0x9e, 0xd0,
+	       0x71, 0xd0, 0x2f, 0xe3, 0x7d, 0x2c, 0x57, 0x27, 0x42, 0x26,
+	       0xd9, 0x2c, 0xb4, 0x55, 0xd5, 0x66, 0xb7, 0xbf, 0x00, 0xa1,
+	       0xcc, 0x61, 0x19, 0x99, 0x61, 0x02, 0x03, 0x01, 0x00, 0x01,
+	       0x02, 0x81, 0x80, 0x50, 0x63, 0x2f, 0xe6, 0xb7, 0x5a, 0xfc,
+	       0x85, 0x0d, 0xfb, 0x14, 0x54, 0x04, 0x65, 0x94, 0xc7, 0x9b,
+	       0x80, 0x6f, 0xdc, 0x27, 0x95, 0x12, 0x8a, 0x48, 0x7d, 0x0a,
+	       0x11, 0x40, 0xe5, 0xc4, 0x8b, 0x29, 0x19, 0x3b, 0x4f, 0x16,
+	       0x89, 0x94, 0xf1, 0x49, 0x31, 0x93, 0x8a, 0x43, 0x69, 0x7c,
+	       0x4b, 0x18, 0xd6, 0x5c, 0x9c, 0xa4, 0x38, 0x99, 0xb8, 0x21,
+	       0xc1, 0xf4, 0x03, 0xe9, 0xe1, 0xa1, 0x8b, 0xcb, 0x51, 0x4f,
+	       0x64, 0x68, 0x1c, 0x73, 0xc0, 0x0d, 0xd2, 0xcd, 0x87, 0xcc,
+	       0x45, 0xe8, 0xbf, 0x88, 0xea, 0x6c, 0x42, 0xfa, 0x03, 0x7a,
+	       0x29, 0xd2, 0xcf, 0xf1, 0xcb, 0xae, 0xea, 0xfb, 0x50, 0xf2,
+	       0xbd, 0x02, 0x4f, 0x2f, 0x4f, 0xba, 0x82, 0xa9, 0xde, 0x60,
+	       0x56, 0xd4, 0x07, 0x73, 0xdf, 0x12, 0x09, 0x73, 0x7b, 0x54,
+	       0x35, 0xc6, 0x28, 0x10, 0xb0, 0xbd, 0xc8, 0xe1, 0x8f, 0xb2,
+	       0x41, 0x02, 0x41, 0x00, 0xd8, 0xf9, 0x6c, 0x70, 0x56, 0x3c,
+	       0x74, 0x44, 0x53, 0x13, 0xed, 0x92, 0xab, 0xbc, 0x0c, 0x5c,
+	       0x66, 0x2c, 0xd7, 0xed, 0x10, 0x82, 0xe3, 0xe3, 0x2e, 0xda,
+	       0x4d, 0x3e, 0x1f, 0xc0, 0x50, 0xa8, 0xf2, 0xce, 0x77, 0xa9,
+	       0xae, 0xa2, 0x2d, 0x49, 0x6a, 0x6f, 0x01, 0xe3, 0xca, 0x57,
+	       0xf4, 0xcc, 0xb4, 0x3f, 0xd9, 0xc3, 0x58, 0x54, 0xe7, 0x62,
+	       0xfc, 0x40, 0xc8, 0xba, 0x18, 0x0d, 0xfe, 0x89, 0x02, 0x41,
+	       0x00, 0xd8, 0xa6, 0xaa, 0x4b, 0xcc, 0xcf, 0xc4, 0x47, 0x84,
+	       0x13, 0x39, 0x5b, 0x2e, 0xcb, 0xe0, 0x41, 0xd0, 0x2c, 0x96,
+	       0x58, 0x73, 0xab, 0xf6, 0x41, 0x0c, 0x7b, 0xbe, 0x60, 0xa1,
+	       0xcb, 0x00, 0x1a, 0xb0, 0x4b, 0xc1, 0xf5, 0x27, 0x43, 0x97,
+	       0x87, 0x30, 0x3c, 0x27, 0xa3, 0xe3, 0xf1, 0xa7, 0x45, 0x01,
+	       0xe2, 0x1c, 0x43, 0xe9, 0x48, 0x43, 0x76, 0x24, 0x4b, 0x2b,
+	       0xc7, 0x67, 0x3e, 0x4e, 0x19, 0x02, 0x40, 0x6a, 0x43, 0x96,
+	       0x31, 0x5a, 0x7a, 0xd7, 0x32, 0x93, 0x41, 0xa2, 0x4c, 0x00,
+	       0x21, 0xe4, 0x27, 0xe8, 0xbe, 0xb3, 0xad, 0xde, 0x35, 0x4c,
+	       0xa8, 0xfa, 0x4c, 0x5e, 0x22, 0x3b, 0xe8, 0xb3, 0x58, 0x5b,
+	       0x3a, 0x75, 0x6e, 0xbc, 0x21, 0x9f, 0x6e, 0x62, 0x5b, 0x25,
+	       0xa0, 0xcb, 0x7b, 0xd2, 0x5f, 0xe3, 0x33, 0x96, 0x52, 0x4e,
+	       0xd3, 0x9e, 0x53, 0x63, 0x59, 0xd3, 0x35, 0x19, 0x0c, 0xd9,
+	       0x89, 0x02, 0x41, 0x00, 0x8f, 0x8d, 0xb7, 0xaf, 0x6c, 0x31,
+	       0x8b, 0x0c, 0x1c, 0x1e, 0xa4, 0xd5, 0x9f, 0x67, 0x65, 0xdc,
+	       0x15, 0xf5, 0x45, 0x55, 0xac, 0xa7, 0x98, 0x0f, 0x38, 0x17,
+	       0x52, 0x69, 0x33, 0x2b, 0x90, 0x91, 0x1e, 0x99, 0xc4, 0x16,
+	       0x0e, 0x03, 0x42, 0x87, 0x48, 0x55, 0xc3, 0xaa, 0x5b, 0xe2,
+	       0x86, 0x84, 0x3a, 0x20, 0x39, 0xbc, 0x61, 0xfa, 0x09, 0x01,
+	       0x62, 0x41, 0x10, 0xec, 0x1a, 0xa3, 0xf5, 0x19, 0x02, 0x41,
+	       0x00, 0x98, 0x04, 0xc8, 0x8e, 0xd0, 0xfa, 0xe5, 0xab, 0x8a,
+	       0x1c, 0x4a, 0xc2, 0xf7, 0xd4, 0xad, 0x52, 0x51, 0x81, 0xa6,
+	       0x59, 0x62, 0x84, 0x16, 0x82, 0xf2, 0x5d, 0xd0, 0x5d, 0x4a,
+	       0xcf, 0x94, 0x2f, 0x13, 0x47, 0xd4, 0xc2, 0x7f, 0x89, 0x2e,
+	       0x40, 0xf5, 0xfc, 0xf8, 0x82, 0xc6, 0x53, 0xd4, 0x75, 0x32,
+	       0x47, 0x1d, 0xcf, 0xd1, 0xae, 0x35, 0x41, 0x7a, 0xdc, 0x10,
+	       0x0c, 0xc2, 0xd6, 0xe1, 0xd5 ),
+	DATA ( 0x30, 0x82, 0x02, 0x7f, 0x30, 0x82, 0x01, 0xe8, 0x02, 0x01,
+	       0x08, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7,
+	       0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x81, 0x88, 0x31,
+	       0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02,
+	       0x47, 0x42, 0x31, 0x17, 0x30, 0x15, 0x06, 0x03, 0x55, 0x04,
+	       0x08, 0x0c, 0x0e, 0x43, 0x61, 0x6d, 0x62, 0x72, 0x69, 0x64,
+	       0x67, 0x65, 0x73, 0x68, 0x69, 0x72, 0x65, 0x31, 0x12, 0x30,
+	       0x10, 0x06, 0x03, 0x55, 0x04, 0x07, 0x0c, 0x09, 0x43, 0x61,
+	       0x6d, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x31, 0x18, 0x30,
+	       0x16, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x0f, 0x46, 0x65,
+	       0x6e, 0x20, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x73, 0x20,
+	       0x4c, 0x74, 0x64, 0x31, 0x11, 0x30, 0x0f, 0x06, 0x03, 0x55,
+	       0x04, 0x0b, 0x0c, 0x08, 0x69, 0x70, 0x78, 0x65, 0x2e, 0x6f,
+	       0x72, 0x67, 0x31, 0x1f, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x04,
+	       0x03, 0x0c, 0x16, 0x69, 0x50, 0x58, 0x45, 0x20, 0x73, 0x65,
+	       0x6c, 0x66, 0x2d, 0x74, 0x65, 0x73, 0x74, 0x20, 0x6c, 0x65,
+	       0x61, 0x66, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, 0x32,
+	       0x34, 0x30, 0x38, 0x32, 0x39, 0x32, 0x32, 0x31, 0x39, 0x35,
+	       0x38, 0x5a, 0x17, 0x0d, 0x32, 0x35, 0x30, 0x38, 0x32, 0x39,
+	       0x32, 0x32, 0x31, 0x39, 0x35, 0x38, 0x5a, 0x30, 0x81, 0x86,
+	       0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13,
+	       0x02, 0x47, 0x42, 0x31, 0x17, 0x30, 0x15, 0x06, 0x03, 0x55,
+	       0x04, 0x08, 0x0c, 0x0e, 0x43, 0x61, 0x6d, 0x62, 0x72, 0x69,
+	       0x64, 0x67, 0x65, 0x73, 0x68, 0x69, 0x72, 0x65, 0x31, 0x12,
+	       0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x07, 0x0c, 0x09, 0x43,
+	       0x61, 0x6d, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x31, 0x18,
+	       0x30, 0x16, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x0f, 0x46,
+	       0x65, 0x6e, 0x20, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x73,
+	       0x20, 0x4c, 0x74, 0x64, 0x31, 0x11, 0x30, 0x0f, 0x06, 0x03,
+	       0x55, 0x04, 0x0b, 0x0c, 0x08, 0x69, 0x70, 0x78, 0x65, 0x2e,
+	       0x6f, 0x72, 0x67, 0x31, 0x1d, 0x30, 0x1b, 0x06, 0x03, 0x55,
+	       0x04, 0x03, 0x0c, 0x14, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	       0x2e, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x69, 0x70, 0x78, 0x65,
+	       0x2e, 0x6f, 0x72, 0x67, 0x30, 0x81, 0x9f, 0x30, 0x0d, 0x06,
+	       0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01,
+	       0x05, 0x00, 0x03, 0x81, 0x8d, 0x00, 0x30, 0x81, 0x89, 0x02,
+	       0x81, 0x81, 0x00, 0xb7, 0x9f, 0xb5, 0x90, 0xfa, 0x47, 0x25,
+	       0xee, 0x3b, 0x04, 0x02, 0x4f, 0xd4, 0x1d, 0x62, 0x46, 0x9d,
+	       0xce, 0x79, 0x3a, 0x80, 0x3a, 0xc4, 0x06, 0x6a, 0x67, 0xd4,
+	       0x3a, 0x61, 0x71, 0xcd, 0xb3, 0xcc, 0x1b, 0xfc, 0x2f, 0x17,
+	       0xa8, 0xf2, 0x26, 0x9e, 0x54, 0xd3, 0x49, 0x81, 0x22, 0xa9,
+	       0x72, 0x4c, 0xe8, 0x92, 0xb7, 0x1e, 0x44, 0x8f, 0xa9, 0x4d,
+	       0x83, 0x0b, 0x89, 0x2a, 0xc7, 0xb3, 0xad, 0x54, 0x32, 0x76,
+	       0x62, 0x1e, 0xe5, 0xbe, 0xc1, 0xa8, 0x7a, 0x2b, 0xf0, 0xa9,
+	       0x9a, 0xfb, 0xb8, 0x18, 0x0c, 0x30, 0x5a, 0x13, 0x9c, 0x21,
+	       0x26, 0x96, 0x4f, 0x39, 0x98, 0x64, 0x41, 0x3b, 0x94, 0xc8,
+	       0xe3, 0xb7, 0x8c, 0x33, 0x9e, 0xd0, 0x71, 0xd0, 0x2f, 0xe3,
+	       0x7d, 0x2c, 0x57, 0x27, 0x42, 0x26, 0xd9, 0x2c, 0xb4, 0x55,
+	       0xd5, 0x66, 0xb7, 0xbf, 0x00, 0xa1, 0xcc, 0x61, 0x19, 0x99,
+	       0x61, 0x02, 0x03, 0x01, 0x00, 0x01, 0x30, 0x0d, 0x06, 0x09,
+	       0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05,
+	       0x00, 0x03, 0x81, 0x81, 0x00, 0x22, 0x6e, 0x8b, 0x81, 0xc3,
+	       0x13, 0x89, 0x31, 0xcc, 0x6d, 0x44, 0xa5, 0x77, 0x31, 0x79,
+	       0xff, 0xea, 0x8a, 0x4e, 0xe6, 0xb6, 0x4a, 0xf0, 0x01, 0xd3,
+	       0x9f, 0xd7, 0x23, 0x14, 0x4c, 0xdd, 0xa3, 0xdf, 0xd6, 0x2b,
+	       0x2a, 0xc4, 0xba, 0xf7, 0xef, 0xfd, 0x78, 0xd5, 0xd7, 0xfd,
+	       0xd7, 0x77, 0x17, 0x20, 0x4f, 0xf3, 0x94, 0xd6, 0x74, 0xd5,
+	       0x09, 0x54, 0x16, 0x49, 0x6e, 0xd9, 0x41, 0xa7, 0x43, 0x77,
+	       0x7f, 0xef, 0xd3, 0xd7, 0xbb, 0x8b, 0xf0, 0x7d, 0x1f, 0x00,
+	       0x4a, 0xac, 0x18, 0xff, 0x20, 0x32, 0xd1, 0x1c, 0x13, 0xf2,
+	       0xe0, 0xd7, 0x70, 0xec, 0xb6, 0xf7, 0xa3, 0x42, 0xa1, 0x64,
+	       0x26, 0x30, 0x9f, 0x47, 0xd4, 0xfb, 0xfb, 0x06, 0xdb, 0x0b,
+	       0xb3, 0x27, 0xe6, 0x04, 0xe2, 0x94, 0x35, 0x8b, 0xa5, 0xfd,
+	       0x80, 0x1d, 0xa6, 0x72, 0x32, 0x30, 0x99, 0x6c, 0xbf, 0xd3,
+	       0x26, 0xa3, 0x8a ) );
+
 /** iPXE self-test root CA certificate */
 static uint8_t root_crt_fingerprint[] =
 	FINGERPRINT ( 0x71, 0x5d, 0x51, 0x37, 0x5e, 0x18, 0xb3, 0xbc,
@@ -1373,6 +1651,28 @@ static void cms_message_okx ( struct cms_test_message *msg,
 #define cms_message_ok( msg ) \
 	cms_message_okx ( msg, __FILE__, __LINE__ )
 
+/**
+ * Report key pair parsing test result
+ *
+ * @v keypair		Test key pair
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void cms_keypair_okx ( struct cms_test_keypair *keypair,
+			      const char *file, unsigned int line ) {
+
+	/* Check ability to parse certificate */
+	okx ( x509_certificate ( keypair->data, keypair->len,
+				 &keypair->cert ) == 0, file, line );
+	okx ( keypair->cert != NULL, file, line );
+
+	/* Check certificate can be identified by public key */
+	okx ( x509_find_key ( NULL, &keypair->privkey ) == keypair->cert,
+	      file, line );
+}
+#define cms_keypair_ok( keypair ) \
+	cms_keypair_okx ( keypair, __FILE__, __LINE__ )
+
 /**
  * Report signature verification test result
  *
@@ -1447,17 +1747,55 @@ static void cms_verify_fail_okx ( struct cms_test_message *msg,
 	cms_verify_fail_okx ( msg, img, name, time, store, root,	\
 			      __FILE__, __LINE__ )
 
+/**
+ * Report decryption test result
+ *
+ * @v img		Encrypted data image
+ * @v envelope		Envelope message
+ * @v keypair		Key pair
+ * @v expected		Expected plaintext image
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void cms_decrypt_okx ( struct cms_test_image *img,
+			      struct cms_test_message *envelope,
+			      struct cms_test_keypair *keypair,
+			      struct cms_test_image *expected,
+			      const char *file, unsigned int line ) {
+	const void *data = ( ( void * ) img->image.data );
+
+	/* Fix up image data pointer */
+	img->image.data = virt_to_user ( data );
+
+	/* Check ability to decrypt image */
+	okx ( cms_decrypt ( envelope->cms, &img->image, NULL,
+			    &keypair->privkey ) == 0, file, line );
+
+	/* Check decrypted image matches expected plaintext */
+	okx ( img->image.len == expected->image.len, file, line );
+	okx ( memcmp_user ( img->image.data, 0, expected->image.data, 0,
+			    expected->image.len ) == 0, file, line );
+}
+#define cms_decrypt_ok( data, envelope, keypair, expected )		\
+	cms_decrypt_okx ( data, envelope, keypair, expected,		\
+			  __FILE__, __LINE__ )
+
 /**
  * Perform CMS self-tests
  *
  */
 static void cms_test_exec ( void ) {
 
+	/* Check that all key pairs can be parsed */
+	cms_keypair_ok ( &client_keypair );
+
 	/* Check that all messages can be parsed */
 	cms_message_ok ( &codesigned_sig );
 	cms_message_ok ( &brokenchain_sig );
 	cms_message_ok ( &genericsigned_sig );
 	cms_message_ok ( &nonsigned_sig );
+	cms_message_ok ( &hidden_code_cbc_env );
+	cms_message_ok ( &hidden_code_gcm_env );
 
 	/* Check good signature */
 	cms_verify_ok ( &codesigned_sig, &test_code, "codesign.test.ipxe.org",
@@ -1494,14 +1832,27 @@ static void cms_test_exec ( void ) {
 	cms_verify_fail_ok ( &codesigned_sig, &test_code,
 			     NULL, test_expired, &empty_store, &test_root );
 
+	/* Check CBC decryption (with padding) */
+	cms_decrypt_ok ( &hidden_code_cbc_dat, &hidden_code_cbc_env,
+			 &client_keypair, &hidden_code );
+
+	/* Check GCM decryption (no padding) */
+	cms_decrypt_ok ( &hidden_code_gcm_dat, &hidden_code_gcm_env,
+			 &client_keypair, &hidden_code );
+
 	/* Sanity check */
 	assert ( list_empty ( &empty_store.links ) );
 
 	/* Drop message references */
+	cms_put ( hidden_code_gcm_env.cms );
+	cms_put ( hidden_code_cbc_env.cms );
 	cms_put ( nonsigned_sig.cms );
 	cms_put ( genericsigned_sig.cms );
 	cms_put ( brokenchain_sig.cms );
 	cms_put ( codesigned_sig.cms );
+
+	/* Drop certificate references */
+	x509_put ( client_keypair.cert );
 }
 
 /** CMS self-test */

From 7f75d320f6d8ac7ec5185b2145da87f698aec273 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 2 Sep 2024 12:24:57 +0100
Subject: [PATCH 188/237] [etherfabric] Fix use of uninitialised variable in
 falcon_xaui_link_ok()

The link status check in falcon_xaui_link_ok() reads from the
FCN_XX_CORE_STAT_REG_MAC register only on production hardware (where
the FPGA version reads as zero), but modifies the value and writes
back to this register unconditionally.  This triggers an uninitialised
variable warning on newer versions of gcc.

Fix by assuming that the register exists only on production hardware,
and so moving the "modify-write" portion of the "read-modify-write"
operation to also be covered by the same conditional check.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/etherfabric.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/drivers/net/etherfabric.c b/src/drivers/net/etherfabric.c
index b40596bea..be30b71f7 100644
--- a/src/drivers/net/etherfabric.c
+++ b/src/drivers/net/etherfabric.c
@@ -2225,13 +2225,16 @@ falcon_xaui_link_ok ( struct efab_nic *efab )
 		sync = ( sync == FCN_XX_SYNC_STAT_DECODE_SYNCED );
 		
 		link_ok = align_done && sync;
-	}
 
-	/* Clear link status ready for next read */
-	EFAB_SET_DWORD_FIELD ( reg, FCN_XX_COMMA_DET, FCN_XX_COMMA_DET_RESET );
-	EFAB_SET_DWORD_FIELD ( reg, FCN_XX_CHARERR, FCN_XX_CHARERR_RESET);
-	EFAB_SET_DWORD_FIELD ( reg, FCN_XX_DISPERR, FCN_XX_DISPERR_RESET);
-	falcon_xmac_writel ( efab, &reg, FCN_XX_CORE_STAT_REG_MAC );
+		/* Clear link status ready for next read */
+		EFAB_SET_DWORD_FIELD ( reg, FCN_XX_COMMA_DET,
+				       FCN_XX_COMMA_DET_RESET );
+		EFAB_SET_DWORD_FIELD ( reg, FCN_XX_CHARERR,
+				       FCN_XX_CHARERR_RESET );
+		EFAB_SET_DWORD_FIELD ( reg, FCN_XX_DISPERR,
+				       FCN_XX_DISPERR_RESET );
+		falcon_xmac_writel ( efab, &reg, FCN_XX_CORE_STAT_REG_MAC );
+	}
 
 	has_phyxs = ( efab->phy_op->mmds & ( 1 << MDIO_MMD_PHYXS ) );
 	if ( link_ok && has_phyxs ) {

From c7f2e75519986f5847ba14ccd0e9b846c138aaaf Mon Sep 17 00:00:00 2001
From: Animesh Bhatt <animeshb@marvell.com>
Date: Mon, 2 Sep 2024 13:45:54 +0100
Subject: [PATCH 189/237] [aqc1xx] Add support for Marvell AQtion Ethernet
 controller

This patch adds support for the AQtion Ethernet controller, enabling
iPXE to recognize and utilize the specific models (AQC114, AQC113, and
AQC107).

Tested-by: Animesh Bhatt <animeshb@marvell.com>
Signed-off-by: Animesh Bhatt <animeshb@marvell.com>
---
 src/Makefile                      |   1 +
 src/drivers/net/marvell/aqc1xx.c  | 629 ++++++++++++++++++++++++++++++
 src/drivers/net/marvell/aqc1xx.h  | 270 +++++++++++++
 src/drivers/net/marvell/atl2_hw.c | 225 +++++++++++
 src/drivers/net/marvell/atl2_hw.h |  94 +++++
 src/drivers/net/marvell/atl_hw.c  | 313 +++++++++++++++
 src/drivers/net/marvell/atl_hw.h  |  83 ++++
 src/include/ipxe/errfile.h        |   3 +
 8 files changed, 1618 insertions(+)
 create mode 100644 src/drivers/net/marvell/aqc1xx.c
 create mode 100644 src/drivers/net/marvell/aqc1xx.h
 create mode 100644 src/drivers/net/marvell/atl2_hw.c
 create mode 100644 src/drivers/net/marvell/atl2_hw.h
 create mode 100644 src/drivers/net/marvell/atl_hw.c
 create mode 100644 src/drivers/net/marvell/atl_hw.h

diff --git a/src/Makefile b/src/Makefile
index bc82cc6fe..95719628f 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -77,6 +77,7 @@ SRCDIRS		+= drivers/net/efi
 SRCDIRS		+= drivers/net/tg3
 SRCDIRS		+= drivers/net/bnxt
 SRCDIRS		+= drivers/net/sfc
+SRCDIRS		+= drivers/net/marvell
 SRCDIRS		+= drivers/block
 SRCDIRS		+= drivers/nvs
 SRCDIRS		+= drivers/bitbash
diff --git a/src/drivers/net/marvell/aqc1xx.c b/src/drivers/net/marvell/aqc1xx.c
new file mode 100644
index 000000000..b108bf75d
--- /dev/null
+++ b/src/drivers/net/marvell/aqc1xx.c
@@ -0,0 +1,629 @@
+/** @file
+ *
+ * Marvell AQtion family network card driver.
+ *
+ * Copyright(C) 2017-2024 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *  TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ *   OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+FILE_LICENCE ( BSD2 );
+
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/malloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/profile.h>
+
+#include "aqc1xx.h"
+
+extern struct atl_hw_ops atl_hw;
+extern struct atl_hw_ops atl2_hw;
+
+/** @file
+*
+* Marvell AQC network card driver
+*
+*/
+
+static int atl_ring_alloc ( const struct atl_nic *nic, struct atl_ring *ring,
+			    uint32_t desc_size, uint32_t reg_base ) {
+	physaddr_t phy_addr;
+
+	/* Allocate ring buffer.*/
+	ring->length = ATL_RING_SIZE * desc_size;
+	ring->ring = dma_alloc ( nic->dma, &ring->map, ring->length,
+				ring->length );
+
+	if ( !ring->ring )
+		return -ENOMEM;
+
+	/* Initialize the descriptor ring */
+	memset ( ring->ring, 0, ring->length );
+
+	/* Program ring address */
+	phy_addr = dma ( &ring->map, ring->ring );
+
+	/* Write ring address (hi & low parts).*/
+	ATL_WRITE_REG ( ( uint32_t )phy_addr, reg_base );
+	ATL_WRITE_REG ( ( uint32_t ) ( ( ( uint64_t )phy_addr ) >> 32 ), reg_base + 4 );
+
+	/* Write ring length.*/
+	ATL_WRITE_REG ( ATL_RING_SIZE, reg_base + 8 );
+
+	ring->sw_head = ring->sw_tail = 0;
+
+	DBGC ( nic, "AQUANTIA: %p ring is at [%08llx,%08llx), reg base %#x\n",
+	    nic, ( ( unsigned long long )phy_addr ),
+	    ( ( unsigned long long ) phy_addr + ring->length ), reg_base );
+
+	return 0;
+}
+
+static void atl_ring_free ( struct atl_ring *ring ) {
+	dma_free ( &ring->map, ring->ring, ring->length );
+	ring->ring = NULL;
+	ring->length = 0;
+}
+
+static void atl_ring_next_dx ( unsigned int *val ) {
+	++( *val );
+	if ( *val == ATL_RING_SIZE )
+		*val = 0;
+}
+
+int atl_ring_full ( const struct atl_ring *ring ) {
+	unsigned int tail = ring->sw_tail;
+	atl_ring_next_dx ( &tail );
+	return tail == ring->sw_head;
+}
+
+void atl_rx_ring_fill ( struct atl_nic *nic ) {
+	struct atl_desc_rx *rx;
+	struct io_buffer *iobuf;
+	physaddr_t address;
+	unsigned int refilled = 0;
+
+	/* Refill ring */
+	while ( !atl_ring_full ( &nic->rx_ring ) ) {
+
+		/* Allocate I/O buffer */
+		iobuf = alloc_rx_iob ( ATL_RX_MAX_LEN, nic->dma );
+		if ( !iobuf ) {
+			/* Wait for next refill */
+			break;
+		}
+
+		/* Get next receive descriptor */
+		rx = ( struct atl_desc_rx * )nic->rx_ring.ring +
+			nic->rx_ring.sw_tail;
+
+		/* Populate receive descriptor */
+		address = iob_dma ( iobuf );
+		rx->data_addr = address;
+		rx->hdr_addr = 0;
+
+		/* Record I/O buffer */
+		assert ( nic->iobufs[nic->rx_ring.sw_tail] == NULL );
+		nic->iobufs[nic->rx_ring.sw_tail] = iobuf;
+
+		DBGC( nic, "AQUANTIA: RX[%d] is [%llx,%llx)\n",
+			nic->rx_ring.sw_tail,
+			( ( unsigned long long )address ),
+			( ( unsigned long long )address + ATL_RX_MAX_LEN ) );
+
+		atl_ring_next_dx ( &nic->rx_ring.sw_tail );
+		refilled++;
+	}
+
+	/* Push descriptors to card, if applicable */
+	if ( refilled ) {
+		wmb();
+		ATL_WRITE_REG ( nic->rx_ring.sw_tail, ATL_RING_TAIL_PTR );
+	}
+}
+
+/**
+* Open network device
+*
+* @v netdev		Network device
+* @ret rc		Return status code
+*/
+static int atl_open ( struct net_device *netdev ) {
+	struct atl_nic *nic = netdev->priv;
+	uint32_t ctrl = 0;
+
+	/* Tx ring */
+	if ( atl_ring_alloc ( nic, &nic->tx_ring, sizeof ( struct atl_desc_tx ),
+				ATL_TX_DMA_DESC_ADDR ) != 0 )
+		goto err_tx_alloc;
+
+	/* Rx ring */
+	if ( atl_ring_alloc ( nic, &nic->rx_ring, sizeof ( struct atl_desc_rx ),
+				ATL_RX_DMA_DESC_ADDR ) != 0 )
+		goto err_rx_alloc;
+
+	/* Allocate interrupt vectors */
+	ATL_WRITE_REG ( ( ATL_IRQ_CTRL_COR_EN | ATL_IRQ_CTRL_REG_RST_DIS ),
+		      ATL_IRQ_CTRL );
+
+	/*TX & RX Interruprt Mapping*/
+	ctrl = ATL_IRQ_MAP_REG1_RX0 | ATL_IRQ_MAP_REG1_RX0_EN |
+		   ATL_IRQ_MAP_REG1_TX0 | ATL_IRQ_MAP_REG1_TX0_EN;
+	ATL_WRITE_REG ( ctrl, ATL_IRQ_MAP_REG1 );
+
+	/*TX interrupt ctrl reg*/
+	ATL_WRITE_REG ( ATL_TX_IRQ_CTRL_WB_EN, ATL_TX_IRQ_CTRL );
+
+	/*RX interrupt ctrl reg*/
+	ATL_WRITE_REG ( ATL_RX_IRQ_CTRL_WB_EN, ATL_RX_IRQ_CTRL );
+
+	/*RX data path*/
+	ctrl = ATL_IRQ_TX | ATL_IRQ_RX;
+	/* itr mask */
+	ATL_WRITE_REG ( ctrl,  ATL_ITR_MSKS );
+	ATL_WRITE_REG ( ( uint32_t )ATL_RX_MAX_LEN / 1024U,
+		      ATL_RX_DMA_DESC_BUF_SIZE );
+
+	/*filter global ctrl */
+	ctrl = ATL_RPF_CTRL1_BRC_EN | ATL_RPF_CTRL1_L2_PROMISC |
+		ATL_RPF_CTRL1_ACTION | ATL_RPF_CTRL1_BRC_TSH;
+	ATL_WRITE_REG ( ctrl, ATL_RPF_CTRL1 );
+
+	/* vlan promisc */
+	ATL_WRITE_REG ( ATL_RPF_CTRL2_VLAN_PROMISC, ATL_RPF_CTRL2 );
+	/* enable rpf2 */
+	ATL_WRITE_REG ( ATL_RPF2_CTRL_EN, ATL_RPF2_CTRL );
+
+	/* RX Packet Buffer 0 Register 1 */
+	ATL_WRITE_REG ( ATL_RPB0_CTRL1_SIZE, ATL_RPB0_CTRL1 );
+
+	/*RX Packet Buffer 0 Register 2 */
+	ctrl = ATL_RPB0_CTRL2_LOW_TSH | ATL_RPB0_CTRL2_HIGH_TSH |
+		ATL_RPB0_CTRL2_FC_EN;
+	ATL_WRITE_REG ( ctrl, ATL_RPB0_CTRL2 );
+
+	/*RPB global ctrl*/
+	ctrl = ATL_READ_REG ( ATL_RPB_CTRL );
+	ctrl |= ( ATL_RPB_CTRL_EN | ATL_RPB_CTRL_FC );
+	ATL_WRITE_REG ( ctrl, ATL_RPB_CTRL );
+
+	/*TX data path*/
+	/* enable tpo2 */
+	ATL_WRITE_REG ( ATL_TPO2_EN, ATL_TPO2_CTRL );
+	/* tpb global ctrl *** */
+	ATL_WRITE_REG ( ATL_TPB0_CTRL1_SIZE, ATL_TPB0_CTRL1 );
+
+	ctrl = ATL_TPB0_CTRL2_LOW_TSH | ATL_TPB0_CTRL2_HIGH_TSH;
+	/* tpb global ctrl *** */
+	ATL_WRITE_REG ( ctrl, ATL_TPB0_CTRL2 );
+
+	ctrl = ATL_READ_REG ( ATL_TPB_CTRL );
+	ctrl |= ( ATL_TPB_CTRL_EN | ATL_TPB_CTRL_PAD_EN );
+	/* tpb global ctrl */
+	ATL_WRITE_REG ( ctrl, ATL_TPB_CTRL );
+
+	/*Enable rings*/
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RING_TX_CTRL ) | ATL_RING_TX_CTRL_EN,
+		      ATL_RING_TX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RING_RX_CTRL ) | ATL_RING_RX_CTRL_EN,
+		      ATL_RING_RX_CTRL );
+
+	if ( nic->flags == ATL_FLAG_A2 ) {
+		ATL_WRITE_REG ( ATL2_RPF_NEW_EN_ADR_EN, ATL2_RPF_NEW_EN_ADR );
+	}
+
+	atl_rx_ring_fill ( nic );
+
+	nic->hw_ops->start ( nic );
+
+	return 0;
+
+err_rx_alloc:
+	atl_ring_free ( &nic->tx_ring );
+
+err_tx_alloc:
+	return -ENOMEM;
+}
+
+/**
+* Close network device
+*
+* @v netdev		Network device
+*/
+static void atl_close ( struct net_device *netdev ) {
+	struct atl_nic *nic = netdev->priv;
+
+	nic->hw_ops->stop ( nic );
+	/* rpb global ctrl */
+	ATL_WRITE_REG ( ATL_RPB_CTRL_DIS, ATL_RPB_CTRL );
+	/* tgb global ctrl */
+	ATL_WRITE_REG ( ATL_TPB_CTRL_DIS, ATL_TPB_CTRL);
+
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RING_TX_CTRL ) | ( ~ATL_RING_TX_CTRL_EN ),
+			ATL_RING_TX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RING_RX_CTRL ) | ( ~ATL_RING_RX_CTRL_EN ),
+			ATL_RING_RX_CTRL );
+
+	/* clear itr mask */
+	ATL_WRITE_REG ( ATL_ITR_MSKS_DIS, ATL_ITR_MSKS );
+
+	/* Reset the NIC */
+	nic->hw_ops->reset ( nic );
+
+	atl_ring_free ( &nic->tx_ring );
+	atl_ring_free ( &nic->rx_ring );
+}
+
+/**
+* Transmit packet
+*
+* @v netdev		Network device
+* @v iobuf		I/O buffer
+* @ret rc		Return status code
+*/
+int atl_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
+	struct atl_nic *nic = netdev->priv;
+	struct atl_desc_tx *tx;
+	physaddr_t address;
+	uint32_t len;
+
+	/* Get next transmit descriptor */
+	if ( atl_ring_full ( &nic->tx_ring ) ) {
+		DBGC ( nic, "AQUANTIA: %p out of transmit descriptors\n", nic );
+		return -ENOBUFS;
+	}
+
+	tx = ( struct atl_desc_tx * )nic->tx_ring.ring + nic->tx_ring.sw_tail;
+
+	/* Populate transmit descriptor */
+	memset ( tx, 0, sizeof ( *tx ) );
+	address = iob_dma ( iobuf );
+	tx->address = address;
+	len = iob_len ( iobuf );
+
+	tx->status = 0x1;
+	tx->status = ( (tx->status) & ~ATL_DESC_TX_BUF_LEN_MASK) |
+			( (len << ATL_DESC_TX_BUF_LEN_OFFSET) &
+			ATL_DESC_TX_BUF_LEN_MASK );
+	tx->status = ((tx->status) & ~ATL_DESC_TX_EOP_MASK) |
+			( (ATL_DESC_TX_DX_EOP_VALUE << ATL_DESC_TX_EOP_OFFSET) &
+			ATL_DESC_TX_EOP_MASK );
+	tx->status = ( (tx->status) & ~ATL_DESC_TX_CMD_MASK) |
+			( (ATL_DESC_TX_CMD_VALUE << ATL_DESC_TX_CMD_OFFSET) &
+			ATL_DESC_TX_CMD_MASK );
+	tx->flag = ( (tx->flag) & ~ATL_DESC_TX_PAY_LEN_MASK) |
+			( (len << ATL_DESC_TX_PAY_LEN_OFFSET) &
+			ATL_DESC_TX_PAY_LEN_MASK );
+	wmb();
+
+	DBGC2 ( nic, "AQUANTIA: %p TX[%d] is [%llx, %llx]\n",
+		nic, nic->tx_ring.sw_tail,
+	    ( ( unsigned long long ) address ),
+	    ( ( unsigned long long ) address + len ) );
+
+	atl_ring_next_dx ( &nic->tx_ring.sw_tail );
+	ATL_WRITE_REG ( nic->tx_ring.sw_tail, ATL_RING_TAIL );
+
+	return 0;
+}
+
+void atl_check_link ( struct net_device *netdev ) {
+	struct atl_nic *nic = netdev->priv;
+	uint32_t link_state;
+
+	/* Read link status */
+	link_state = nic->hw_ops->get_link ( nic );
+
+	DBGC ( nic, "AQUANTIA: %p link status is %08x\n", nic, link_state );
+
+	if ( link_state != nic->link_state ) {
+		if ( link_state ) {
+			DBGC ( nic, "AQUANTIA: link up\n");
+			netdev_link_up ( netdev );
+		} else {
+			DBGC ( nic, "AQUANTIA: link lost\n");
+			netdev_link_down ( netdev );
+		}
+		nic->link_state = link_state;
+	}
+}
+
+/**
+* Poll for completed packets
+*
+* @v netdev		Network device
+*/
+void atl_poll_tx ( struct net_device *netdev ) {
+	struct atl_nic *nic = netdev->priv;
+	struct atl_desc_tx_wb *tx;
+
+	/* Check for completed packets */
+	while ( nic->tx_ring.sw_head != nic->tx_ring.sw_tail ) {
+
+		/* Get next transmit descriptor */
+		tx = ( struct atl_desc_tx_wb * )nic->tx_ring.ring +
+		nic->tx_ring.sw_head;
+
+		/* Stop if descriptor is still in use */
+		if ( !( tx->status & cpu_to_le32 ( ATL_TX_DESC_STATUS_DD ) ) )
+			return;
+
+		DBGC2 ( nic, "AQUANTIA: %p TX[%d] complete\n",
+			nic, nic->tx_ring.sw_head );
+
+		/* Complete TX descriptor */
+		atl_ring_next_dx ( &nic->tx_ring.sw_head );
+		netdev_tx_complete_next ( netdev );
+	}
+}
+
+/**
+* Poll for received packets
+*
+* @v netdev		Network device
+*/
+void atl_poll_rx ( struct net_device *netdev ) {
+	struct atl_nic *nic = netdev->priv;
+	struct atl_desc_rx_wb *rx;
+	struct io_buffer *iobuf;
+	size_t len;
+
+	/* Check for received packets */
+	while ( nic->rx_ring.sw_head != nic->rx_ring.sw_tail ) {
+
+		/* Get next receive descriptor */
+		rx = ( struct atl_desc_rx_wb * )nic->rx_ring.ring +
+		nic->rx_ring.sw_head;
+
+		/* Stop if descriptor is still in use */
+		if ( !( rx->status & cpu_to_le16( ATL_RX_DESC_STATUS_DD ) ) )
+			return;
+
+		/* Populate I/O buffer */
+		iobuf = nic->iobufs[nic->rx_ring.sw_head];
+		nic->iobufs[nic->rx_ring.sw_head] = NULL;
+		len = le16_to_cpu ( rx->pkt_len );
+		iob_put ( iobuf, len );
+
+		/* Hand off to network stack */
+		DBGC ( nic, "AQUANTIA: %p RX[%d] complete (length %zd)\n",
+		    nic, nic->rx_ring.sw_head, len );
+
+		netdev_rx ( netdev, iobuf );
+
+		atl_ring_next_dx ( &nic->rx_ring.sw_head );
+	}
+}
+
+/**
+* Poll for completed and received packets
+*
+* @v netdev		Network device
+*/
+static void atl_poll ( struct net_device *netdev ) {
+	struct atl_nic *nic = netdev->priv;
+
+	/* Check link state */
+	atl_check_link ( netdev );
+
+	/* Poll for TX completions */
+	atl_poll_tx ( netdev );
+
+	/* Poll for RX completions */
+	atl_poll_rx ( netdev );
+
+	/* Refill RX ring */
+	atl_rx_ring_fill ( nic );
+}
+
+/**
+* Enable or disable interrupts
+*
+* @v netdev		Network device
+* @v enable		Interrupts should be enabled
+*/
+static void atl_irq ( struct net_device *netdev, int enable ) {
+	struct atl_nic *nic = netdev->priv;
+	uint32_t mask;
+
+	mask = ( ATL_IRQ_TX | ATL_IRQ_RX );
+	if ( enable )
+		ATL_WRITE_REG ( mask, ATL_ITR_MSKS );
+	else
+		ATL_WRITE_REG ( mask, ATL_ITR_MSKC );
+}
+
+/** Marvell network device operations */
+static struct net_device_operations atl_operations = {
+	.open = atl_open,
+	.close = atl_close,
+	.transmit = atl_transmit,
+	.poll = atl_poll,
+	.irq = atl_irq,
+};
+
+/******************************************************************************
+*
+* PCI interface
+*
+*******************************************************************************
+*/
+
+/**
+* Probe PCI device
+*
+* @v pci		PCI device
+* @ret rc		Return status code
+*/
+static int atl_probe ( struct pci_device *pci ) {
+	struct net_device *netdev;
+	struct atl_nic *nic;
+	int rc = ENOERR;
+	uint32_t io_size = 0;
+
+	/* Allocate and initialise net device */
+	netdev = alloc_etherdev ( sizeof( *nic ) );
+	if ( !netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &atl_operations );
+	nic = netdev->priv;
+	pci_set_drvdata ( pci, netdev );
+	netdev->dev = &pci->dev;
+	memset( nic, 0, sizeof ( *nic ) );
+	nic->flags = pci->id->driver_data;
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	switch ( nic->flags ) {
+		case ATL_FLAG_A1:
+			nic->hw_ops = &atl_hw;
+			io_size = ATL_BAR_SIZE;
+			break;
+		case ATL_FLAG_A2:
+			nic->hw_ops = &atl2_hw;
+			io_size = ATL2_BAR_SIZE;
+			break;
+		default:
+			goto err_unsupported;
+			break;
+	}
+
+	/* Map registers */
+	nic->regs = pci_ioremap ( pci, pci->membase, io_size );
+	if ( !nic->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	 /* Configure DMA */
+	nic->dma = &pci->dma;
+
+	/* Reset the NIC */
+	if ( ( rc = nic->hw_ops->reset ( nic ) ) != 0 )
+		goto err_reset;
+
+	/* Get MAC Address */
+	if ( ( rc = nic->hw_ops->get_mac ( nic, netdev->hw_addr ) ) != 0 )
+		goto err_mac;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register_netdev;
+
+	/* Set initial link state */
+	netdev_link_down ( netdev );
+
+	return 0;
+
+err_register_netdev:
+err_mac:
+	nic->hw_ops->reset ( nic );
+err_reset:
+	iounmap ( nic->regs );
+err_ioremap:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+err_unsupported:
+err_alloc:
+	return rc;
+}
+
+/**
+* Remove PCI device
+*
+* @v pci		PCI device
+*/
+static void atl_remove ( struct pci_device *pci ) {
+	struct net_device *netdev = pci_get_drvdata ( pci );
+	struct atl_nic *nic = netdev->priv;
+
+	/* Unregister network device */
+	unregister_netdev ( netdev );
+
+	/* Reset the NIC */
+	nic->hw_ops->reset ( nic );
+
+	/* Free network device */
+	iounmap ( nic->regs );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** Marvell PCI device IDs */
+static struct pci_device_id atl_nics[] = {
+	/* Atlantic 1 */
+	/* 10G */
+	PCI_ROM ( 0x1D6A, 0x0001, "AQC07", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0xD107, "AQC07", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x07B1, "AQC07", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x87B1, "AQC07", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A1 ),
+
+	/* SFP */
+	PCI_ROM ( 0x1D6A, 0xD100, "AQC00", "Felicity Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x00B1, "AQC00", "Felicity Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x80B1, "AQC00", "Felicity Network Adapter", ATL_FLAG_A1 ),
+
+	/* 5G */
+	PCI_ROM ( 0x1D6A, 0xD108, "AQC08", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x08B1, "AQC08", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x88B1, "AQC08", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x11B1, "AQC11", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x91B1, "AQC11", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1 ),
+
+	/* 2.5G */
+	PCI_ROM ( 0x1D6A, 0xD109, "AQC09", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x09B1, "AQC09", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x89B1, "AQC09", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x12B1, "AQC12", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1 ),
+	PCI_ROM ( 0x1D6A, 0x92B1, "AQC12", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1 ),
+
+	/* Atlantic 2 */
+	PCI_ROM ( 0x1D6A, 0x00C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2 ),
+	PCI_ROM ( 0x1D6A, 0x94C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2 ),
+	PCI_ROM ( 0x1D6A, 0x93C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2 ),
+	PCI_ROM ( 0x1D6A, 0x04C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2 ),
+	PCI_ROM ( 0x1D6A, 0x14C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2 ),
+	PCI_ROM ( 0x1D6A, 0x12C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2 ),
+	PCI_ROM ( 0x1D6A, 0x03C0, "AQC14", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A2 ),
+};
+
+/** Marvell PCI driver */
+struct pci_driver atl_driver __pci_driver = {
+	.ids = atl_nics,
+	.id_count = ( sizeof( atl_nics ) / sizeof ( atl_nics[0] ) ),
+	.probe = atl_probe,
+	.remove = atl_remove,
+};
diff --git a/src/drivers/net/marvell/aqc1xx.h b/src/drivers/net/marvell/aqc1xx.h
new file mode 100644
index 000000000..26bdd00f0
--- /dev/null
+++ b/src/drivers/net/marvell/aqc1xx.h
@@ -0,0 +1,270 @@
+/** @file
+ *
+ * Marvell AQtion family network card driver definitions.
+ *
+ * Copyright(C) 2017-2024 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ *   OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef _ATLANTIC_H
+#define _ATLANTIC_H
+
+FILE_LICENCE ( BSD2 );
+
+#include <stdint.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/nvs.h>
+
+#define ATL_BAR_SIZE   0x10000
+#define ATL2_BAR_SIZE   0x40000
+#define ATL_RING_SIZE  64
+#define ATL_RING_ALIGN 128
+#define ATL_RX_MAX_LEN 2048
+
+#define ATL_IRQ_TX   0x00000001U
+#define ATL_IRQ_RX   0x00000002U
+
+/*IRQ Status Register*/
+#define ATL_IRQ_STAT_REG 0x00002000U
+
+/* Interrupt Vector Allocation Register */
+#define ATL_IRQ_CTRL			 0x00002300U
+#define ATL_IRQ_CTRL_COR_EN	  0x00000080U /*IRQ clear on read */
+#define ATL_IRQ_CTRL_REG_RST_DIS 0x20000000U /*Register reset disable */
+
+/*TX/RX Interruprt Mapping*/
+#define ATL_IRQ_MAP_REG1		0x00002100U /*IRQ mapping register */
+
+#define ATL_IRQ_MAP_REG1_RX0_EN 0x00008000U /*IRQ RX0 enable*/
+#define ATL_IRQ_MAP_REG1_RX0	0x00000100U /*IRQ RX0*/
+
+#define ATL_IRQ_MAP_REG1_TX0_EN 0x80000000U /*IRQ TX0 enable*/
+#define ATL_IRQ_MAP_REG1_TX0	0x00000000U /*IRQ TX0*/
+
+/*TX interrupt ctrl reg*/
+#define ATL_TX_IRQ_CTRL	   0x00007B40U
+#define ATL_TX_IRQ_CTRL_WB_EN 0x00000002U
+
+/*RX interrupt ctrl reg*/
+#define ATL_RX_IRQ_CTRL	   0x00005A30U
+#define ATL_RX_IRQ_CTRL_WB_EN 0x00000004U
+
+#define ATL_GLB_CTRL  0x00000000U
+
+#define ATL_PCI_CTRL		 0x00001000U
+#define ATL_PCI_CTRL_RST_DIS 0x20000000U
+
+#define ATL_RX_CTRL		 0x00005000U
+#define ATL_RX_CTRL_RST_DIS 0x20000000U /*RPB reset disable */
+#define ATL_TX_CTRL		 0x00007000U
+#define ATL_TX_CTRL_RST_DIS 0x20000000U /*TPB reset disable */
+
+/*RX data path control registers*/
+#define ATL_RPF2_CTRL	0x00005040U
+#define ATL_RPF2_CTRL_EN 0x000F0000U /* RPF2 enable*/
+#define ATL2_RPF_NEW_EN_ADR_EN	  0x00000001U /*enable*/
+#define ATL2_RPF_NEW_EN_ADR 0x5104
+
+#define ATL_RPF_CTRL1		0x00005100U
+#define ATL_RPF_CTRL1_BRC_EN	 0x00000001U /*Allow broadcast receive*/
+#define ATL_RPF_CTRL1_L2_PROMISC 0x00000008U /*L2 promiscious*/
+#define ATL_RPF_CTRL1_ACTION	 0x00001000U /*Action to host*/
+#define ATL_RPF_CTRL1_BRC_TSH	0x00010000U /*Brc threshold 256 units per sec*/
+
+#define ATL_RPF_CTRL2		0x00005280U
+#define ATL_RPF_CTRL2_VLAN_PROMISC 0x00000002U /*VLAN promisc*/
+
+#define ATL_RPB_CTRL_DIS      0x0
+#define ATL_RPB_CTRL		 0x00005700U
+#define ATL_RPB_CTRL_EN	  0x00000001U /*RPB Enable*/
+#define ATL_RPB_CTRL_FC	  0x00000010U /*RPB Enable*/
+#define ATL_RPB_CTRL_TC_MODE 0x00000100U /*RPB Traffic Class Mode*/
+
+#define ATL_RPB0_CTRL1	  0x00005710U
+#define ATL_RPB0_CTRL1_SIZE 0x00000140U /*RPB size (in unit 1KB) \*/
+
+#define ATL_RPB0_CTRL2		0x00005714U
+
+/*Buffer Low Threshold (70% of RPB size in unit 32B)*/
+#define ATL_RPB0_CTRL2_LOW_TSH  0x00000C00U
+/*Buffer High Threshold(30% of RPB size in unit 32B)*/
+#define ATL_RPB0_CTRL2_HIGH_TSH 0x1C000000U
+#define ATL_RPB0_CTRL2_FC_EN	0x80000000U /*Flow control Enable*/
+
+#define ATL_RX_DMA_DESC_BUF_SIZE 0x00005b18U
+#define ATL_RX_DMA_DESC_ADDR 0x00005b00U
+
+/*TX data path  control registers*/
+#define ATL_TPO2_CTRL 0x00007040U
+#define ATL_TPO2_EN   0x00010000U /*TPO2 Enable*/
+
+#define ATL_TPB_CTRL_DIS      0x0
+#define ATL_TPB_CTRL		 0x00007900U
+#define ATL_TPB_CTRL_EN	  0x00000001U /*TPB enable*/
+#define ATL_TPB_CTRL_PAD_EN  0x00000004U /*Tx pad insert enable*/
+#define ATL_TPB_CTRL_TC_MODE 0x00000100U /*Tx traffic Class Mode*/
+
+#define ATL_TPB0_CTRL1	  0x00007910U
+#define ATL_TPB0_CTRL1_SIZE 0x000000A0U /*TPB Size (in unit 1KB)*/
+
+#define ATL_TPB0_CTRL2		  0x00007914U
+/*Buffer Low  Threshold(30% of RPB size in unit 32B)*/
+#define ATL_TPB0_CTRL2_LOW_TSH  0x00000600U
+/*Buffer High  Threshold(30% of RPB size in unit 32B)*/
+#define ATL_TPB0_CTRL2_HIGH_TSH 0x0E000000U
+
+#define ATL_TX_DMA_DESC_ADDR 0x00007c00U
+
+/*Rings control registers*/
+#define ATL_RING_TX_CTRL	0x00007c08U
+#define ATL_RING_TX_CTRL_EN 0x80000000U /*Tx descriptor Enable*/
+
+#define ATL_RING_RX_CTRL	0x00005b08U
+#define ATL_RING_RX_CTRL_EN 0x80000000U /*Rx descriptor Enable*/
+
+#define ATL_RING_TAIL	 0x00007c10U
+#define ATL_RING_TAIL_PTR 0x00005b10U
+
+/*IRQ control registers*/
+#define ATL_ITR_MSKS_DIS      0x0
+#define ATL_ITR_MSKS	 0x00002060U
+#define ATL_ITR_MSKS_LSW 0x0000000CU
+#define ATL_ITR_MSKC	 0x00002070U
+#define ATL_ITR_MSKC_LSW 0x0000000CU
+
+/*Link advertising*/
+#define ATL_LINK_ADV		   0x00000368U
+#define ATL_SHUT_LINK      0x0
+#define ATL_LINK_ADV_AUTONEG   0xF20U
+
+#define ATL_LINK_ST	 0x00000370U
+
+/*Semaphores*/
+#define ATL_SEM_RAM 0x000003a8U
+#define ATL_SEM_RAM_RESET 0X1
+
+/*Mailbox*/
+#define ATL_MBOX_ADDR  0x00000360U
+#define ATL_MBOX_CTRL1 0x00000200U
+#define ATL_MBOX_CTRL1_START_MBOX_OPT 0x8000
+
+#define ATL_MBOX_CTRL3 0x00000208U
+#define ATL_MBOX_CTRL5 0x0000020cU
+
+#define ATL_FLAG_A1 0x1
+#define ATL_FLAG_A2 0x2
+
+/*write register*/
+#define ATL_WRITE_REG( VAL, REG )	writel( VAL, nic->regs + (REG) )
+#define ATL_READ_REG( REG )	readl( nic->regs + (REG) ) /*read register*/
+
+struct atl_desc_tx {
+	uint64_t address;
+	uint32_t status;
+	uint32_t flag;
+} __attribute__ (( packed ));
+
+#define ATL_DESC_TX_DX_TYPE_VALUE 0x1
+
+#define  ATL_DESC_TX_DX_EOP_VALUE  0x1
+#define  ATL_DESC_TX_EOP_MASK     0x00200000
+#define  ATL_DESC_TX_EOP_OFFSET   21
+
+#define ATL_DESC_TX_CMD_MASK    0x3FC00000UL
+#define ATL_DESC_TX_CMD_OFFSET  22
+#define ATL_DESC_TX_CMD_VALUE   0x22
+
+#define ATL_DESC_TX_BUF_LEN_MASK   0x000FFFF0
+#define ATL_DESC_TX_BUF_LEN_OFFSET 5
+
+#define ATL_DESC_TX_PAY_LEN_MASK   0xFFFFC000
+#define ATL_DESC_TX_PAY_LEN_OFFSET 14
+
+struct atl_desc_tx_wb {
+	uint64_t rsvd1;
+	uint32_t status;
+	uint32_t rsvd4;
+} __attribute__ (( packed ));
+
+#define ATL_TX_DESC_STATUS_DD  0x00100000UL
+
+struct atl_desc_rx {
+	uint64_t data_addr;
+	uint64_t hdr_addr;
+
+} __attribute__ (( packed ));
+
+struct atl_desc_rx_wb {
+	uint64_t rsvd2;
+	uint16_t status;
+	uint16_t pkt_len;
+	uint32_t rsvd4;
+} __attribute__ (( packed ));
+
+#define ATL_RX_DESC_STATUS_DD  0x0001UL
+#define ATL_RX_DESC_STATUS_EOP 0x0002UL
+struct atl_ring {
+	unsigned int sw_tail;
+	unsigned int sw_head;
+	void *ring;
+	/** Descriptor ring DMA mapping */
+	struct dma_mapping map;
+	unsigned int length;
+};
+
+struct atl_nic;
+
+struct atl_hw_ops {
+	int ( *reset ) ( struct atl_nic *nic );
+	int ( *start ) ( struct atl_nic *nic );
+	int ( *stop ) ( struct atl_nic *nic );
+	int ( *get_link ) ( struct atl_nic *nic );
+	int ( *get_mac ) ( struct atl_nic *, uint8_t *mac );
+};
+
+/** An aQuanita network card */
+struct atl_nic {
+	/** Registers */
+	void *regs;
+	/** Port number (for multi-port devices) */
+	unsigned int port;
+	/** DMA device */
+	struct dma_device *dma;
+	/** Flags */
+	unsigned int flags;
+	struct atl_ring tx_ring;
+	struct atl_ring rx_ring;
+	struct io_buffer *iobufs[ATL_RING_SIZE];
+	uint32_t link_state;
+	uint32_t mbox_addr;
+	struct atl_hw_ops *hw_ops;
+};
+
+struct atl_hw_stats {
+	uint32_t version;
+	uint32_t tid;
+};
+
+#endif /* _AQUANTIA_H */
diff --git a/src/drivers/net/marvell/atl2_hw.c b/src/drivers/net/marvell/atl2_hw.c
new file mode 100644
index 000000000..805820709
--- /dev/null
+++ b/src/drivers/net/marvell/atl2_hw.c
@@ -0,0 +1,225 @@
+/** @file
+ *
+ * Marvell AQtion family network card driver, hardware-specific functions.
+ *
+ * Copyright(C) 2017-2024 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *  TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ *   OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+FILE_LICENCE ( BSD2 );
+
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <byteswap.h>
+#include <ipxe/pci.h>
+#include "aqc1xx.h"
+#include "atl2_hw.h"
+
+static int atl2_hw_boot_completed_ ( struct atl_nic *nic ) {
+	uint32_t reset_status = ATL_READ_REG ( ATL2_GLB_RST_CTRL2 );
+
+	return ( reset_status & ATL2_RESET_STATUS_BOOT_COMPLETED_MASK ) ||
+		   ( ATL_READ_REG ( ATL2_HOST_ITR_REQ )
+		   & ATL2_FW_HOST_INTERRUPT_REQUEST_READY );
+}
+
+void atl2_hw_read_shared_in_ ( struct atl_nic *nic, uint32_t offset,
+								uint32_t *data, uint32_t len ) {
+	uint32_t i;
+
+	for (i = 0; i < len; ++i )
+	{
+		data[i] = ATL_READ_REG ( ATL2_MIF_SHARED_BUF_IN + offset + i * 4 );
+	}
+}
+
+void atl2_hw_write_shared_in_ ( struct atl_nic *nic, uint32_t offset,
+							  uint32_t *data, uint32_t len ) {
+	uint32_t i;
+
+	for ( i = 0; i < len; ++i )
+	{
+		ATL_WRITE_REG ( data[i], ATL2_MIF_SHARED_BUF_IN + offset + i * 4 );
+	}
+}
+
+int atl2_hw_finish_ack_ ( struct atl_nic *nic, uint32_t ms ) {
+	uint32_t i;
+	int err = 0;
+
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL2_HOST_FINISHED_WRITE )
+	| 1, ATL2_HOST_FINISHED_WRITE );
+
+	for ( i = 0; i < ( ms / 100 ); ++i )
+	{
+		if ( ( ATL_READ_REG ( ATL2_MCP_BUSY_WRITE ) & 1 ) == 0 )
+		{
+			break;
+		}
+		udelay ( ATL2_DELAY_100 );
+	}
+	if (i == ( ms / 100 ) )
+		err = -ETIME;
+
+	return err;
+}
+
+int atl2_hw_fw_init_ ( struct atl_nic *nic ) {
+	uint32_t val;
+	int err = 0;
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_LINK_CTRL_IN_OFF, &val, 1 );
+	val |= ( ATL2_HOST_MODE_ACTIVE | ( 1U << 13 ) );
+	atl2_hw_write_shared_in_ ( nic, ATL2_LINK_CTRL_IN_OFF, &val, 1 );
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_MTU_IN_OFF, &val, 1 );
+	val = 16352;
+	atl2_hw_write_shared_in_ ( nic, ATL2_MTU_IN_OFF, &val, 1 );
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+	val = 0;
+	atl2_hw_write_shared_in_( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+	err = atl2_hw_finish_ack_ ( nic, 50000000 );
+
+	return err;
+}
+
+int atl2_hw_reset ( struct atl_nic *nic ) {
+	int completed = 0;
+	uint32_t status = 0;
+	uint32_t request;
+	int err = 0;
+	int i;
+
+	request = ATL2_RESET_STATUS_REQ_GSR;
+
+	ATL_WRITE_REG ( request, ATL2_GLB_RST_CTRL2 );
+
+	/* Wait for boot code started every 10us, 200 ms */
+	for ( i = 0; i < 20000; ++i )
+	{
+		status = ATL_READ_REG ( ATL2_GLB_RST_CTRL2 );
+
+		if ( ( ( status & ATL2_RESET_STATUS_BC_STARTED ) &&
+			 ( status != 0xFFFFFFFFu ) ) )
+			break;
+
+		udelay ( ATL2_DELAY_10 );
+	}
+	if ( i == 20000 )
+	{
+		DBGC ( nic, "Boot code hanged" );
+		err = -EIO;
+		goto err_exit;
+	}
+
+	/* Wait for boot succeed, failed or host request every 10us, 480ms */
+	for ( i = 0; i < 48000; ++i )
+	{
+		completed = atl2_hw_boot_completed_ ( nic );
+		if ( completed )
+			break;
+
+		udelay ( ATL2_DELAY_10 );
+	}
+
+	if ( !completed )
+	{
+		DBGC ( nic, "FW Restart timed out" );
+		err = -ETIME;
+		goto err_exit;
+	}
+
+	status = ATL_READ_REG ( ATL2_GLB_RST_CTRL2 );
+
+	if ( status & ATL2_RESET_STATUS_BOOT_FAILED_MASK )
+	{
+		err = -EIO;
+		DBGC ( nic, "FW Restart failed" );
+		DBGC ( nic, "status = 0x%x", status );
+		goto err_exit;
+	}
+
+	if ( ATL_READ_REG ( ATL2_HOST_ITR_REQ )
+			& ATL2_FW_HOST_INTERRUPT_REQUEST_READY )
+	{
+		err = -ENOTSUP;
+		DBGC ( nic, "Dynamic FW load not implemented" );
+		goto err_exit;
+	}
+
+	err = atl2_hw_fw_init_ ( nic );
+
+err_exit:
+	return err;
+}
+
+int atl2_hw_start ( struct atl_nic *nic ) {
+	uint32_t val;
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+	val = 0x4B00FFE1;
+	atl2_hw_write_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+
+	return atl2_hw_finish_ack_ ( nic, 100000 );
+}
+
+int atl2_hw_stop ( struct atl_nic *nic ) {
+	uint32_t val;
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+	val = 0;
+	atl2_hw_write_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+
+	return atl2_hw_finish_ack_ ( nic, 100000 );
+}
+
+int atl2_hw_get_link ( struct atl_nic *nic ) {
+	uint32_t val;
+
+	val = ATL_READ_REG ( ATL2_MIF_SHARED_BUF_OUT + ATL2_LINK_STS_OUT_OFF );
+
+	return ( ( val & 0xf ) != 0 ) && ( ( val & 0xF0 ) != 0 );
+}
+
+int atl2_hw_get_mac ( struct atl_nic *nic, uint8_t *mac ) {
+	uint32_t mac_addr[2] = {0};
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_MAC_ADDR_IN_OFF, mac_addr, 2 );
+
+	memcpy ( mac, ( uint8_t * )mac_addr, 6 );
+
+	return 0;
+}
+
+struct atl_hw_ops atl2_hw = {
+	.reset = atl2_hw_reset,
+	.start = atl2_hw_start,
+	.stop = atl2_hw_stop,
+	.get_link = atl2_hw_get_link,
+	.get_mac = atl2_hw_get_mac,
+};
diff --git a/src/drivers/net/marvell/atl2_hw.h b/src/drivers/net/marvell/atl2_hw.h
new file mode 100644
index 000000000..d044c21fa
--- /dev/null
+++ b/src/drivers/net/marvell/atl2_hw.h
@@ -0,0 +1,94 @@
+/*
+ * Copyright(C) 2017-2024 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *   TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ *   OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __ATL2_HW_H
+#define __ATL2_HW_H
+
+FILE_LICENCE ( BSD2 );
+
+#define ATL2_GLB_RST_CTRL2 0x3040
+#define ATL2_HOST_FINISHED_WRITE 0xE00
+#define ATL2_MCP_BUSY_WRITE 0xE04
+#define ATL2_HOST_ITR_REQ 0xF00
+
+
+#define ATL2_RESET_STATUS_REQ_GSR		( 1U << 0x0 )
+#define ATL2_RESET_STATUS_REQ_HOST_BOOT		( 1U << 0x8 )
+#define ATL2_RESET_STATUS_REQ_MAC_FAST_BOOT	( 1U << 0xA )
+#define ATL2_RESET_STATUS_REQ_PHY_FAST_BOOT	( 1U << 0xB )
+
+#define ATL2_RESET_STATUS_HOST_LOAD_COMPLETED	( 1U << 0x10 )
+#define ATL2_RESET_STATUS_REQUIRE_HOST_LOAD	( 1U << 0x11 )
+#define ATL2_RESET_STATUS_BC_STARTED		( 1U << 0x18 )
+#define ATL2_RESET_STATUS_CRASH_DURING_INIT	( 1U << 0x1B )
+#define ATL2_RESET_STATUS_BC_FAILED		( 1U << 0x1C )
+#define ATL2_RESET_STATUS_FW_FAILED		( 1U << 0x1D )
+#define ATL2_RESET_STATUS_FW_SUCCEED		( 1U << 0x1F )
+
+#define ATL2_RESET_STATUS_BOOT_FAILED_MASK ( ATL2_RESET_STATUS_CRASH_DURING_INIT | ATL2_RESET_STATUS_BC_FAILED | ATL2_RESET_STATUS_FW_FAILED )
+#define ATL2_RESET_STATUS_BOOT_COMPLETED_MASK ( ATL2_RESET_STATUS_BOOT_FAILED_MASK | ATL2_RESET_STATUS_FW_SUCCEED )
+
+#define ATL2_FW_HOST_INTERRUPT_REQUEST_READY		0x0001
+#define ATL2_FW_HOST_INTERRUPT_MAC_READY		0x0004
+#define ATL2_FW_HOST_INTERRUPT_DATA_HANDLED		0x0100
+#define ATL2_FW_HOST_INTERRUPT_LINK_UP			0x0200
+#define ATL2_FW_HOST_INTERRUPT_LINK_DOWN		0x0400
+#define ATL2_FW_HOST_INTERRUPT_PHY_FAULT		0x0800
+#define ATL2_FW_HOST_INTERRUPT_MAC_FAULT		0x1000
+#define ATL2_FW_HOST_INTERRUPT_TEMPERATURE_WARNING	0x2000
+#define ATL2_FW_HOST_INTERRUPT_HEARTBEAT		0x4000
+
+#define ATL2_FW_LINK_RATE_INVALID 0
+#define ATL2_FW_LINK_RATE_10M     1
+#define ATL2_FW_LINK_RATE_100M    2
+#define ATL2_FW_LINK_RATE_1G      3
+#define ATL2_FW_LINK_RATE_2G5     4
+#define ATL2_FW_LINK_RATE_5G      5
+#define ATL2_FW_LINK_RATE_10G     6
+
+#define ATL2_HOST_MODE_INVALID      0U
+#define ATL2_HOST_MODE_ACTIVE       1U
+#define ATL2_HOST_MODE_SLEEP_PROXY  2U
+#define ATL2_HOST_MODE_LOW_POWER    3U
+#define ATL2_HOST_MODE_SHUTDOWN     4U
+
+#define ATL2_MIF_SHARED_BUF_IN 0x12000
+#define ATL2_MIF_SHARED_BUF_OUT 0x13000
+
+#define ATL2_MTU_IN_OFF 0x0
+#define ATL2_MAC_ADDR_IN_OFF 0x8
+#define ATL2_LINK_CTRL_IN_OFF 0x10
+#define ATL2_LINK_OPTS_IN_OFF 0x18
+
+#define ATL2_FW_OUT_OFF 0x8
+#define ATL2_LINK_STS_OUT_OFF 0x14
+
+#define ATL2_DELAY_10  10
+#define ATL2_DELAY_100  100
+
+#endif
diff --git a/src/drivers/net/marvell/atl_hw.c b/src/drivers/net/marvell/atl_hw.c
new file mode 100644
index 000000000..e0843e6f4
--- /dev/null
+++ b/src/drivers/net/marvell/atl_hw.c
@@ -0,0 +1,313 @@
+/** @file
+ *
+ * Marvell AQtion family network card driver, hardware-specific functions.
+ *
+ * Copyright(C) 2017-2024 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *  TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ *   OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+FILE_LICENCE ( BSD2 );
+
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <byteswap.h>
+#include <ipxe/pci.h>
+#include "aqc1xx.h"
+#include "atl_hw.h"
+#include <compiler.h>
+
+
+int atl_hw_reset_flb_ ( struct atl_nic *nic ) {
+	uint32_t val;
+	int k = 0;
+
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_STALL, ATL_GLB_CTRL2 );
+	mdelay ( ATL_DELAY_50_MNS );
+
+	/* Cleanup SPI */
+	val = ATL_READ_REG ( ATL_GLB_NVR_PROV4 );
+	ATL_WRITE_REG ( val | ATL_GBL_NVR_PROV4_RESET, ATL_GLB_NVR_PROV4 );
+
+	ATL_WRITE_REG( ( ATL_READ_REG ( ATL_GLB_STD_CTRL ) &
+		      ~ATL_GLB_CTRL_RST_DIS ) | ATL_GLB_STD_CTRL_RESET,
+			ATL_GLB_STD_CTRL );
+
+	/* Kickstart MAC */
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_FW_RESET, ATL_GLB_CTRL2 );
+	ATL_WRITE_REG ( ATL_MIF_PWR_GATING_EN_CTRL_RESET,
+			ATL_MIF_PWR_GATING_EN_CTRL );
+
+	ATL_WRITE_REG ( ATL_GEN_PROV9_ENABLE, ATL_GEN_PROV9 );
+
+	/* Reset SPI again because of possible interrupted SPI burst */
+	val = ATL_READ_REG ( ATL_GLB_NVR_PROV4 );
+	ATL_WRITE_REG ( val | ATL_GBL_NVR_PROV4_RESET, ATL_GLB_NVR_PROV4 );
+	mdelay ( ATL_DELAY_10_MNS );
+	/* Clear SPI reset state */
+	ATL_WRITE_REG ( val & ~ATL_GBL_NVR_PROV4_RESET, ATL_GLB_NVR_PROV4 );
+
+	/* MAC Kickstart */
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_MAC_KICK_START, ATL_GLB_CTRL2 );
+
+	for (k = 0; k < 1000; k++) {
+		uint32_t flb_status = ATL_READ_REG ( ATL_MPI_DAISY_CHAIN_STS );
+
+		flb_status = flb_status & FLB_LOAD_STS;
+		if ( flb_status )
+			break;
+		mdelay ( ATL_DELAY_10_MNS );
+	}
+	if ( k == 1000 ) {
+		DBGC (nic, "MAC kickstart failed\n" );
+		return -EIO;
+	}
+
+	/* FW reset */
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_FW_RESET, ATL_GLB_CTRL2 );
+	mdelay ( ATL_DELAY_50_MNS );
+
+	ATL_WRITE_REG ( ATL_GBL_MCP_SEM1_RELEASE, ATL_GLB_MCP_SEM1 );
+
+	/* Global software reset*/
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RX_CTRL ) &
+			~ATL_RX_CTRL_RST_DIS, ATL_RX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_TX_CTRL ) &
+			~ATL_TX_CTRL_RST_DIS, ATL_TX_CTRL );
+
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_MAC_PHY_CTRL ) &
+			~ATL_MAC_PHY_CTRL_RST_DIS, ATL_MAC_PHY_CTRL );
+
+	ATL_WRITE_REG ( ( ATL_READ_REG ( ATL_GLB_STD_CTRL ) &
+			~ATL_GLB_CTRL_RST_DIS ) | ATL_GLB_STD_CTRL_RESET,
+			ATL_GLB_STD_CTRL );
+
+	for (k = 0; k < 1000; k++) {
+		u32 fw_state = ATL_READ_REG ( ATL_FW_VER );
+
+		if ( fw_state )
+			break;
+		mdelay ( ATL_DELAY_10_MNS );
+	}
+	if ( k == 1000 ) {
+		DBGC ( nic, "FW kickstart failed\n" );
+		return -EIO;
+	}
+	/* Old FW requires fixed delay after init */
+	mdelay ( ATL_DELAY_15_MNS );
+
+	return 0;
+}
+
+int atl_hw_reset_rbl_ ( struct atl_nic *nic ) {
+	uint32_t val, rbl_status;
+	int k;
+
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_STALL, ATL_GLB_CTRL2 );
+	ATL_WRITE_REG ( ATL_GBL_MCP_SEM1_RELEASE, ATL_GLB_MCP_SEM1 );
+	ATL_WRITE_REG ( ATL_MIF_PWR_GATING_EN_CTRL_RESET,
+			ATL_MIF_PWR_GATING_EN_CTRL );
+
+	/* Alter RBL status */
+	ATL_WRITE_REG ( POISON_SIGN, ATL_MPI_BOOT_EXIT_CODE );
+
+	/* Cleanup SPI */
+	val = ATL_READ_REG ( ATL_GLB_NVR_PROV4 );
+	ATL_WRITE_REG ( val | ATL_GBL_NVR_PROV4_RESET, ATL_GLB_NVR_PROV4 );
+
+	/* Global software reset*/
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RX_CTRL ) & ~ATL_RX_CTRL_RST_DIS,
+			ATL_RX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_TX_CTRL ) & ~ATL_TX_CTRL_RST_DIS,
+			ATL_TX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_MAC_PHY_CTRL ) &
+			~ATL_MAC_PHY_CTRL_RST_DIS, ATL_MAC_PHY_CTRL );
+
+	ATL_WRITE_REG ( ( ATL_READ_REG ( ATL_GLB_STD_CTRL ) &
+			~ATL_GLB_CTRL_RST_DIS ) | ATL_GLB_STD_CTRL_RESET,
+			ATL_GLB_STD_CTRL );
+
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_NORMAL, ATL_GLB_CTRL2 );
+
+	/* Wait for RBL boot */
+	for ( k = 0; k < 1000; k++ ) {
+		rbl_status = ATL_READ_REG ( ATL_MPI_BOOT_EXIT_CODE ) & 0xFFFF;
+		if ( rbl_status && rbl_status != POISON_SIGN )
+			break;
+		mdelay ( ATL_DELAY_10_MNS );
+	}
+	if ( !rbl_status || rbl_status == POISON_SIGN ) {
+		DBGC ( nic, "RBL Restart failed\n" );
+		return -EIO;
+	}
+
+	if ( rbl_status == FW_NOT_SUPPORT )
+		return -ENOTSUP;
+
+	for ( k = 0; k < 1000; k++ ) {
+		u32 fw_state = ATL_READ_REG ( ATL_FW_VER );
+
+		if ( fw_state )
+			break;
+		mdelay ( ATL_DELAY_10_MNS );
+	}
+	if ( k == 1000 ) {
+		DBGC ( nic, "FW kickstart failed\n" );
+		return -EIO;
+	}
+	/* Old FW requires fixed delay after init */
+	mdelay ( ATL_DELAY_15_MNS );
+
+	return 0;
+}
+
+int atl_hw_reset ( struct atl_nic *nic ) {
+	uint32_t boot_exit_code = 0;
+	uint32_t k;
+	int rbl_enabled;
+	uint32_t fw_ver;
+	uint32_t sem_timeout;
+
+	for ( k = 0; k < 1000; ++k ) {
+		uint32_t flb_status = ATL_READ_REG ( ATL_MPI_DAISY_CHAIN_STS );
+		boot_exit_code = ATL_READ_REG ( ATL_MPI_BOOT_EXIT_CODE );
+		if ( flb_status != ATL_MPI_DAISY_CHAIN_STS_ERROR_STATUS ||
+			boot_exit_code != 0 )
+			break;
+	}
+
+	if ( k == 1000 ) {
+		DBGC ( nic, "Neither RBL nor FLB firmware started\n" );
+		return -ENOTSUP;
+	}
+
+	rbl_enabled = ( boot_exit_code != 0 );
+
+	fw_ver = ATL_READ_REG ( ATL_FW_VER );
+	if ( ( ( fw_ver >> 24 ) & 0xFF ) >= 4 ) {
+		sem_timeout = ATL_READ_REG ( ATL_SEM_TIMEOUT );
+		if ( sem_timeout > ATL_SEM_MAX_TIMEOUT )
+			sem_timeout = ATL_SEM_MAX_TIMEOUT;
+
+		for ( k = 0; k < sem_timeout; ++k ) {
+			if ( ATL_READ_REG ( ATL_GLB_MCP_SEM4 ) )
+				break;
+
+			mdelay ( ATL_DELAY_1_MNS );
+		}
+		for ( k = 0; k < sem_timeout; ++k ) {
+			if ( ATL_READ_REG ( ATL_GLB_MCP_SEM5 ) )
+				break;
+
+			mdelay ( ATL_DELAY_1_MNS );
+		}
+	}
+
+
+	if ( rbl_enabled )
+		return atl_hw_reset_rbl_ ( nic );
+	else
+		return atl_hw_reset_flb_ ( nic );
+}
+
+int atl_hw_start ( struct atl_nic *nic ) {
+	ATL_WRITE_REG ( ATL_LINK_ADV_AUTONEG, ATL_LINK_ADV );
+	return 0;
+}
+
+int atl_hw_stop ( struct atl_nic *nic ) {
+	ATL_WRITE_REG ( ATL_SHUT_LINK, ATL_LINK_ADV );
+	return 0;
+}
+
+int atl_hw_get_link ( struct atl_nic *nic ) {
+	return ( ATL_READ_REG ( ATL_LINK_ST) & ATL_LINK_ADV_AUTONEG ) != 0;
+}
+
+int atl_hw_read_mem ( struct atl_nic *nic, uint32_t addr, uint32_t *buffer,
+		    uint32_t size ) {
+	uint32_t i;
+
+	for ( i = 0; i < 100; ++i ) {
+		if ( ATL_READ_REG( ATL_SEM_RAM ) )
+			break;
+		mdelay ( ATL_DELAY_1_MNS );
+	}
+	if ( i == 100 ) {
+		DBGC ( nic, "Semaphore Register not set\n" );
+		return -EIO;
+	}
+
+	ATL_WRITE_REG ( addr, ATL_MBOX_CTRL3 );
+
+	for ( i = 0; i < size; ++i, addr += 4 ) {
+		uint32_t j;
+
+		ATL_WRITE_REG ( ATL_MBOX_CTRL1_START_MBOX_OPT, ATL_MBOX_CTRL1 );
+		for ( j = 0; j < 10000; ++j ) {
+			if ( ATL_READ_REG (ATL_MBOX_CTRL3 ) != addr )
+				break;
+			udelay ( ATL_DELAY_10_MNS );
+		}
+		if ( j == 10000 ) {
+			DBGC ( nic, "Reading from CTRL3 Register Failed\n" );
+			return -EIO;
+		}
+
+		buffer[i] = ATL_READ_REG ( ATL_MBOX_CTRL5 );
+	}
+
+	ATL_WRITE_REG( ATL_SEM_RAM_RESET, ATL_SEM_RAM );
+
+	return 0;
+}
+
+int atl_hw_get_mac ( struct atl_nic *nic, uint8_t *mac ) {
+	uint32_t mac_addr[2] = {0};
+	int err = 0;
+	uint32_t efuse_addr = ATL_READ_REG ( ATL_GLB_MCP_SP26 );
+
+	if ( efuse_addr != 0) {
+		uint32_t mac_efuse_addr = efuse_addr + 40 * sizeof ( uint32_t );
+		err = atl_hw_read_mem ( nic, mac_efuse_addr, mac_addr, 2 );
+		if ( err != 0 )
+			return err;
+
+		mac_addr[0] = cpu_to_be32 ( mac_addr[0] );
+		mac_addr[1] = cpu_to_be32 ( mac_addr[1] );
+
+		memcpy ( mac, ( uint8_t * )mac_addr, ATL_MAC_ADDRESS_SIZE );
+	}
+	return 0;
+}
+
+struct atl_hw_ops atl_hw = {
+	.reset = atl_hw_reset,
+	.start = atl_hw_start,
+	.stop = atl_hw_stop,
+	.get_link = atl_hw_get_link,
+	.get_mac = atl_hw_get_mac,
+};
diff --git a/src/drivers/net/marvell/atl_hw.h b/src/drivers/net/marvell/atl_hw.h
new file mode 100644
index 000000000..efc9f86c1
--- /dev/null
+++ b/src/drivers/net/marvell/atl_hw.h
@@ -0,0 +1,83 @@
+/*
+ * Copyright(C) 2017-2024 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *   TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ *   OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef __ATL_HW_H
+#define __ATL_HW_H
+
+FILE_LICENCE ( BSD2 );
+
+#define ATL_GLB_STD_CTRL 0x0
+#define ATL_GLB_CTRL_RST_DIS 0x4000
+#define ATL_FW_VER 0x18
+
+#define ATL_MPI_DAISY_CHAIN_STS 0x704
+#define ATL_MPI_RX_DAISY_CHAIN_DATA 0x04000000
+#define ATL_MPI_RX_DAISY_CHAIN_SOF  0x02000000
+#define FLB_LOAD_STS 0x10
+
+#define ATL_MPI_BOOT_EXIT_CODE 0x388
+
+#define ATL_SEM_TIMEOUT 0x348
+#define ATL_SEM_MAX_TIMEOUT 3000
+
+#define ATL_GLB_CTRL2 0x404
+#define ATL_GLB_MCP_SEM1 0x3A0
+#define ATL_GBL_MCP_SEM1_RELEASE 0x1
+
+#define ATL_GLB_MCP_SEM4 0x3AC
+#define ATL_GLB_MCP_SEM5 0x3B0
+#define ATL_GLB_MCP_SP26 0x364
+#define ATL_MIF_PWR_GATING_EN_CTRL 0x32A8
+
+#define ATL_GLB_NVR_PROV4       0x53C
+#define ATL_GBL_NVR_PROV4_RESET 0x10
+
+
+#define ATL_GEN_PROV9 0x520
+
+#define ATL_MAC_PHY_CTRL 0x00004000U
+#define ATL_MAC_PHY_CTRL_RST_DIS 0x20000000U
+
+#define ATL_MIF_PWR_GATING_EN_CTRL_RESET      0x0
+#define ATL_GEN_PROV9_ENABLE                  0x1
+#define ATL_GLB_CTRL2_MAC_KICK_START          0x180e0
+#define ATL_GLB_CTRL2_FW_RESET                0x80e0
+#define ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_STALL   0x40e1
+#define ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_NORMAL  0x40e0
+#define ATL_GLB_STD_CTRL_RESET                0x8000
+#define ATL_MPI_DAISY_CHAIN_STS_ERROR_STATUS  0x06000000
+
+#define ATL_DELAY_1_MNS   1
+#define ATL_DELAY_10_MNS  10
+#define ATL_DELAY_15_MNS  15
+#define ATL_DELAY_50_MNS  50
+
+#define ATL_MAC_ADDRESS_SIZE 6
+#define POISON_SIGN 0xDEAD
+#define FW_NOT_SUPPORT 0xF1A7
+
+#endif
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 7615075bc..42bc17789 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -225,6 +225,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_pcibridge	     ( ERRFILE_DRIVER | 0x00d40000 )
 #define ERRFILE_mnpnet		     ( ERRFILE_DRIVER | 0x00d50000 )
 #define ERRFILE_gve		     ( ERRFILE_DRIVER | 0x00d60000 )
+#define ERRFILE_aqc1xx		     ( ERRFILE_DRIVER | 0x00d70000 )
+#define ERRFILE_atl_hw		     ( ERRFILE_DRIVER | 0x00d80000 )
+#define ERRFILE_atl2_hw		     ( ERRFILE_DRIVER | 0x00d90000 )
 
 #define ERRFILE_aoe			( ERRFILE_NET | 0x00000000 )
 #define ERRFILE_arp			( ERRFILE_NET | 0x00010000 )

From 804f35cb5a6ac7d35912822372b74fd8856f850b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 3 Sep 2024 14:56:07 +0100
Subject: [PATCH 190/237] [build] Centralise dummy architecture-specific
 headers

Simplify the process of adding a new CPU architecture by providing
common implementations of typically empty architecture-specific header
files.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/Makefile.housekeeping                 |  5 ----
 src/arch/arm/Makefile                     |  4 +--
 src/arch/arm/include/bits/acpi.h          | 12 ---------
 src/arch/arm/include/bits/hyperv.h        | 12 ---------
 src/arch/arm/include/bits/iomap.h         | 12 ---------
 src/arch/arm/include/bits/mp.h            | 12 ---------
 src/arch/arm/include/bits/pci_io.h        | 12 ---------
 src/arch/arm/include/bits/reboot.h        | 12 ---------
 src/arch/arm/include/bits/sanboot.h       | 12 ---------
 src/arch/arm/include/bits/smbios.h        | 12 ---------
 src/arch/arm/include/bits/time.h          | 12 ---------
 src/arch/arm/include/bits/uaccess.h       | 12 ---------
 src/arch/arm/include/bits/uart.h          | 12 ---------
 src/arch/arm/include/bits/umalloc.h       | 12 ---------
 src/arch/arm32/include/bits/tcpip.h       | 19 --------------
 src/arch/loong64/Makefile                 |  3 +++
 src/arch/loong64/include/bits/acpi.h      | 12 ---------
 src/arch/loong64/include/bits/hyperv.h    | 12 ---------
 src/arch/loong64/include/bits/iomap.h     | 12 ---------
 src/arch/loong64/include/bits/mp.h        | 12 ---------
 src/arch/loong64/include/bits/pci_io.h    | 12 ---------
 src/arch/loong64/include/bits/reboot.h    | 12 ---------
 src/arch/loong64/include/bits/sanboot.h   | 12 ---------
 src/arch/loong64/include/bits/smbios.h    | 12 ---------
 src/arch/loong64/include/bits/tcpip.h     | 19 --------------
 src/arch/loong64/include/bits/time.h      | 12 ---------
 src/arch/loong64/include/bits/uaccess.h   | 12 ---------
 src/arch/loong64/include/bits/uart.h      | 12 ---------
 src/arch/loong64/include/bits/umalloc.h   | 12 ---------
 src/arch/x86/Makefile                     |  4 +--
 src/include/bits/acpi.h                   | 15 +++++++++++
 src/include/bits/hyperv.h                 | 15 +++++++++++
 src/include/bits/iomap.h                  | 15 +++++++++++
 src/include/bits/mp.h                     | 15 +++++++++++
 src/include/bits/nap.h                    | 15 +++++++++++
 src/include/bits/pci_io.h                 | 15 +++++++++++
 src/include/bits/reboot.h                 | 15 +++++++++++
 src/include/bits/sanboot.h                | 15 +++++++++++
 src/include/bits/smbios.h                 | 15 +++++++++++
 src/include/bits/tcpip.h                  | 32 +++++++++++++++++++++++
 src/include/bits/time.h                   | 15 +++++++++++
 src/include/bits/uaccess.h                | 15 +++++++++++
 src/include/bits/uart.h                   | 15 +++++++++++
 src/include/bits/umalloc.h                | 15 +++++++++++
 src/{arch/loong64 => }/include/bits/xen.h |  5 +++-
 45 files changed, 238 insertions(+), 336 deletions(-)
 delete mode 100644 src/arch/arm/include/bits/acpi.h
 delete mode 100644 src/arch/arm/include/bits/hyperv.h
 delete mode 100644 src/arch/arm/include/bits/iomap.h
 delete mode 100644 src/arch/arm/include/bits/mp.h
 delete mode 100644 src/arch/arm/include/bits/pci_io.h
 delete mode 100644 src/arch/arm/include/bits/reboot.h
 delete mode 100644 src/arch/arm/include/bits/sanboot.h
 delete mode 100644 src/arch/arm/include/bits/smbios.h
 delete mode 100644 src/arch/arm/include/bits/time.h
 delete mode 100644 src/arch/arm/include/bits/uaccess.h
 delete mode 100644 src/arch/arm/include/bits/uart.h
 delete mode 100644 src/arch/arm/include/bits/umalloc.h
 delete mode 100644 src/arch/arm32/include/bits/tcpip.h
 delete mode 100644 src/arch/loong64/include/bits/acpi.h
 delete mode 100644 src/arch/loong64/include/bits/hyperv.h
 delete mode 100644 src/arch/loong64/include/bits/iomap.h
 delete mode 100644 src/arch/loong64/include/bits/mp.h
 delete mode 100644 src/arch/loong64/include/bits/pci_io.h
 delete mode 100644 src/arch/loong64/include/bits/reboot.h
 delete mode 100644 src/arch/loong64/include/bits/sanboot.h
 delete mode 100644 src/arch/loong64/include/bits/smbios.h
 delete mode 100644 src/arch/loong64/include/bits/tcpip.h
 delete mode 100644 src/arch/loong64/include/bits/time.h
 delete mode 100644 src/arch/loong64/include/bits/uaccess.h
 delete mode 100644 src/arch/loong64/include/bits/uart.h
 delete mode 100644 src/arch/loong64/include/bits/umalloc.h
 create mode 100644 src/include/bits/acpi.h
 create mode 100644 src/include/bits/hyperv.h
 create mode 100644 src/include/bits/iomap.h
 create mode 100644 src/include/bits/mp.h
 create mode 100644 src/include/bits/nap.h
 create mode 100644 src/include/bits/pci_io.h
 create mode 100644 src/include/bits/reboot.h
 create mode 100644 src/include/bits/sanboot.h
 create mode 100644 src/include/bits/smbios.h
 create mode 100644 src/include/bits/tcpip.h
 create mode 100644 src/include/bits/time.h
 create mode 100644 src/include/bits/uaccess.h
 create mode 100644 src/include/bits/uart.h
 create mode 100644 src/include/bits/umalloc.h
 rename src/{arch/loong64 => }/include/bits/xen.h (50%)

diff --git a/src/Makefile.housekeeping b/src/Makefile.housekeeping
index d13cb3671..5d9c03930 100644
--- a/src/Makefile.housekeeping
+++ b/src/Makefile.housekeeping
@@ -366,11 +366,6 @@ MAKEDEPS	+= arch/$(ARCH)/Makefile
 include arch/$(ARCH)/Makefile
 endif
 
-# Include architecture-specific include path
-ifdef ARCH
-INCDIRS		+= arch/$(ARCH)/include
-endif
-
 ###############################################################################
 #
 # Especially ugly workarounds
diff --git a/src/arch/arm/Makefile b/src/arch/arm/Makefile
index b6509dda0..f827ed422 100644
--- a/src/arch/arm/Makefile
+++ b/src/arch/arm/Makefile
@@ -3,9 +3,9 @@
 ASM_TCHAR	:= %
 ASM_TCHAR_OPS	:= %%
 
-# Include common ARM headers
+# Include ARM-specific headers
 #
-INCDIRS		+= arch/arm/include
+INCDIRS		:= arch/$(ARCH)/include arch/arm/include $(INCDIRS)
 
 # ARM-specific directories containing source files
 #
diff --git a/src/arch/arm/include/bits/acpi.h b/src/arch/arm/include/bits/acpi.h
deleted file mode 100644
index f9f2f00e7..000000000
--- a/src/arch/arm/include/bits/acpi.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_ACPI_H
-#define _BITS_ACPI_H
-
-/** @file
- *
- * ARM-specific ACPI API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_ACPI_H */
diff --git a/src/arch/arm/include/bits/hyperv.h b/src/arch/arm/include/bits/hyperv.h
deleted file mode 100644
index f0e0c8793..000000000
--- a/src/arch/arm/include/bits/hyperv.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_HYPERV_H
-#define _BITS_HYPERV_H
-
-/** @file
- *
- * Hyper-V interface
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_HYPERV_H */
diff --git a/src/arch/arm/include/bits/iomap.h b/src/arch/arm/include/bits/iomap.h
deleted file mode 100644
index ae953c450..000000000
--- a/src/arch/arm/include/bits/iomap.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_IOMAP_H
-#define _BITS_IOMAP_H
-
-/** @file
- *
- * ARM-specific I/O mapping API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_IOMAP_H */
diff --git a/src/arch/arm/include/bits/mp.h b/src/arch/arm/include/bits/mp.h
deleted file mode 100644
index e7d4c0c16..000000000
--- a/src/arch/arm/include/bits/mp.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_MP_H
-#define _BITS_MP_H
-
-/** @file
- *
- * ARM-specific multiprocessor API implementation
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_MP_H */
diff --git a/src/arch/arm/include/bits/pci_io.h b/src/arch/arm/include/bits/pci_io.h
deleted file mode 100644
index 91f507a44..000000000
--- a/src/arch/arm/include/bits/pci_io.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_PCI_IO_H
-#define _BITS_PCI_IO_H
-
-/** @file
- *
- * ARM PCI I/O API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_PCI_IO_H */
diff --git a/src/arch/arm/include/bits/reboot.h b/src/arch/arm/include/bits/reboot.h
deleted file mode 100644
index 88c50250c..000000000
--- a/src/arch/arm/include/bits/reboot.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_REBOOT_H
-#define _BITS_REBOOT_H
-
-/** @file
- *
- * ARM-specific reboot API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_REBOOT_H */
diff --git a/src/arch/arm/include/bits/sanboot.h b/src/arch/arm/include/bits/sanboot.h
deleted file mode 100644
index abd4c79a5..000000000
--- a/src/arch/arm/include/bits/sanboot.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_SANBOOT_H
-#define _BITS_SANBOOT_H
-
-/** @file
- *
- * ARM-specific sanboot API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_SANBOOT_H */
diff --git a/src/arch/arm/include/bits/smbios.h b/src/arch/arm/include/bits/smbios.h
deleted file mode 100644
index d94218116..000000000
--- a/src/arch/arm/include/bits/smbios.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_SMBIOS_H
-#define _BITS_SMBIOS_H
-
-/** @file
- *
- * ARM-specific SMBIOS API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_SMBIOS_H */
diff --git a/src/arch/arm/include/bits/time.h b/src/arch/arm/include/bits/time.h
deleted file mode 100644
index 724d8b932..000000000
--- a/src/arch/arm/include/bits/time.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_TIME_H
-#define _BITS_TIME_H
-
-/** @file
- *
- * ARM-specific time API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_TIME_H */
diff --git a/src/arch/arm/include/bits/uaccess.h b/src/arch/arm/include/bits/uaccess.h
deleted file mode 100644
index 87f11509c..000000000
--- a/src/arch/arm/include/bits/uaccess.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_UACCESS_H
-#define _BITS_UACCESS_H
-
-/** @file
- *
- * ARM-specific user access API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_UACCESS_H */
diff --git a/src/arch/arm/include/bits/uart.h b/src/arch/arm/include/bits/uart.h
deleted file mode 100644
index 6f85975f7..000000000
--- a/src/arch/arm/include/bits/uart.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_UART_H
-#define _BITS_UART_H
-
-/** @file
- *
- * 16550-compatible UART
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_UART_H */
diff --git a/src/arch/arm/include/bits/umalloc.h b/src/arch/arm/include/bits/umalloc.h
deleted file mode 100644
index 27970d7b2..000000000
--- a/src/arch/arm/include/bits/umalloc.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_UMALLOC_H
-#define _BITS_UMALLOC_H
-
-/** @file
- *
- * ARM-specific user memory allocation API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_UMALLOC_H */
diff --git a/src/arch/arm32/include/bits/tcpip.h b/src/arch/arm32/include/bits/tcpip.h
deleted file mode 100644
index fc3c5b3ff..000000000
--- a/src/arch/arm32/include/bits/tcpip.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef _BITS_TCPIP_H
-#define _BITS_TCPIP_H
-
-/** @file
- *
- * Transport-network layer interface
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-static inline __attribute__ (( always_inline )) uint16_t
-tcpip_continue_chksum ( uint16_t partial, const void *data, size_t len ) {
-
-	/* Not yet optimised */
-	return generic_tcpip_continue_chksum ( partial, data, len );
-}
-
-#endif /* _BITS_TCPIP_H */
diff --git a/src/arch/loong64/Makefile b/src/arch/loong64/Makefile
index fd0bf137f..90d0ec8e1 100644
--- a/src/arch/loong64/Makefile
+++ b/src/arch/loong64/Makefile
@@ -18,6 +18,9 @@ endif
 # EFI requires -fshort-wchar, and nothing else currently uses wchar_t
 CFLAGS		+= -fshort-wchar
 
+# Include LoongArch64-specific headers
+INCDIRS		:= arch/$(ARCH)/include $(INCDIRS)
+
 # LoongArch64-specific directories containing source files
 SRCDIRS		+= arch/loong64/core
 SRCDIRS		+= arch/loong64/interface/efi
diff --git a/src/arch/loong64/include/bits/acpi.h b/src/arch/loong64/include/bits/acpi.h
deleted file mode 100644
index 83dd1df9a..000000000
--- a/src/arch/loong64/include/bits/acpi.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_ACPI_H
-#define _BITS_ACPI_H
-
-/** @file
- *
- * LoongArch64-specific ACPI API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_ACPI_H */
diff --git a/src/arch/loong64/include/bits/hyperv.h b/src/arch/loong64/include/bits/hyperv.h
deleted file mode 100644
index f0e0c8793..000000000
--- a/src/arch/loong64/include/bits/hyperv.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_HYPERV_H
-#define _BITS_HYPERV_H
-
-/** @file
- *
- * Hyper-V interface
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_HYPERV_H */
diff --git a/src/arch/loong64/include/bits/iomap.h b/src/arch/loong64/include/bits/iomap.h
deleted file mode 100644
index 041171d22..000000000
--- a/src/arch/loong64/include/bits/iomap.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_IOMAP_H
-#define _BITS_IOMAP_H
-
-/** @file
- *
- * LoongArch64-specific I/O mapping API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_IOMAP_H */
diff --git a/src/arch/loong64/include/bits/mp.h b/src/arch/loong64/include/bits/mp.h
deleted file mode 100644
index fef2fd59a..000000000
--- a/src/arch/loong64/include/bits/mp.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_MP_H
-#define _BITS_MP_H
-
-/** @file
- *
- * LoongArch64-specific multiprocessor API implementation
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_MP_H */
diff --git a/src/arch/loong64/include/bits/pci_io.h b/src/arch/loong64/include/bits/pci_io.h
deleted file mode 100644
index fdc5141cf..000000000
--- a/src/arch/loong64/include/bits/pci_io.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_PCI_IO_H
-#define _BITS_PCI_IO_H
-
-/** @file
- *
- * LoongArch64-specific PCI I/O API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_PCI_IO_H */
diff --git a/src/arch/loong64/include/bits/reboot.h b/src/arch/loong64/include/bits/reboot.h
deleted file mode 100644
index 96a1eb1ce..000000000
--- a/src/arch/loong64/include/bits/reboot.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_REBOOT_H
-#define _BITS_REBOOT_H
-
-/** @file
- *
- * LoongArch64-specific reboot API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_REBOOT_H */
diff --git a/src/arch/loong64/include/bits/sanboot.h b/src/arch/loong64/include/bits/sanboot.h
deleted file mode 100644
index f9205e2ad..000000000
--- a/src/arch/loong64/include/bits/sanboot.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_SANBOOT_H
-#define _BITS_SANBOOT_H
-
-/** @file
- *
- * LoongArch64-specific sanboot API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_SANBOOT_H */
diff --git a/src/arch/loong64/include/bits/smbios.h b/src/arch/loong64/include/bits/smbios.h
deleted file mode 100644
index 6c87db430..000000000
--- a/src/arch/loong64/include/bits/smbios.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_SMBIOS_H
-#define _BITS_SMBIOS_H
-
-/** @file
- *
- * LoongArch64-specific SMBIOS API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_SMBIOS_H */
diff --git a/src/arch/loong64/include/bits/tcpip.h b/src/arch/loong64/include/bits/tcpip.h
deleted file mode 100644
index fc3c5b3ff..000000000
--- a/src/arch/loong64/include/bits/tcpip.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef _BITS_TCPIP_H
-#define _BITS_TCPIP_H
-
-/** @file
- *
- * Transport-network layer interface
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-static inline __attribute__ (( always_inline )) uint16_t
-tcpip_continue_chksum ( uint16_t partial, const void *data, size_t len ) {
-
-	/* Not yet optimised */
-	return generic_tcpip_continue_chksum ( partial, data, len );
-}
-
-#endif /* _BITS_TCPIP_H */
diff --git a/src/arch/loong64/include/bits/time.h b/src/arch/loong64/include/bits/time.h
deleted file mode 100644
index 4cd7485cf..000000000
--- a/src/arch/loong64/include/bits/time.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_TIME_H
-#define _BITS_TIME_H
-
-/** @file
- *
- * LoongArch64-specific time API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_TIME_H */
diff --git a/src/arch/loong64/include/bits/uaccess.h b/src/arch/loong64/include/bits/uaccess.h
deleted file mode 100644
index dddd9be04..000000000
--- a/src/arch/loong64/include/bits/uaccess.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_UACCESS_H
-#define _BITS_UACCESS_H
-
-/** @file
- *
- * LoongArch64-specific user access API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_UACCESS_H */
diff --git a/src/arch/loong64/include/bits/uart.h b/src/arch/loong64/include/bits/uart.h
deleted file mode 100644
index 6f85975f7..000000000
--- a/src/arch/loong64/include/bits/uart.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_UART_H
-#define _BITS_UART_H
-
-/** @file
- *
- * 16550-compatible UART
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_UART_H */
diff --git a/src/arch/loong64/include/bits/umalloc.h b/src/arch/loong64/include/bits/umalloc.h
deleted file mode 100644
index f6978b8bd..000000000
--- a/src/arch/loong64/include/bits/umalloc.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_UMALLOC_H
-#define _BITS_UMALLOC_H
-
-/** @file
- *
- * LoongArch64-specific user memory allocation API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_UMALLOC_H */
diff --git a/src/arch/x86/Makefile b/src/arch/x86/Makefile
index ef801365e..4a4d8ee91 100644
--- a/src/arch/x86/Makefile
+++ b/src/arch/x86/Makefile
@@ -3,9 +3,9 @@
 ASM_TCHAR	:= @
 ASM_TCHAR_OPS	:= @
 
-# Include common x86 headers
+# Include x86-specific headers
 #
-INCDIRS		+= arch/x86/include
+INCDIRS		:= arch/$(ARCH)/include arch/x86/include $(INCDIRS)
 
 # x86-specific directories containing source files
 #
diff --git a/src/include/bits/acpi.h b/src/include/bits/acpi.h
new file mode 100644
index 000000000..d98420502
--- /dev/null
+++ b/src/include/bits/acpi.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_ACPI_H
+#define _BITS_ACPI_H
+
+/** @file
+ *
+ * Dummy architecture-specific ACPI API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_ACPI_H */
diff --git a/src/include/bits/hyperv.h b/src/include/bits/hyperv.h
new file mode 100644
index 000000000..fd7ceeea3
--- /dev/null
+++ b/src/include/bits/hyperv.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_HYPERV_H
+#define _BITS_HYPERV_H
+
+/** @file
+ *
+ * Dummy architecture-specific Hyper-V interface
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_HYPERV_H */
diff --git a/src/include/bits/iomap.h b/src/include/bits/iomap.h
new file mode 100644
index 000000000..b3d8d0c0d
--- /dev/null
+++ b/src/include/bits/iomap.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_IOMAP_H
+#define _BITS_IOMAP_H
+
+/** @file
+ *
+ * Dummy architecture-specific I/O mapping API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_IOMAP_H */
diff --git a/src/include/bits/mp.h b/src/include/bits/mp.h
new file mode 100644
index 000000000..d7d2cdc28
--- /dev/null
+++ b/src/include/bits/mp.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_MP_H
+#define _BITS_MP_H
+
+/** @file
+ *
+ * Dummy architecture-specific multiprocessor API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_MP_H */
diff --git a/src/include/bits/nap.h b/src/include/bits/nap.h
new file mode 100644
index 000000000..7b01f94e6
--- /dev/null
+++ b/src/include/bits/nap.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_NAP_H
+#define _BITS_NAP_H
+
+/** @file
+ *
+ * Dummy architecture-specific CPU sleeping API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_NAP_H */
diff --git a/src/include/bits/pci_io.h b/src/include/bits/pci_io.h
new file mode 100644
index 000000000..711449f19
--- /dev/null
+++ b/src/include/bits/pci_io.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_PCI_IO_H
+#define _BITS_PCI_IO_H
+
+/** @file
+ *
+ * Dummy architecture-specific PCI I/O API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_PCI_IO_H */
diff --git a/src/include/bits/reboot.h b/src/include/bits/reboot.h
new file mode 100644
index 000000000..023647fd6
--- /dev/null
+++ b/src/include/bits/reboot.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_REBOOT_H
+#define _BITS_REBOOT_H
+
+/** @file
+ *
+ * Dummy architecture-specific reboot API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_REBOOT_H */
diff --git a/src/include/bits/sanboot.h b/src/include/bits/sanboot.h
new file mode 100644
index 000000000..addbb5b07
--- /dev/null
+++ b/src/include/bits/sanboot.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_SANBOOT_H
+#define _BITS_SANBOOT_H
+
+/** @file
+ *
+ * Dummy architecture-specific sanboot API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_SANBOOT_H */
diff --git a/src/include/bits/smbios.h b/src/include/bits/smbios.h
new file mode 100644
index 000000000..3e06aadb4
--- /dev/null
+++ b/src/include/bits/smbios.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_SMBIOS_H
+#define _BITS_SMBIOS_H
+
+/** @file
+ *
+ * Dummy architecture-specific SMBIOS API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_SMBIOS_H */
diff --git a/src/include/bits/tcpip.h b/src/include/bits/tcpip.h
new file mode 100644
index 000000000..9cb54d432
--- /dev/null
+++ b/src/include/bits/tcpip.h
@@ -0,0 +1,32 @@
+#ifndef _BITS_TCPIP_H
+#define _BITS_TCPIP_H
+
+/** @file
+ *
+ * Generic architecture-specific transport-network layer interface
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/**
+ * Calculate continued TCP/IP checkum
+ *
+ * @v partial		Checksum of already-summed data, in network byte order
+ * @v data		Data buffer
+ * @v len		Length of data buffer
+ * @ret cksum		Updated checksum, in network byte order
+ */
+static inline __attribute__ (( always_inline )) uint16_t
+tcpip_continue_chksum ( uint16_t partial, const void *data, size_t len ) {
+
+	/* Not yet optimised */
+	return generic_tcpip_continue_chksum ( partial, data, len );
+}
+
+#endif /* _BITS_TCPIP_H */
diff --git a/src/include/bits/time.h b/src/include/bits/time.h
new file mode 100644
index 000000000..aec9c2f39
--- /dev/null
+++ b/src/include/bits/time.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_TIME_H
+#define _BITS_TIME_H
+
+/** @file
+ *
+ * Dummy architecture-specific time API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_TIME_H */
diff --git a/src/include/bits/uaccess.h b/src/include/bits/uaccess.h
new file mode 100644
index 000000000..09f5f46c8
--- /dev/null
+++ b/src/include/bits/uaccess.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_UACCESS_H
+#define _BITS_UACCESS_H
+
+/** @file
+ *
+ * Dummy architecture-specific user access API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_UACCESS_H */
diff --git a/src/include/bits/uart.h b/src/include/bits/uart.h
new file mode 100644
index 000000000..e132d5c3d
--- /dev/null
+++ b/src/include/bits/uart.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_UART_H
+#define _BITS_UART_H
+
+/** @file
+ *
+ * Dummy architecture-specific UART
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_UART_H */
diff --git a/src/include/bits/umalloc.h b/src/include/bits/umalloc.h
new file mode 100644
index 000000000..4927f0d00
--- /dev/null
+++ b/src/include/bits/umalloc.h
@@ -0,0 +1,15 @@
+#ifndef _BITS_UMALLOC_H
+#define _BITS_UMALLOC_H
+
+/** @file
+ *
+ * Dummy architecture-specific user memory allocation API implementations
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#endif /* _BITS_UMALLOC_H */
diff --git a/src/arch/loong64/include/bits/xen.h b/src/include/bits/xen.h
similarity index 50%
rename from src/arch/loong64/include/bits/xen.h
rename to src/include/bits/xen.h
index 2a3d7741c..7d0036bc0 100644
--- a/src/arch/loong64/include/bits/xen.h
+++ b/src/include/bits/xen.h
@@ -3,7 +3,10 @@
 
 /** @file
  *
- * Xen interface
+ * Dummy architecture-specific Xen interface
+ *
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
  *
  */
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

From 2b82007571dbe02b85338dc9fa85251cd5d68028 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 4 Sep 2024 13:29:30 +0100
Subject: [PATCH 191/237] [gdb] Allow CPU architectures to omit support for GDB

Move the <gdbmach.h> file to <bits/gdbmach.h>, and provide a common
dummy implementation for all architectures that have not yet
implemented support for GDB.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm64/include/gdbmach.h              | 45 -------------------
 src/arch/i386/include/{ => bits}/gdbmach.h    |  0
 src/arch/loong64/include/gdbmach.h            | 45 -------------------
 src/arch/x86/core/gdbmach.c                   |  1 -
 src/arch/x86_64/include/{ => bits}/gdbmach.h  |  0
 src/core/gdbstub.c                            |  1 -
 .../arm32/include => include/bits}/gdbmach.h  | 12 ++---
 src/include/ipxe/gdbstub.h                    |  2 +-
 8 files changed, 7 insertions(+), 99 deletions(-)
 delete mode 100644 src/arch/arm64/include/gdbmach.h
 rename src/arch/i386/include/{ => bits}/gdbmach.h (100%)
 delete mode 100644 src/arch/loong64/include/gdbmach.h
 rename src/arch/x86_64/include/{ => bits}/gdbmach.h (100%)
 rename src/{arch/arm32/include => include/bits}/gdbmach.h (76%)

diff --git a/src/arch/arm64/include/gdbmach.h b/src/arch/arm64/include/gdbmach.h
deleted file mode 100644
index cd152eedd..000000000
--- a/src/arch/arm64/include/gdbmach.h
+++ /dev/null
@@ -1,45 +0,0 @@
-#ifndef GDBMACH_H
-#define GDBMACH_H
-
-/** @file
- *
- * GDB architecture specifics
- *
- * This file declares functions for manipulating the machine state and
- * debugging context.
- *
- */
-
-#include <stdint.h>
-
-typedef unsigned long gdbreg_t;
-
-/* Register snapshot */
-enum {
-	/* Not yet implemented */
-	GDBMACH_NREGS,
-};
-
-#define GDBMACH_SIZEOF_REGS ( GDBMACH_NREGS * sizeof ( gdbreg_t ) )
-
-static inline void gdbmach_set_pc ( gdbreg_t *regs, gdbreg_t pc ) {
-	/* Not yet implemented */
-	( void ) regs;
-	( void ) pc;
-}
-
-static inline void gdbmach_set_single_step ( gdbreg_t *regs, int step ) {
-	/* Not yet implemented */
-	( void ) regs;
-	( void ) step;
-}
-
-static inline void gdbmach_breakpoint ( void ) {
-	/* Not yet implemented */
-}
-
-extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len,
-				    int enable );
-extern void gdbmach_init ( void );
-
-#endif /* GDBMACH_H */
diff --git a/src/arch/i386/include/gdbmach.h b/src/arch/i386/include/bits/gdbmach.h
similarity index 100%
rename from src/arch/i386/include/gdbmach.h
rename to src/arch/i386/include/bits/gdbmach.h
diff --git a/src/arch/loong64/include/gdbmach.h b/src/arch/loong64/include/gdbmach.h
deleted file mode 100644
index cd152eedd..000000000
--- a/src/arch/loong64/include/gdbmach.h
+++ /dev/null
@@ -1,45 +0,0 @@
-#ifndef GDBMACH_H
-#define GDBMACH_H
-
-/** @file
- *
- * GDB architecture specifics
- *
- * This file declares functions for manipulating the machine state and
- * debugging context.
- *
- */
-
-#include <stdint.h>
-
-typedef unsigned long gdbreg_t;
-
-/* Register snapshot */
-enum {
-	/* Not yet implemented */
-	GDBMACH_NREGS,
-};
-
-#define GDBMACH_SIZEOF_REGS ( GDBMACH_NREGS * sizeof ( gdbreg_t ) )
-
-static inline void gdbmach_set_pc ( gdbreg_t *regs, gdbreg_t pc ) {
-	/* Not yet implemented */
-	( void ) regs;
-	( void ) pc;
-}
-
-static inline void gdbmach_set_single_step ( gdbreg_t *regs, int step ) {
-	/* Not yet implemented */
-	( void ) regs;
-	( void ) step;
-}
-
-static inline void gdbmach_breakpoint ( void ) {
-	/* Not yet implemented */
-}
-
-extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len,
-				    int enable );
-extern void gdbmach_init ( void );
-
-#endif /* GDBMACH_H */
diff --git a/src/arch/x86/core/gdbmach.c b/src/arch/x86/core/gdbmach.c
index af6abfedd..d4d187e35 100644
--- a/src/arch/x86/core/gdbmach.c
+++ b/src/arch/x86/core/gdbmach.c
@@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/uaccess.h>
 #include <ipxe/gdbstub.h>
 #include <librm.h>
-#include <gdbmach.h>
 
 /** @file
  *
diff --git a/src/arch/x86_64/include/gdbmach.h b/src/arch/x86_64/include/bits/gdbmach.h
similarity index 100%
rename from src/arch/x86_64/include/gdbmach.h
rename to src/arch/x86_64/include/bits/gdbmach.h
diff --git a/src/core/gdbstub.c b/src/core/gdbstub.c
index 8b57ddf56..d668f454a 100644
--- a/src/core/gdbstub.c
+++ b/src/core/gdbstub.c
@@ -36,7 +36,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ctype.h>
 #include <byteswap.h>
 #include <ipxe/gdbstub.h>
-#include "gdbmach.h"
 
 enum {
 	POSIX_EINVAL = 0x1c,  /* used to report bad arguments to GDB */
diff --git a/src/arch/arm32/include/gdbmach.h b/src/include/bits/gdbmach.h
similarity index 76%
rename from src/arch/arm32/include/gdbmach.h
rename to src/include/bits/gdbmach.h
index cd152eedd..7e349856f 100644
--- a/src/arch/arm32/include/gdbmach.h
+++ b/src/include/bits/gdbmach.h
@@ -1,12 +1,12 @@
-#ifndef GDBMACH_H
-#define GDBMACH_H
+#ifndef _BITS_GDBMACH_H
+#define _BITS_GDBMACH_H
 
 /** @file
  *
- * GDB architecture specifics
+ * Dummy GDB architecture specifics
  *
- * This file declares functions for manipulating the machine state and
- * debugging context.
+ * This file is included only if the architecture does not provide its
+ * own version of this file.
  *
  */
 
@@ -42,4 +42,4 @@ extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len,
 				    int enable );
 extern void gdbmach_init ( void );
 
-#endif /* GDBMACH_H */
+#endif /* _BITS_GDBMACH_H */
diff --git a/src/include/ipxe/gdbstub.h b/src/include/ipxe/gdbstub.h
index 13ca33ddb..2240cca42 100644
--- a/src/include/ipxe/gdbstub.h
+++ b/src/include/ipxe/gdbstub.h
@@ -11,7 +11,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
-#include <gdbmach.h>
+#include <bits/gdbmach.h>
 
 /**
  * A transport mechanism for the GDB protocol

From f88761ef491c16a33078e46ad9d49ebd8f36fd47 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 5 Sep 2024 14:01:19 +0100
Subject: [PATCH 192/237] [ena] Change reported operating system type to "iPXE"

As described in commit 3b81a4e ("[ena] Provide a host information
page"), we currently report an operating system type of "Linux" in
order to work around broken versions of the ENA firmware that will
fail to create a completion queue if we report the correct operating
system type.

As of September 2024, the ENA team at AWS assures us that the entire
AWS fleet has been upgraded to fix this bug, and that we are now safe
to report the correct operating system type value in the "type" field
of struct ena_host_info.

The ENA team has also clarified that at least some deployed versions
of the ENA firmware still have the defect that requires us to report
an operating system version number of 2 (regardless of operating
system type), and so we continue to report ENA_HOST_INFO_VERSION_WTF
in the "version" field of struct ena_host_info.

Add an explicit warning on the previous known failure path, in case
some deployed versions of the ENA firmware turn out to not have been
upgraded as expected.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/ena.c |  7 +++++--
 src/drivers/net/ena.h | 15 +++++++++------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/drivers/net/ena.c b/src/drivers/net/ena.c
index 7ce5b9eb9..89483eae1 100644
--- a/src/drivers/net/ena.c
+++ b/src/drivers/net/ena.c
@@ -560,8 +560,11 @@ static int ena_create_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
 	req->create_cq.address = cpu_to_le64 ( virt_to_bus ( cq->cqe.raw ) );
 
 	/* Issue request */
-	if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
+	if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
+		DBGC ( ena, "ENA %p CQ%d creation failed (broken firmware?)\n",
+		       ena, cq->id );
 		goto err_admin;
+	}
 
 	/* Parse response */
 	cq->id = le16_to_cpu ( rsp->create_cq.id );
@@ -1163,7 +1166,7 @@ static int ena_probe ( struct pci_device *pci ) {
 	}
 	ena->info = info;
 	memset ( info, 0, PAGE_SIZE );
-	info->type = cpu_to_le32 ( ENA_HOST_INFO_TYPE_LINUX );
+	info->type = cpu_to_le32 ( ENA_HOST_INFO_TYPE_IPXE );
 	snprintf ( info->dist_str, sizeof ( info->dist_str ), "%s",
 		   ( product_name[0] ? product_name : product_short_name ) );
 	snprintf ( info->kernel_str, sizeof ( info->kernel_str ), "%s",
diff --git a/src/drivers/net/ena.h b/src/drivers/net/ena.h
index 0f280c700..9fda9979f 100644
--- a/src/drivers/net/ena.h
+++ b/src/drivers/net/ena.h
@@ -191,14 +191,17 @@ struct ena_host_info {
 	uint32_t features;
 } __attribute__ (( packed ));
 
-/** Linux operating system type
+/** Operating system type
  *
- * There is a defined "iPXE" operating system type (with value 5).
- * However, some very broken versions of the ENA firmware will refuse
- * to allow a completion queue to be created if the "iPXE" type is
- * used.
+ * Some very broken older versions of the ENA firmware will refuse to
+ * allow a completion queue to be created if "iPXE" (type 5) is used,
+ * and require us to pretend that we are "Linux" (type 1) instead.
+ *
+ * The ENA team at AWS assures us that the entire AWS fleet has been
+ * upgraded to fix this bug, and that we are now safe to use the
+ * correct operating system type value.
  */
-#define ENA_HOST_INFO_TYPE_LINUX 1
+#define ENA_HOST_INFO_TYPE_IPXE 5
 
 /** Driver version
  *

From 49f9e036fffa33671a3ad16ab5fc0644a057218c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 6 Sep 2024 15:09:12 +0100
Subject: [PATCH 193/237] [cloud] Add family and architecture tags to AWS
 snapshots and images

Allow for easier identification of images and snapshots created by the
aws-import script by adding tags for image family (e.g. "iPXE") and
architecture (e.g. "x86_64") to both.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 contrib/cloud/aws-import | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/contrib/cloud/aws-import b/contrib/cloud/aws-import
index ace005870..64b299622 100755
--- a/contrib/cloud/aws-import
+++ b/contrib/cloud/aws-import
@@ -22,11 +22,12 @@ def detect_architecture(image):
     return 'x86_64'
 
 
-def create_snapshot(region, description, image):
+def create_snapshot(region, description, image, tags):
     """Create an EBS snapshot"""
     client = boto3.client('ebs', region_name=region)
     snapshot = client.start_snapshot(VolumeSize=1,
-                                     Description=description)
+                                     Description=description,
+                                     Tags=tags)
     snapshot_id = snapshot['SnapshotId']
     with open(image, 'rb') as fh:
         for block in count():
@@ -46,11 +47,15 @@ def create_snapshot(region, description, image):
     return snapshot_id
 
 
-def import_image(region, name, architecture, image, public, overwrite):
+def import_image(region, name, family, architecture, image, public, overwrite):
     """Import an AMI image"""
     client = boto3.client('ec2', region_name=region)
     resource = boto3.resource('ec2', region_name=region)
     description = '%s (%s)' % (name, architecture)
+    tags = [
+        {'Key': 'family', 'Value': family},
+        {'Key': 'architecture', 'Value': architecture},
+    ]
     images = client.describe_images(Filters=[{'Name': 'name',
                                               'Values': [description]}])
     if overwrite and images['Images']:
@@ -60,7 +65,7 @@ def import_image(region, name, architecture, image, public, overwrite):
         resource.Image(image_id).deregister()
         resource.Snapshot(snapshot_id).delete()
     snapshot_id = create_snapshot(region=region, description=description,
-                                  image=image)
+                                  image=image, tags=tags)
     client.get_waiter('snapshot_completed').wait(SnapshotIds=[snapshot_id])
     image = client.register_image(Architecture=architecture,
                                   BlockDeviceMappings=[{
@@ -72,6 +77,10 @@ def import_image(region, name, architecture, image, public, overwrite):
                                   }],
                                   EnaSupport=True,
                                   Name=description,
+                                  TagSpecifications=[{
+                                      'ResourceType': 'image',
+                                      'Tags': tags,
+                                  }],
                                   RootDeviceName='/dev/sda1',
                                   SriovNetSupport='simple',
                                   VirtualizationType='hvm')
@@ -94,6 +103,8 @@ def launch_link(region, image_id):
 parser = argparse.ArgumentParser(description="Import AWS EC2 image (AMI)")
 parser.add_argument('--name', '-n',
                     help="Image name")
+parser.add_argument('--family', '-f',
+                    help="Image family name")
 parser.add_argument('--public', '-p', action='store_true',
                     help="Make image public")
 parser.add_argument('--overwrite', action='store_true',
@@ -108,9 +119,13 @@ args = parser.parse_args()
 # Detect CPU architectures
 architectures = {image: detect_architecture(image) for image in args.image}
 
+# Use default family name if none specified
+if not args.family:
+    args.family = 'iPXE'
+
 # Use default name if none specified
 if not args.name:
-    args.name = 'iPXE (%s)' % date.today().strftime('%Y-%m-%d')
+    args.name = '%s (%s)' % (args.family, date.today().strftime('%Y-%m-%d'))
 
 # Use all regions if none specified
 if not args.region:
@@ -123,6 +138,7 @@ with ThreadPoolExecutor(max_workers=len(imports)) as executor:
     futures = {executor.submit(import_image,
                                region=region,
                                name=args.name,
+                               family=args.family,
                                architecture=architectures[image],
                                image=image,
                                public=args.public,

From a1830ff43c5f8ee2957eb537f24670265ebd233b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 9 Sep 2024 15:00:24 +0100
Subject: [PATCH 194/237] [cloud] Add ability to delete old AMI images

Add the "--retain <N>" option to limit the number of retained old AMI
images (within the same family, architecture, and public visibility).

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 contrib/cloud/aws-import | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/contrib/cloud/aws-import b/contrib/cloud/aws-import
index 64b299622..3b6b5223a 100755
--- a/contrib/cloud/aws-import
+++ b/contrib/cloud/aws-import
@@ -47,7 +47,22 @@ def create_snapshot(region, description, image, tags):
     return snapshot_id
 
 
-def import_image(region, name, family, architecture, image, public, overwrite):
+def delete_images(region, filters, retain):
+    client = boto3.client('ec2', region_name=region)
+    resource = boto3.resource('ec2', region_name=region)
+    images = client.describe_images(Owners=['self'], Filters=filters)
+    old_images = sorted(images['Images'], key=lambda x: x['CreationDate'])
+    if retain > 0:
+        old_images = old_images[:-retain]
+    for image in old_images:
+        image_id = image['ImageId']
+        snapshot_id = image['BlockDeviceMappings'][0]['Ebs']['SnapshotId']
+        resource.Image(image_id).deregister()
+        resource.Snapshot(snapshot_id).delete()
+
+
+def import_image(region, name, family, architecture, image, public, overwrite,
+                 retain):
     """Import an AMI image"""
     client = boto3.client('ec2', region_name=region)
     resource = boto3.resource('ec2', region_name=region)
@@ -56,14 +71,16 @@ def import_image(region, name, family, architecture, image, public, overwrite):
         {'Key': 'family', 'Value': family},
         {'Key': 'architecture', 'Value': architecture},
     ]
-    images = client.describe_images(Filters=[{'Name': 'name',
-                                              'Values': [description]}])
-    if overwrite and images['Images']:
-        images = images['Images'][0]
-        image_id = images['ImageId']
-        snapshot_id = images['BlockDeviceMappings'][0]['Ebs']['SnapshotId']
-        resource.Image(image_id).deregister()
-        resource.Snapshot(snapshot_id).delete()
+    if overwrite:
+        filters = [{'Name': 'name', 'Values': [description]}]
+        delete_images(region=region, filters=filters, retain=0)
+    if retain is not None:
+        filters = [
+            {'Name': 'tag:family', 'Values': [family]},
+            {'Name': 'tag:architecture', 'Values': [architecture]},
+            {'Name': 'is-public', 'Values': [str(public).lower()]},
+        ]
+        delete_images(region=region, filters=filters, retain=retain)
     snapshot_id = create_snapshot(region=region, description=description,
                                   image=image, tags=tags)
     client.get_waiter('snapshot_completed').wait(SnapshotIds=[snapshot_id])
@@ -109,6 +126,8 @@ parser.add_argument('--public', '-p', action='store_true',
                     help="Make image public")
 parser.add_argument('--overwrite', action='store_true',
                     help="Overwrite any existing image with same name")
+parser.add_argument('--retain', type=int, metavar='NUM',
+                    help="Retain at most <NUM> old images")
 parser.add_argument('--region', '-r', action='append',
                     help="AWS region(s)")
 parser.add_argument('--wiki', '-w', metavar='FILE',
@@ -142,7 +161,8 @@ with ThreadPoolExecutor(max_workers=len(imports)) as executor:
                                architecture=architectures[image],
                                image=image,
                                public=args.public,
-                               overwrite=args.overwrite): (region, image)
+                               overwrite=args.overwrite,
+                               retain=args.retain): (region, image)
                for region, image in imports}
     results = {futures[future]: future.result()
                for future in as_completed(futures)}

From 5de5d4626e192f5d0e6c97e14ce03b92a43622b3 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 12 Sep 2024 14:17:20 +0100
Subject: [PATCH 195/237] [libc] Centralise architecture-independent portions
 of setjmp.h

The definitions of the setjmp() and longjmp() functions are common to
all architectures, with only the definition of the jump buffer
structure being architecture-specific.

Move the architecture-specific portions to bits/setjmp.h and provide a
common setjmp.h for the function definitions.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/include/{ => bits}/setjmp.h  | 12 +++--------
 src/arch/arm64/include/{ => bits}/setjmp.h  | 12 +++--------
 src/arch/i386/include/{ => bits}/setjmp.h   | 12 +++--------
 src/arch/loong64/include/bits/setjmp.h      | 23 +++++++++++++++++++++
 src/arch/x86_64/include/{ => bits}/setjmp.h | 12 +++--------
 src/{arch/loong64 => }/include/setjmp.h     | 19 +----------------
 6 files changed, 36 insertions(+), 54 deletions(-)
 rename src/arch/arm32/include/{ => bits}/setjmp.h (68%)
 rename src/arch/arm64/include/{ => bits}/setjmp.h (72%)
 rename src/arch/i386/include/{ => bits}/setjmp.h (60%)
 create mode 100644 src/arch/loong64/include/bits/setjmp.h
 rename src/arch/x86_64/include/{ => bits}/setjmp.h (64%)
 rename src/{arch/loong64 => }/include/setjmp.h (52%)

diff --git a/src/arch/arm32/include/setjmp.h b/src/arch/arm32/include/bits/setjmp.h
similarity index 68%
rename from src/arch/arm32/include/setjmp.h
rename to src/arch/arm32/include/bits/setjmp.h
index 4828b47a2..9ee264ecd 100644
--- a/src/arch/arm32/include/setjmp.h
+++ b/src/arch/arm32/include/bits/setjmp.h
@@ -1,5 +1,5 @@
-#ifndef _SETJMP_H
-#define _SETJMP_H
+#ifndef _BITS_SETJMP_H
+#define _BITS_SETJMP_H
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
@@ -29,10 +29,4 @@ typedef struct {
 	uint32_t lr;
 } jmp_buf[1];
 
-extern int __asmcall __attribute__ (( returns_twice ))
-setjmp ( jmp_buf env );
-
-extern void __asmcall __attribute__ (( noreturn ))
-longjmp ( jmp_buf env, int val );
-
-#endif /* _SETJMP_H */
+#endif /* _BITS_SETJMP_H */
diff --git a/src/arch/arm64/include/setjmp.h b/src/arch/arm64/include/bits/setjmp.h
similarity index 72%
rename from src/arch/arm64/include/setjmp.h
rename to src/arch/arm64/include/bits/setjmp.h
index 85a7a9cad..6ffd2fb0a 100644
--- a/src/arch/arm64/include/setjmp.h
+++ b/src/arch/arm64/include/bits/setjmp.h
@@ -1,5 +1,5 @@
-#ifndef _SETJMP_H
-#define _SETJMP_H
+#ifndef _BITS_SETJMP_H
+#define _BITS_SETJMP_H
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
@@ -35,10 +35,4 @@ typedef struct {
 	uint64_t sp;
 } jmp_buf[1];
 
-extern int __asmcall __attribute__ (( returns_twice ))
-setjmp ( jmp_buf env );
-
-extern void __asmcall __attribute__ (( noreturn ))
-longjmp ( jmp_buf env, int val );
-
-#endif /* _SETJMP_H */
+#endif /* _BITS_SETJMP_H */
diff --git a/src/arch/i386/include/setjmp.h b/src/arch/i386/include/bits/setjmp.h
similarity index 60%
rename from src/arch/i386/include/setjmp.h
rename to src/arch/i386/include/bits/setjmp.h
index 98566696a..6b2ec9613 100644
--- a/src/arch/i386/include/setjmp.h
+++ b/src/arch/i386/include/bits/setjmp.h
@@ -1,5 +1,5 @@
-#ifndef _SETJMP_H
-#define _SETJMP_H
+#ifndef _BITS_SETJMP_H
+#define _BITS_SETJMP_H
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
@@ -21,10 +21,4 @@ typedef struct {
 	uint32_t ebp;
 } jmp_buf[1];
 
-extern int __asmcall __attribute__ (( returns_twice ))
-setjmp ( jmp_buf env );
-
-extern void __asmcall __attribute__ (( noreturn ))
-longjmp ( jmp_buf env, int val );
-
-#endif /* _SETJMP_H */
+#endif /* _BITS_SETJMP_H */
diff --git a/src/arch/loong64/include/bits/setjmp.h b/src/arch/loong64/include/bits/setjmp.h
new file mode 100644
index 000000000..c8d7cef0e
--- /dev/null
+++ b/src/arch/loong64/include/bits/setjmp.h
@@ -0,0 +1,23 @@
+#ifndef _BITS_SETJMP_H
+#define _BITS_SETJMP_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** A jump buffer */
+typedef struct {
+	uint64_t s0;
+	uint64_t s1;
+	uint64_t s2;
+	uint64_t s3;
+	uint64_t s4;
+	uint64_t s5;
+	uint64_t s6;
+	uint64_t s7;
+	uint64_t s8;
+
+	uint64_t fp;
+	uint64_t sp;
+	uint64_t ra;
+} jmp_buf[1];
+
+#endif /* _BITS_SETJMP_H */
diff --git a/src/arch/x86_64/include/setjmp.h b/src/arch/x86_64/include/bits/setjmp.h
similarity index 64%
rename from src/arch/x86_64/include/setjmp.h
rename to src/arch/x86_64/include/bits/setjmp.h
index 69835d9fa..adfb869ea 100644
--- a/src/arch/x86_64/include/setjmp.h
+++ b/src/arch/x86_64/include/bits/setjmp.h
@@ -1,5 +1,5 @@
-#ifndef _SETJMP_H
-#define _SETJMP_H
+#ifndef _BITS_SETJMP_H
+#define _BITS_SETJMP_H
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
@@ -25,10 +25,4 @@ typedef struct {
 	uint64_t r15;
 } jmp_buf[1];
 
-extern int __asmcall __attribute__ (( returns_twice ))
-setjmp ( jmp_buf env );
-
-extern void __asmcall __attribute__ (( noreturn ))
-longjmp ( jmp_buf env, int val );
-
-#endif /* _SETJMP_H */
+#endif /* _BITS_SETJMP_H */
diff --git a/src/arch/loong64/include/setjmp.h b/src/include/setjmp.h
similarity index 52%
rename from src/arch/loong64/include/setjmp.h
rename to src/include/setjmp.h
index 1e5168338..d8cd02f46 100644
--- a/src/arch/loong64/include/setjmp.h
+++ b/src/include/setjmp.h
@@ -3,24 +3,7 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#include <stdint.h>
-
-/** jump buffer env*/
-typedef struct {
-	uint64_t s0;
-	uint64_t s1;
-	uint64_t s2;
-	uint64_t s3;
-	uint64_t s4;
-	uint64_t s5;
-	uint64_t s6;
-	uint64_t s7;
-	uint64_t s8;
-
-	uint64_t fp;
-	uint64_t sp;
-	uint64_t ra;
-} jmp_buf[1];
+#include <bits/setjmp.h>
 
 extern int __asmcall __attribute__ (( returns_twice ))
 setjmp ( jmp_buf env );

From c85ad1246890cf3c0c5f2ac6de06ab046ddd0043 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 13 Sep 2024 14:26:34 +0100
Subject: [PATCH 196/237] [efi] Centralise definition of efi_cpu_nap()

Define a cpu_halt() function which is architecture-specific but
platform-independent, and merge the multiple architecture-specific
implementations of the EFI cpu_nap() function into a single central
efi_cpu_nap() that uses cpu_halt() if applicable.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm/include/bits/nap.h               | 10 +++-
 src/arch/arm/include/ipxe/efi/efiarm_nap.h    | 18 ------
 src/arch/arm/interface/efi/efiarm_nap.c       | 57 -------------------
 src/arch/loong64/include/bits/nap.h           |  8 ++-
 .../loong64/include/ipxe/efi/efiloong64_nap.h | 18 ------
 .../loong64/interface/efi/efiloong64_nap.c    | 57 -------------------
 src/arch/x86/include/bits/nap.h               | 11 +++-
 src/arch/x86/include/ipxe/efi/efix86_nap.h    | 18 ------
 src/config/defaults/efi.h                     |  4 +-
 src/include/ipxe/efi/efi_nap.h                | 18 ++++++
 src/include/ipxe/nap.h                        |  7 +++
 .../efix86_nap.c => interface/efi/efi_nap.c}  |  6 +-
 12 files changed, 53 insertions(+), 179 deletions(-)
 delete mode 100644 src/arch/arm/include/ipxe/efi/efiarm_nap.h
 delete mode 100644 src/arch/arm/interface/efi/efiarm_nap.c
 delete mode 100644 src/arch/loong64/include/ipxe/efi/efiloong64_nap.h
 delete mode 100644 src/arch/loong64/interface/efi/efiloong64_nap.c
 delete mode 100644 src/arch/x86/include/ipxe/efi/efix86_nap.h
 create mode 100644 src/include/ipxe/efi/efi_nap.h
 rename src/{arch/x86/interface/efi/efix86_nap.c => interface/efi/efi_nap.c} (93%)

diff --git a/src/arch/arm/include/bits/nap.h b/src/arch/arm/include/bits/nap.h
index e30a7146b..dbdf37166 100644
--- a/src/arch/arm/include/bits/nap.h
+++ b/src/arch/arm/include/bits/nap.h
@@ -9,6 +9,12 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#include <ipxe/efi/efiarm_nap.h>
+/**
+ * Sleep until next CPU interrupt
+ *
+ */
+static inline __attribute__ (( always_inline )) void cpu_halt ( void ) {
+	__asm__ __volatile__ ( "wfi" );
+}
 
-#endif /* _BITS_MAP_H */
+#endif /* _BITS_NAP_H */
diff --git a/src/arch/arm/include/ipxe/efi/efiarm_nap.h b/src/arch/arm/include/ipxe/efi/efiarm_nap.h
deleted file mode 100644
index dcbdd3e20..000000000
--- a/src/arch/arm/include/ipxe/efi/efiarm_nap.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#ifndef _IPXE_EFIARM_NAP_H
-#define _IPXE_EFIARM_NAP_H
-
-/** @file
- *
- * EFI CPU sleeping
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#ifdef NAP_EFIARM
-#define NAP_PREFIX_efiarm
-#else
-#define NAP_PREFIX_efiarm __efiarm_
-#endif
-
-#endif /* _IPXE_EFIARM_NAP_H */
diff --git a/src/arch/arm/interface/efi/efiarm_nap.c b/src/arch/arm/interface/efi/efiarm_nap.c
deleted file mode 100644
index fba7a5d82..000000000
--- a/src/arch/arm/interface/efi/efiarm_nap.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <ipxe/nap.h>
-#include <ipxe/efi/efi.h>
-
-/** @file
- *
- * iPXE CPU sleeping API for EFI
- *
- */
-
-/**
- * Sleep until next interrupt
- *
- */
-static void efiarm_cpu_nap ( void ) {
-	/*
-	 * I can't find any EFI API that allows us to put the CPU to
-	 * sleep.  The CpuSleep() function is defined in CpuLib.h, but
-	 * isn't part of any exposed protocol so we have no way to
-	 * call it.
-	 *
-	 * The EFI shell doesn't seem to bother sleeping the CPU; it
-	 * just sits there idly burning power.
-	 *
-	 * If a shutdown is in progess, there may be nothing to
-	 * generate an interrupt since the timer is disabled in the
-	 * first step of ExitBootServices().
-	 */
-	if ( ! efi_shutdown_in_progress )
-		__asm__ __volatile__ ( "wfi" );
-}
-
-PROVIDE_NAP ( efiarm, cpu_nap, efiarm_cpu_nap );
diff --git a/src/arch/loong64/include/bits/nap.h b/src/arch/loong64/include/bits/nap.h
index 2deba3558..d904db537 100644
--- a/src/arch/loong64/include/bits/nap.h
+++ b/src/arch/loong64/include/bits/nap.h
@@ -9,6 +9,12 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#include <ipxe/efi/efiloong64_nap.h>
+/**
+ * Sleep until next CPU interrupt
+ *
+ */
+static inline __attribute__ (( always_inline )) void cpu_halt ( void ) {
+	__asm__ __volatile__ ( "idle 0" );
+}
 
 #endif /* _BITS_NAP_H */
diff --git a/src/arch/loong64/include/ipxe/efi/efiloong64_nap.h b/src/arch/loong64/include/ipxe/efi/efiloong64_nap.h
deleted file mode 100644
index 5c0d38636..000000000
--- a/src/arch/loong64/include/ipxe/efi/efiloong64_nap.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#ifndef _IPXE_EFILOONG64_NAP_H
-#define _IPXE_EFILOONG64_NAP_H
-
-/** @file
- *
- * EFI CPU sleeping
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#ifdef NAP_EFILOONG64
-#define NAP_PREFIX_efiloong64
-#else
-#define NAP_PREFIX_efiloong64 __efiloong64_
-#endif
-
-#endif /* _IPXE_EFILOONG64_NAP_H */
diff --git a/src/arch/loong64/interface/efi/efiloong64_nap.c b/src/arch/loong64/interface/efi/efiloong64_nap.c
deleted file mode 100644
index 0a7609783..000000000
--- a/src/arch/loong64/interface/efi/efiloong64_nap.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2023, Xiaotian Wu <wuxiaotian@loongson.cn>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <ipxe/nap.h>
-#include <ipxe/efi/efi.h>
-
-/** @file
- *
- * iPXE CPU sleeping API for EFI
- *
- */
-
-/**
- * Sleep until next interrupt
- *
- */
-static void efiloong64_cpu_nap ( void ) {
-	/*
-	 * I can't find any EFI API that allows us to put the CPU to
-	 * sleep.  The CpuSleep() function is defined in CpuLib.h, but
-	 * isn't part of any exposed protocol so we have no way to
-	 * call it.
-	 *
-	 * The EFI shell doesn't seem to bother sleeping the CPU; it
-	 * just sits there idly burning power.
-	 *
-	 * If a shutdown is in progess, there may be nothing to
-	 * generate an interrupt since the timer is disabled in the
-	 * first step of ExitBootServices().
-	 */
-	if ( ! efi_shutdown_in_progress )
-		__asm__ __volatile__ ( "idle 0" );
-}
-
-PROVIDE_NAP ( efiloong64, cpu_nap, efiloong64_cpu_nap );
diff --git a/src/arch/x86/include/bits/nap.h b/src/arch/x86/include/bits/nap.h
index 7103b94c0..b7dea736d 100644
--- a/src/arch/x86/include/bits/nap.h
+++ b/src/arch/x86/include/bits/nap.h
@@ -10,6 +10,13 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_nap.h>
-#include <ipxe/efi/efix86_nap.h>
 
-#endif /* _BITS_MAP_H */
+/**
+ * Sleep until next CPU interrupt
+ *
+ */
+static inline __attribute__ (( always_inline )) void cpu_halt ( void ) {
+	__asm__ __volatile__ ( "hlt" );
+}
+
+#endif /* _BITS_NAP_H */
diff --git a/src/arch/x86/include/ipxe/efi/efix86_nap.h b/src/arch/x86/include/ipxe/efi/efix86_nap.h
deleted file mode 100644
index 1a391c9b6..000000000
--- a/src/arch/x86/include/ipxe/efi/efix86_nap.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#ifndef _IPXE_EFIX86_NAP_H
-#define _IPXE_EFIX86_NAP_H
-
-/** @file
- *
- * EFI CPU sleeping
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#ifdef NAP_EFIX86
-#define NAP_PREFIX_efix86
-#else
-#define NAP_PREFIX_efix86 __efix86_
-#endif
-
-#endif /* _IPXE_EFIX86_NAP_H */
diff --git a/src/config/defaults/efi.h b/src/config/defaults/efi.h
index b62ddb46a..178b448cb 100644
--- a/src/config/defaults/efi.h
+++ b/src/config/defaults/efi.h
@@ -26,6 +26,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ACPI_EFI
 #define FDT_EFI
 #define MPAPI_EFI
+#define NAP_EFI
 
 #define	NET_PROTO_IPV6		/* IPv6 protocol */
 #define	NET_PROTO_LLDP		/* Link Layer Discovery protocol */
@@ -53,7 +54,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #if defined ( __i386__ ) || defined ( __x86_64__ )
 #define IOAPI_X86
-#define NAP_EFIX86
 #define ENTROPY_RDRAND
 #define	CPUID_CMD		/* x86 CPU feature detection command */
 #define	UNSAFE_STD		/* Avoid setting direction flag */
@@ -61,7 +61,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #if defined ( __arm__ ) || defined ( __aarch64__ )
 #define IOAPI_ARM
-#define NAP_EFIARM
 #endif
 
 #if defined ( __aarch64__ )
@@ -70,7 +69,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #if defined ( __loongarch__ )
 #define IOAPI_LOONG64
-#define NAP_EFILOONG64
 #endif
 
 #endif /* CONFIG_DEFAULTS_EFI_H */
diff --git a/src/include/ipxe/efi/efi_nap.h b/src/include/ipxe/efi/efi_nap.h
new file mode 100644
index 000000000..1ffb05569
--- /dev/null
+++ b/src/include/ipxe/efi/efi_nap.h
@@ -0,0 +1,18 @@
+#ifndef _IPXE_EFI_NAP_H
+#define _IPXE_EFI_NAP_H
+
+/** @file
+ *
+ * CPU sleeping
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#ifdef NAP_EFI
+#define NAP_PREFIX_efi
+#else
+#define NAP_PREFIX_efi __efi_
+#endif
+
+#endif /* _IPXE_EFI_NAP_H */
diff --git a/src/include/ipxe/nap.h b/src/include/ipxe/nap.h
index f4de778c4..8d5d8e3df 100644
--- a/src/include/ipxe/nap.h
+++ b/src/include/ipxe/nap.h
@@ -43,6 +43,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Include all architecture-independent I/O API headers */
 #include <ipxe/null_nap.h>
+#include <ipxe/efi/efi_nap.h>
 #include <ipxe/linux/linux_nap.h>
 
 /* Include all architecture-dependent I/O API headers */
@@ -52,6 +53,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Sleep until next CPU interrupt
  *
  */
+void cpu_halt ( void );
+
+/**
+ * Sleep with interrupts enabled until next CPU interrupt
+ *
+ */
 void cpu_nap ( void );
 
 #endif /* _IPXE_NAP_H */
diff --git a/src/arch/x86/interface/efi/efix86_nap.c b/src/interface/efi/efi_nap.c
similarity index 93%
rename from src/arch/x86/interface/efi/efix86_nap.c
rename to src/interface/efi/efi_nap.c
index 296876b85..2bb47627f 100644
--- a/src/arch/x86/interface/efi/efix86_nap.c
+++ b/src/interface/efi/efi_nap.c
@@ -36,7 +36,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Sleep until next interrupt
  *
  */
-static void efix86_cpu_nap ( void ) {
+static void efi_cpu_nap ( void ) {
 	/*
 	 * I can't find any EFI API that allows us to put the CPU to
 	 * sleep.  The CpuSleep() function is defined in CpuLib.h, but
@@ -51,7 +51,7 @@ static void efix86_cpu_nap ( void ) {
 	 * first step of ExitBootServices().
 	 */
 	if ( ! efi_shutdown_in_progress )
-		__asm__ __volatile__ ( "hlt" );
+		cpu_halt();
 }
 
-PROVIDE_NAP ( efix86, cpu_nap, efix86_cpu_nap );
+PROVIDE_NAP ( efi, cpu_nap, efi_cpu_nap );

From 1d43e535fb86537cb87f30c7fa45b175534489d6 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 15 Sep 2024 02:00:14 +0100
Subject: [PATCH 197/237] [test] Add tests for 64-bit logical and arithmetic
 shifts

For some 32-bit CPUs, we need to provide implementations of 64-bit
shifts as libgcc helper functions.  Add test cases to cover these.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/tests/math_test.c | 117 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 117 insertions(+)

diff --git a/src/tests/math_test.c b/src/tests/math_test.c
index 1a244f1eb..68e66c3e3 100644
--- a/src/tests/math_test.c
+++ b/src/tests/math_test.c
@@ -78,6 +78,42 @@ __attribute__ (( noinline )) int flsll_var ( long long value ) {
 	return flsll ( value );
 }
 
+/**
+ * Force a use of runtime 64-bit shift left
+ *
+ * @v value		Value
+ * @v shift		Shift amount
+ * @ret value		Shifted value
+ */
+__attribute__ (( noinline )) uint64_t lsl64_var ( uint64_t value,
+						  unsigned int shift ) {
+	return ( value << shift );
+}
+
+/**
+ * Force a use of runtime 64-bit logical shift right
+ *
+ * @v value		Value
+ * @v shift		Shift amount
+ * @ret value		Shifted value
+ */
+__attribute__ (( noinline )) uint64_t lsr64_var ( uint64_t value,
+						  unsigned int shift ) {
+	return ( value >> shift );
+}
+
+/**
+ * Force a use of runtime 64-bit arithmetic shift right
+ *
+ * @v value		Value
+ * @v shift		Shift amount
+ * @ret value		Shifted value
+ */
+__attribute__ (( noinline )) int64_t asr64_var ( int64_t value,
+						 unsigned int shift ) {
+	return ( value >> shift );
+}
+
 /**
  * Check current stack pointer
  *
@@ -272,6 +308,72 @@ flsll_okx ( long long value, int msb, const char *file, unsigned int line ) {
 }
 #define flsll_ok( value, msb ) flsll_okx ( value, msb, __FILE__, __LINE__ )
 
+/**
+ * Report a 64-bit shift left test result
+ *
+ * @v value		Value
+ * @v shift		Shift amount
+ * @v expected		Expected value
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static inline __attribute__ (( always_inline )) void
+lsl64_okx ( uint64_t value, unsigned int shift, uint64_t expected,
+	    const char *file, unsigned int line ) {
+
+	/* Verify as a compile-time calculation */
+	okx ( ( value << shift ) == expected, file, line );
+
+	/* Verify as a runtime calculation */
+	okx ( lsl64_var ( value, shift ) == expected, file, line );
+}
+#define lsl64_ok( value, shift, expected ) \
+	lsl64_okx ( value, shift, expected, __FILE__, __LINE__ )
+
+/**
+ * Report a 64-bit logical shift right test result
+ *
+ * @v value		Value
+ * @v shift		Shift amount
+ * @v expected		Expected value
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static inline __attribute__ (( always_inline )) void
+lsr64_okx ( uint64_t value, unsigned int shift, uint64_t expected,
+	    const char *file, unsigned int line ) {
+
+	/* Verify as a compile-time calculation */
+	okx ( ( value >> shift ) == expected, file, line );
+
+	/* Verify as a runtime calculation */
+	okx ( lsr64_var ( value, shift ) == expected, file, line );
+}
+#define lsr64_ok( value, shift, expected ) \
+	lsr64_okx ( value, shift, expected, __FILE__, __LINE__ )
+
+/**
+ * Report a 64-bit arithmetic shift right test result
+ *
+ * @v value		Value
+ * @v shift		Shift amount
+ * @v expected		Expected value
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static inline __attribute__ (( always_inline )) void
+asr64_okx ( int64_t value, unsigned int shift, int64_t expected,
+	    const char *file, unsigned int line ) {
+
+	/* Verify as a compile-time calculation */
+	okx ( ( value >> shift ) == expected, file, line );
+
+	/* Verify as a runtime calculation */
+	okx ( asr64_var ( value, shift ) == expected, file, line );
+}
+#define asr64_ok( value, shift, expected ) \
+	asr64_okx ( value, shift, expected, __FILE__, __LINE__ )
+
 /**
  * Report a 64-bit unsigned integer division test result
  *
@@ -375,6 +477,21 @@ static void math_test_exec ( void ) {
 	 * etc. (including checking that the implicit calling
 	 * convention assumed by gcc matches our expectations).
 	 */
+	lsl64_ok ( 0x06760c14710540c2ULL,  0, 0x06760c14710540c2ULL );
+	lsr64_ok ( 0x06760c14710540c2ULL,  0, 0x06760c14710540c2ULL );
+	asr64_ok ( 0x06760c14710540c2ULL,  0, 0x06760c14710540c2ULL );
+	lsl64_ok ( 0xccafd1a8cb724c13ULL, 20, 0x1a8cb724c1300000ULL );
+	lsr64_ok ( 0xccafd1a8cb724c13ULL, 20, 0x00000ccafd1a8cb7ULL );
+	asr64_ok ( 0xccafd1a8cb724c13ULL, 20, 0xfffffccafd1a8cb7ULL );
+	lsl64_ok ( 0x83567264b1234518ULL, 32, 0xb123451800000000ULL );
+	lsr64_ok ( 0x83567264b1234518ULL, 32, 0x0000000083567264ULL );
+	asr64_ok ( 0x83567264b1234518ULL, 32, 0xffffffff83567264ULL );
+	lsl64_ok ( 0x69ee42fcbf1a4ea4ULL, 47, 0x2752000000000000ULL );
+	lsr64_ok ( 0x69ee42fcbf1a4ea4ULL, 47, 0x000000000000d3dcULL );
+	asr64_ok ( 0x69ee42fcbf1a4ea4ULL, 47, 0x000000000000d3dcULL );
+	lsl64_ok ( 0xaa20b8caddee4269ULL, 63, 0x8000000000000000ULL );
+	lsr64_ok ( 0xaa20b8caddee4269ULL, 63, 0x0000000000000001ULL );
+	asr64_ok ( 0xaa20b8caddee4269ULL, 63, 0xffffffffffffffffULL );
 	u64divmod_ok ( 0x2b90ddccf699f765ULL, 0xed9f5e73ULL,
 		       0x2eef6ab4ULL, 0x0e12f089ULL );
 	s64divmod_ok ( 0x2b90ddccf699f765ULL, 0xed9f5e73ULL,

From 670810bed841678e6303719bb123edad9857f320 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 15 Sep 2024 02:01:46 +0100
Subject: [PATCH 198/237] [efi] Use standard va_args macros instead of
 VA_START() etc

The EDK2 header macros VA_START(), VA_ARG() etc produce build errors
on some CPU architectures (notably on 32-bit RISC-V, which is not yet
supported by EDK2).

Fix by using the standard variable argument list macros.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_wrap.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/interface/efi/efi_wrap.c b/src/interface/efi/efi_wrap.c
index 5d5d2caee..c2fab6647 100644
--- a/src/interface/efi/efi_wrap.c
+++ b/src/interface/efi/efi_wrap.c
@@ -1059,7 +1059,7 @@ efi_install_multiple_protocol_interfaces_wrapper ( EFI_HANDLE *handle, ... ) {
 	void *retaddr = __builtin_return_address ( 0 );
 	EFI_GUID *protocol[ MAX_WRAP_MULTI + 1 ];
 	VOID *interface[MAX_WRAP_MULTI];
-	VA_LIST ap;
+	va_list ap;
 	unsigned int i;
 	EFI_STATUS efirc;
 
@@ -1067,20 +1067,20 @@ efi_install_multiple_protocol_interfaces_wrapper ( EFI_HANDLE *handle, ... ) {
 	       efi_handle_name ( *handle ) );
 	memset ( protocol, 0, sizeof ( protocol ) );
 	memset ( interface, 0, sizeof ( interface ) );
-	VA_START ( ap, handle );
-	for ( i = 0 ; ( protocol[i] = VA_ARG ( ap, EFI_GUID * ) ) ; i++ ) {
+	va_start ( ap, handle );
+	for ( i = 0 ; ( protocol[i] = va_arg ( ap, EFI_GUID * ) ) ; i++ ) {
 		if ( i == MAX_WRAP_MULTI ) {
-			VA_END ( ap );
+			va_end ( ap );
 			efirc = EFI_OUT_OF_RESOURCES;
 			DBGC ( colour, "<FATAL: too many arguments> ) = %s "
 			       "-> %p\n", efi_status ( efirc ), retaddr );
 			return efirc;
 		}
-		interface[i] = VA_ARG ( ap, VOID * );
+		interface[i] = va_arg ( ap, VOID * );
 		DBGC ( colour, ", %s, %p",
 		       efi_guid_ntoa ( protocol[i] ), interface[i] );
 	}
-	VA_END ( ap );
+	va_end ( ap );
 	DBGC ( colour, " ) " );
 	efirc = bs->InstallMultipleProtocolInterfaces ( handle,
 		protocol[0], interface[0], protocol[1], interface[1],
@@ -1109,7 +1109,7 @@ efi_uninstall_multiple_protocol_interfaces_wrapper ( EFI_HANDLE handle, ... ) {
 	void *retaddr = __builtin_return_address ( 0 );
 	EFI_GUID *protocol[ MAX_WRAP_MULTI  + 1 ];
 	VOID *interface[MAX_WRAP_MULTI];
-	VA_LIST ap;
+	va_list ap;
 	unsigned int i;
 	EFI_STATUS efirc;
 
@@ -1117,20 +1117,20 @@ efi_uninstall_multiple_protocol_interfaces_wrapper ( EFI_HANDLE handle, ... ) {
 	       efi_handle_name ( handle ) );
 	memset ( protocol, 0, sizeof ( protocol ) );
 	memset ( interface, 0, sizeof ( interface ) );
-	VA_START ( ap, handle );
-	for ( i = 0 ; ( protocol[i] = VA_ARG ( ap, EFI_GUID * ) ) ; i++ ) {
+	va_start ( ap, handle );
+	for ( i = 0 ; ( protocol[i] = va_arg ( ap, EFI_GUID * ) ) ; i++ ) {
 		if ( i == MAX_WRAP_MULTI ) {
-			VA_END ( ap );
+			va_end ( ap );
 			efirc = EFI_OUT_OF_RESOURCES;
 			DBGC ( colour, "<FATAL: too many arguments> ) = %s "
 			       "-> %p\n", efi_status ( efirc ), retaddr );
 			return efirc;
 		}
-		interface[i] = VA_ARG ( ap, VOID * );
+		interface[i] = va_arg ( ap, VOID * );
 		DBGC ( colour, ", %s, %p",
 		       efi_guid_ntoa ( protocol[i] ), interface[i] );
 	}
-	VA_END ( ap );
+	va_end ( ap );
 	DBGC ( colour, " ) " );
 	efirc = bs->UninstallMultipleProtocolInterfaces ( handle,
 		protocol[0], interface[0], protocol[1], interface[1],

From 68db9a3cb3d73aae83ee4b7a0cbe9c69d7f32482 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 15 Sep 2024 02:07:45 +0100
Subject: [PATCH 199/237] [linux] Allow a sysroot to be specified via
 SYSROOT=...

The cross-compiler will typically use the appropriate sysroot
directory automatically.  This may not work for toolchains where a
single cross-compiler is used to produce output for multiple CPU
variants (e.g. 32-bit and 64-bit RISC-V).

Add a SYSROOT=... parameter that may be used to specify the relevant
sysroot directory, e.g.

 make CROSS=riscv64-linux-gnu- SYSROOT=/usr/riscv32-linux-gnu/sys-root \
      bin-riscv32-linux/tests.linux

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/Makefile.linux | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Makefile.linux b/src/Makefile.linux
index b278c8c0d..762437af5 100644
--- a/src/Makefile.linux
+++ b/src/Makefile.linux
@@ -29,6 +29,9 @@ NON_AUTO_MEDIA = linux
 # Compiler flags for building host API wrapper
 #
 LINUX_CFLAGS	+= -Os -idirafter include -DSYMBOL_PREFIX=$(SYMBOL_PREFIX)
+ifneq ($(SYSROOT),)
+LINUX_CFLAGS	+= --sysroot=$(SYSROOT)
+endif
 
 # Check for libslirp
 #

From c215048ddaac75969c22c018871295a5748a47e8 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Sun, 15 Sep 2024 10:54:04 +0100
Subject: [PATCH 200/237] [riscv] Add support for the RISC-V CPU architecture

Add support for building iPXE as a 64-bit or 32-bit RISC-V binary, for
either UEFI or Linux userspace platforms.  For example:

  # RISC-V 64-bit UEFI
  make CROSS=riscv64-linux-gnu- bin-riscv64-efi/ipxe.efi

  # RISC-V 32-bit UEFI
  make CROSS=riscv64-linux-gnu- bin-riscv32-efi/ipxe.efi

  # RISC-V 64-bit Linux
  make CROSS=riscv64-linux-gnu- bin-riscv64-linux/tests.linux
  qemu-riscv64 -L /usr/riscv64-linux-gnu/sys-root \
               ./bin-riscv64-linux/tests.linux

  # RISC-V 32-bit Linux
  make CROSS=riscv64-linux-gnu- SYSROOT=/usr/riscv32-linux-gnu/sys-root \
       bin-riscv32-linux/tests.linux
  qemu-riscv32 -L /usr/riscv32-linux-gnu/sys-root \
               ./bin-riscv32-linux/tests.linux

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/riscv/Makefile                      |  20 +
 src/arch/riscv/Makefile.efi                  |  10 +
 src/arch/riscv/Makefile.linux                |   6 +
 src/arch/riscv/core/riscv_bigint.c           | 112 ++++++
 src/arch/riscv/core/riscv_io.c               |  46 +++
 src/arch/riscv/core/riscv_string.c           | 272 ++++++++++++++
 src/arch/riscv/core/riscv_strings.S          |  70 ++++
 src/arch/riscv/core/setjmp.S                 | 105 ++++++
 src/arch/riscv/include/bits/bigint.h         | 362 +++++++++++++++++++
 src/arch/riscv/include/bits/bitops.h         |  82 +++++
 src/arch/riscv/include/bits/byteswap.h       |  48 +++
 src/arch/riscv/include/bits/compiler.h       |  40 ++
 src/arch/riscv/include/bits/endian.h         |   8 +
 src/arch/riscv/include/bits/errfile.h        |  19 +
 src/arch/riscv/include/bits/io.h             |  17 +
 src/arch/riscv/include/bits/nap.h            |  20 +
 src/arch/riscv/include/bits/setjmp.h         |  16 +
 src/arch/riscv/include/bits/stdint.h         |  23 ++
 src/arch/riscv/include/bits/string.h         |  89 +++++
 src/arch/riscv/include/bits/strings.h        |  91 +++++
 src/arch/riscv/include/ipxe/riscv_io.h       | 137 +++++++
 src/arch/riscv32/Makefile                    |  20 +
 src/arch/riscv32/Makefile.efi                |  14 +
 src/arch/riscv32/Makefile.linux              |  14 +
 src/arch/riscv32/core/riscv32_byteswap.S     |  63 ++++
 src/arch/riscv32/include/bits/profile.h      |  36 ++
 src/arch/riscv32/include/ipxe/efi/dhcparch.h |  20 +
 src/arch/riscv32/include/limits.h            |  61 ++++
 src/arch/riscv32/libgcc/llshift.S            | 112 ++++++
 src/arch/riscv64/Makefile                    |  19 +
 src/arch/riscv64/Makefile.efi                |  14 +
 src/arch/riscv64/Makefile.linux              |  10 +
 src/arch/riscv64/core/riscv64_byteswap.S     |  64 ++++
 src/arch/riscv64/include/bits/profile.h      |  28 ++
 src/arch/riscv64/include/ipxe/efi/dhcparch.h |  20 +
 src/arch/riscv64/include/limits.h            |  61 ++++
 src/config/defaults/efi.h                    |   4 +
 src/include/ipxe/efi/ProcessorBind.h         |   4 +
 src/include/ipxe/efi/RiscV64/ProcessorBind.h | 175 +++++++++
 src/util/efirom.c                            |   2 +
 src/util/elf2efi.c                           |  65 ++++
 src/util/genfsimg                            |   6 +
 42 files changed, 2405 insertions(+)
 create mode 100644 src/arch/riscv/Makefile
 create mode 100644 src/arch/riscv/Makefile.efi
 create mode 100644 src/arch/riscv/Makefile.linux
 create mode 100644 src/arch/riscv/core/riscv_bigint.c
 create mode 100644 src/arch/riscv/core/riscv_io.c
 create mode 100644 src/arch/riscv/core/riscv_string.c
 create mode 100644 src/arch/riscv/core/riscv_strings.S
 create mode 100644 src/arch/riscv/core/setjmp.S
 create mode 100644 src/arch/riscv/include/bits/bigint.h
 create mode 100644 src/arch/riscv/include/bits/bitops.h
 create mode 100644 src/arch/riscv/include/bits/byteswap.h
 create mode 100644 src/arch/riscv/include/bits/compiler.h
 create mode 100644 src/arch/riscv/include/bits/endian.h
 create mode 100644 src/arch/riscv/include/bits/errfile.h
 create mode 100644 src/arch/riscv/include/bits/io.h
 create mode 100644 src/arch/riscv/include/bits/nap.h
 create mode 100644 src/arch/riscv/include/bits/setjmp.h
 create mode 100644 src/arch/riscv/include/bits/stdint.h
 create mode 100644 src/arch/riscv/include/bits/string.h
 create mode 100644 src/arch/riscv/include/bits/strings.h
 create mode 100644 src/arch/riscv/include/ipxe/riscv_io.h
 create mode 100644 src/arch/riscv32/Makefile
 create mode 100644 src/arch/riscv32/Makefile.efi
 create mode 100644 src/arch/riscv32/Makefile.linux
 create mode 100644 src/arch/riscv32/core/riscv32_byteswap.S
 create mode 100644 src/arch/riscv32/include/bits/profile.h
 create mode 100644 src/arch/riscv32/include/ipxe/efi/dhcparch.h
 create mode 100644 src/arch/riscv32/include/limits.h
 create mode 100644 src/arch/riscv32/libgcc/llshift.S
 create mode 100644 src/arch/riscv64/Makefile
 create mode 100644 src/arch/riscv64/Makefile.efi
 create mode 100644 src/arch/riscv64/Makefile.linux
 create mode 100644 src/arch/riscv64/core/riscv64_byteswap.S
 create mode 100644 src/arch/riscv64/include/bits/profile.h
 create mode 100644 src/arch/riscv64/include/ipxe/efi/dhcparch.h
 create mode 100644 src/arch/riscv64/include/limits.h
 create mode 100644 src/include/ipxe/efi/RiscV64/ProcessorBind.h

diff --git a/src/arch/riscv/Makefile b/src/arch/riscv/Makefile
new file mode 100644
index 000000000..668d7db39
--- /dev/null
+++ b/src/arch/riscv/Makefile
@@ -0,0 +1,20 @@
+# Assembler section type character
+#
+ASM_TCHAR	:= @
+ASM_TCHAR_OPS	:= @
+
+# Include RISCV-specific headers
+#
+INCDIRS		:= arch/$(ARCH)/include arch/riscv/include $(INCDIRS)
+
+# RISCV-specific directories containing source files
+#
+SRCDIRS		+= arch/riscv/core
+
+# RISCV-specific flags
+#
+CFLAGS		+= -mno-strict-align -mno-plt
+
+# EFI requires -fshort-wchar, and nothing else currently uses wchar_t
+#
+CFLAGS          += -fshort-wchar
diff --git a/src/arch/riscv/Makefile.efi b/src/arch/riscv/Makefile.efi
new file mode 100644
index 000000000..957e8b884
--- /dev/null
+++ b/src/arch/riscv/Makefile.efi
@@ -0,0 +1,10 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# RISCV-specific flags
+#
+CFLAGS		+= -mcmodel=medany
+
+# Include generic EFI Makefile
+#
+MAKEDEPS	+= Makefile.efi
+include Makefile.efi
diff --git a/src/arch/riscv/Makefile.linux b/src/arch/riscv/Makefile.linux
new file mode 100644
index 000000000..42590441e
--- /dev/null
+++ b/src/arch/riscv/Makefile.linux
@@ -0,0 +1,6 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Include generic Linux Makefile
+#
+MAKEDEPS	+= Makefile.linux
+include Makefile.linux
diff --git a/src/arch/riscv/core/riscv_bigint.c b/src/arch/riscv/core/riscv_bigint.c
new file mode 100644
index 000000000..cbc5631a3
--- /dev/null
+++ b/src/arch/riscv/core/riscv_bigint.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <string.h>
+#include <ipxe/bigint.h>
+
+/** @file
+ *
+ * Big integer support
+ */
+
+/**
+ * Multiply big integers
+ *
+ * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
+ * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
+ * @v result0		Element 0 of big integer to hold result
+ */
+void bigint_multiply_raw ( const unsigned long *multiplicand0,
+			   unsigned int multiplicand_size,
+			   const unsigned long *multiplier0,
+			   unsigned int multiplier_size,
+			   unsigned long *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
+	unsigned int i;
+	unsigned int j;
+	unsigned long multiplicand_element;
+	unsigned long multiplier_element;
+	unsigned long *result_elements;
+	unsigned long discard_low;
+	unsigned long discard_high;
+	unsigned long discard_temp;
+	unsigned long discard_carry;
+
+	/* Zero result */
+	memset ( result, 0, sizeof ( *result ) );
+
+	/* Multiply integers one element at a time */
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
+		multiplicand_element = multiplicand->element[i];
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
+			multiplier_element = multiplier->element[j];
+			result_elements = &result->element[ i + j ];
+			/* Perform a single multiply, and add the
+			 * resulting double-element into the result,
+			 * carrying as necessary.  The carry can
+			 * never overflow beyond the end of the
+			 * result, since:
+			 *
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
+			 */
+			__asm__ __volatile__ ( /* Perform multiplication */
+					       "mulhu %2, %6, %7\n\t"
+					       "mul %1, %6, %7\n\t"
+					       /* Accumulate low half */
+					       LOADN " %3, (%0)\n\t"
+					       "add %3, %3, %1\n\t"
+					       "sltu %4, %3, %1\n\t"
+					       STOREN " %3, 0(%0)\n\t"
+					       /* Carry into high half */
+					       "add %4, %4, %2\n\t"
+					       /* Propagate as necessary */
+					       "\n1:\n\t"
+					       "addi %0, %0, %8\n\t"
+					       LOADN " %3, 0(%0)\n\t"
+					       "add %3, %3, %4\n\t"
+					       "sltu %4, %3, %4\n\t"
+					       STOREN " %3, 0(%0)\n\t"
+					       "bnez %4, 1b\n\t"
+					       : "+r" ( result_elements ),
+						 "=r" ( discard_low ),
+						 "=r" ( discard_high ),
+						 "=r" ( discard_temp ),
+						 "=r" ( discard_carry ),
+						 "+m" ( *result )
+					       : "r" ( multiplicand_element ),
+						 "r" ( multiplier_element ),
+						 "i" ( sizeof ( *result0 ) ) );
+		}
+	}
+}
diff --git a/src/arch/riscv/core/riscv_io.c b/src/arch/riscv/core/riscv_io.c
new file mode 100644
index 000000000..756b39752
--- /dev/null
+++ b/src/arch/riscv/core/riscv_io.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/io.h>
+#include <ipxe/riscv_io.h>
+
+/** @file
+ *
+ * iPXE I/O API for RISC-V
+ *
+ */
+
+PROVIDE_IOAPI_INLINE ( riscv, phys_to_bus );
+PROVIDE_IOAPI_INLINE ( riscv, bus_to_phys );
+PROVIDE_IOAPI_INLINE ( riscv, readb );
+PROVIDE_IOAPI_INLINE ( riscv, readw );
+PROVIDE_IOAPI_INLINE ( riscv, readl );
+PROVIDE_IOAPI_INLINE ( riscv, writeb );
+PROVIDE_IOAPI_INLINE ( riscv, writew );
+PROVIDE_IOAPI_INLINE ( riscv, writel );
+PROVIDE_IOAPI_INLINE ( riscv, readq );
+PROVIDE_IOAPI_INLINE ( riscv, writeq );
+PROVIDE_IOAPI_INLINE ( riscv, mb );
+PROVIDE_DUMMY_PIO ( riscv );
diff --git a/src/arch/riscv/core/riscv_string.c b/src/arch/riscv/core/riscv_string.c
new file mode 100644
index 000000000..3155c3c21
--- /dev/null
+++ b/src/arch/riscv/core/riscv_string.c
@@ -0,0 +1,272 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+/** @file
+ *
+ * Optimised string operations
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+
+/**
+ * Copy memory area
+ *
+ * @v dest		Destination address
+ * @v src		Source address
+ * @v len		Length
+ * @ret dest		Destination address
+ */
+void riscv_memcpy ( void *dest, const void *src, size_t len ) {
+	size_t len_pre;
+	size_t len_mid;
+	size_t len_post;
+	unsigned long discard_data;
+
+	/* Calculate pre-aligned, aligned, and post-aligned lengths.
+	 * (Align on the destination address, on the assumption that
+	 * misaligned stores are likely to be more expensive than
+	 * misaligned loads.)
+	 */
+	len_pre = ( ( sizeof ( unsigned long ) - ( ( intptr_t ) dest ) ) &
+		    ( sizeof ( unsigned long ) - 1 ) );
+	if ( len_pre > len )
+		len_pre = len;
+	len -= len_pre;
+	len_mid = ( len & ~( sizeof ( unsigned long ) - 1 ) );
+	len -= len_mid;
+	len_post = len;
+
+	/* Copy pre-aligned section */
+	__asm__ __volatile__ ( "j 2f\n\t"
+			       "\n1:\n\t"
+			       "lb %2, (%1)\n\t"
+			       "sb %2, (%0)\n\t"
+			       "addi %0, %0, 1\n\t"
+			       "addi %1, %1, 1\n\t"
+			       "\n2:\n\t"
+			       "bne %0, %3, 1b\n\t"
+			       : "+r" ( dest ), "+r" ( src ),
+				 "=&r" ( discard_data )
+			       : "r" ( dest + len_pre )
+			       : "memory" );
+
+	/* Copy aligned section */
+	__asm__ __volatile__ ( "j 2f\n\t"
+			       "\n1:\n\t"
+			       LOADN " %2, (%1)\n\t"
+			       STOREN " %2, (%0)\n\t"
+			       "addi %0, %0, %4\n\t"
+			       "addi %1, %1, %4\n\t"
+			       "\n2:\n\t"
+			       "bne %0, %3, 1b\n\t"
+			       : "+r" ( dest ), "+r" ( src ),
+				 "=&r" ( discard_data )
+			       : "r" ( dest + len_mid ),
+				 "i" ( sizeof ( unsigned long ) )
+			       : "memory" );
+
+	/* Copy post-aligned section */
+	__asm__ __volatile__ ( "j 2f\n\t"
+			       "\n1:\n\t"
+			       "lb %2, (%1)\n\t"
+			       "sb %2, (%0)\n\t"
+			       "addi %0, %0, 1\n\t"
+			       "addi %1, %1, 1\n\t"
+			       "\n2:\n\t"
+			       "bne %0, %3, 1b\n\t"
+			       : "+r" ( dest ), "+r" ( src ),
+				 "=&r" ( discard_data )
+			       : "r" ( dest + len_post )
+			       : "memory" );
+}
+
+/**
+ * Zero memory region
+ *
+ * @v dest		Destination region
+ * @v len		Length
+ */
+void riscv_bzero ( void *dest, size_t len ) {
+	size_t len_pre;
+	size_t len_mid;
+	size_t len_post;
+
+	/* Calculate pre-aligned, aligned, and post-aligned lengths */
+	len_pre = ( ( sizeof ( unsigned long ) - ( ( intptr_t ) dest ) ) &
+		    ( sizeof ( unsigned long ) - 1 ) );
+	if ( len_pre > len )
+		len_pre = len;
+	len -= len_pre;
+	len_mid = ( len & ~( sizeof ( unsigned long ) - 1 ) );
+	len -= len_mid;
+	len_post = len;
+
+	/* Zero pre-aligned section */
+	__asm__ __volatile__ ( "j 2f\n\t"
+			       "\n1:\n\t"
+			       "sb zero, (%0)\n\t"
+			       "addi %0, %0, 1\n\t"
+			       "\n2:\n\t"
+			       "bne %0, %1, 1b\n\t"
+			       : "+r" ( dest )
+			       : "r" ( dest + len_pre )
+			       : "memory" );
+
+	/* Zero aligned section */
+	__asm__ __volatile__ ( "j 2f\n\t"
+			       "\n1:\n\t"
+			       STOREN " zero, (%0)\n\t"
+			       "addi %0, %0, %2\n\t"
+			       "\n2:\n\t"
+			       "bne %0, %1, 1b\n\t"
+			       : "+r" ( dest )
+			       : "r" ( dest + len_mid ),
+				 "i" ( sizeof ( unsigned long ) )
+			       : "memory" );
+
+	/* Zero post-aligned section */
+	__asm__ __volatile__ ( "j 2f\n\t"
+			       "\n1:\n\t"
+			       "sb zero, (%0)\n\t"
+			       "addi %0, %0, 1\n\t"
+			       "\n2:\n\t"
+			       "bne %0, %1, 1b\n\t"
+			       : "+r" ( dest )
+			       : "r" ( dest + len_post )
+			       : "memory" );
+}
+
+/**
+ * Fill memory region
+ *
+ * @v dest		Destination region
+ * @v len		Length
+ * @v character		Fill character
+ *
+ * The unusual parameter order is to allow for more efficient
+ * tail-calling to riscv_bzero() when zeroing a region.
+ */
+void riscv_memset ( void *dest, size_t len, int character ) {
+
+	/* Do nothing if length is zero */
+	if ( ! len )
+		return;
+
+	/* Use optimised zeroing code if applicable */
+	if ( character == 0 ) {
+		riscv_bzero ( dest, len );
+		return;
+	}
+
+	/* Fill one byte at a time.  Calling memset() with a non-zero
+	 * value is relatively rare and unlikely to be
+	 * performance-critical.
+	 */
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       "sb %2, (%0)\n\t"
+			       "addi %0, %0, 1\n\t"
+			       "bne %0, %1, 1b\n\t"
+			       : "+r" ( dest )
+			       : "r" ( dest + len ), "r" ( character )
+			       : "memory" );
+}
+
+/**
+ * Copy (possibly overlapping) memory region forwards
+ *
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ */
+void riscv_memmove_forwards ( void *dest, const void *src, size_t len ) {
+	unsigned long discard_data;
+
+	/* Do nothing if length is zero */
+	if ( ! len )
+		return;
+
+	/* Assume memmove() is not performance-critical, and perform a
+	 * bytewise copy for simplicity.
+	 */
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       "lb %2, (%1)\n\t"
+			       "sb %2, (%0)\n\t"
+			       "addi %1, %1, 1\n\t"
+			       "addi %0, %0, 1\n\t"
+			       "bne %0, %3, 1b\n\t"
+			       : "+r" ( dest ), "+r" ( src ),
+				 "=&r" ( discard_data )
+			       : "r" ( dest + len )
+			       : "memory" );
+}
+
+/**
+ * Copy (possibly overlapping) memory region backwards
+ *
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ */
+void riscv_memmove_backwards ( void *dest, const void *src, size_t len ) {
+	void *orig_dest = dest;
+	unsigned long discard_data;
+
+	/* Do nothing if length is zero */
+	if ( ! len )
+		return;
+
+	/* Assume memmove() is not performance-critical, and perform a
+	 * bytewise copy for simplicity.
+	 */
+	dest += len;
+	src += len;
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       "addi %1, %1, -1\n\t"
+			       "addi %0, %0, -1\n\t"
+			       "lb %2, (%1)\n\t"
+			       "sb %2, (%0)\n\t"
+			       "bne %0, %3, 1b\n\t"
+			       : "+r" ( dest ), "+r" ( src ),
+				 "=&r" ( discard_data )
+			       : "r" ( orig_dest )
+			       : "memory" );
+}
+
+/**
+ * Copy (possibly overlapping) memory region
+ *
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ */
+void riscv_memmove ( void *dest, const void *src, size_t len ) {
+
+	if ( dest <= src ) {
+		riscv_memmove_forwards ( dest, src, len );
+	} else {
+		riscv_memmove_backwards ( dest, src, len );
+	}
+}
diff --git a/src/arch/riscv/core/riscv_strings.S b/src/arch/riscv/core/riscv_strings.S
new file mode 100644
index 000000000..eb1b397b9
--- /dev/null
+++ b/src/arch/riscv/core/riscv_strings.S
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * Byte swapping
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+	.section ".text.riscv_ffs"
+	.globl	riscv_ffs
+riscv_ffs:
+	beqz	a0, 2f
+	mv	t0, a0
+	li	a0, ( __riscv_xlen + 1 )
+1:	slli	t0, t0, 1
+	addi	a0, a0, -1
+	bnez	t0, 1b
+2:	ret
+	.size	riscv_ffs, . - riscv_ffs
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v value		Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
+	.section ".text.riscv_fls"
+	.globl	riscv_fls
+riscv_fls:
+	beqz	a0, 2f
+	mv	t0, a0
+	li	a0, __riscv_xlen
+	bltz	t0, 2f
+1:	slli	t0, t0, 1
+	addi	a0, a0, -1
+	bgez	t0, 1b
+2:	ret
+	.size	riscv_fls, . - riscv_fls
diff --git a/src/arch/riscv/core/setjmp.S b/src/arch/riscv/core/setjmp.S
new file mode 100644
index 000000000..69d844ac9
--- /dev/null
+++ b/src/arch/riscv/core/setjmp.S
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * Long jumps
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+	/* Must match jmp_buf structure layout */
+	.struct 0
+env_ra:		.space ( __riscv_xlen / 8 )
+env_sp:		.space ( __riscv_xlen / 8 )
+env_s0:		.space ( __riscv_xlen / 8 )
+env_s1:		.space ( __riscv_xlen / 8 )
+env_s2:		.space ( __riscv_xlen / 8 )
+env_s3:		.space ( __riscv_xlen / 8 )
+env_s4:		.space ( __riscv_xlen / 8 )
+env_s5:		.space ( __riscv_xlen / 8 )
+env_s6:		.space ( __riscv_xlen / 8 )
+env_s7:		.space ( __riscv_xlen / 8 )
+env_s8:		.space ( __riscv_xlen / 8 )
+env_s9:		.space ( __riscv_xlen / 8 )
+env_s10:	.space ( __riscv_xlen / 8 )
+env_s11:	.space ( __riscv_xlen / 8 )
+	.previous
+
+/*
+ * Save stack context for non-local goto
+ */
+	.section ".text.setjmp", "ax", @progbits
+	.globl	setjmp
+setjmp:
+	/* Save registers */
+	STOREN	ra, env_ra(a0)
+	STOREN	sp, env_sp(a0)
+	STOREN	s0, env_s0(a0)
+	STOREN	s1, env_s1(a0)
+	STOREN	s2, env_s2(a0)
+	STOREN	s3, env_s3(a0)
+	STOREN	s4, env_s4(a0)
+	STOREN	s5, env_s5(a0)
+	STOREN	s6, env_s6(a0)
+	STOREN	s7, env_s7(a0)
+	STOREN	s8, env_s8(a0)
+	STOREN	s9, env_s9(a0)
+	STOREN	s10, env_s10(a0)
+	STOREN	s11, env_s11(a0)
+	/* Return zero when returning as setjmp() */
+	mv	a0, zero
+	ret
+	.size	setjmp, . - setjmp
+
+/*
+ * Non-local jump to a saved stack context
+ */
+	.section ".text.longjmp", "ax", @progbits
+	.globl	longjmp
+longjmp:
+	/* Restore registers */
+	LOADN	s11, env_s11(a0)
+	LOADN	s10, env_s10(a0)
+	LOADN	s9, env_s9(a0)
+	LOADN	s8, env_s8(a0)
+	LOADN	s7, env_s7(a0)
+	LOADN	s6, env_s6(a0)
+	LOADN	s5, env_s5(a0)
+	LOADN	s4, env_s4(a0)
+	LOADN	s3, env_s3(a0)
+	LOADN	s2, env_s2(a0)
+	LOADN	s1, env_s1(a0)
+	LOADN	s0, env_s0(a0)
+	LOADN	sp, env_sp(a0)
+	LOADN	ra, env_ra(a0)
+	/* Force result to non-zero */
+	seqz	a0, a1
+	or	a0, a0, a1
+	/* Return to setjmp() caller */
+	ret
+	.size	longjmp, . - longjmp
diff --git a/src/arch/riscv/include/bits/bigint.h b/src/arch/riscv/include/bits/bigint.h
new file mode 100644
index 000000000..fcc0d51c3
--- /dev/null
+++ b/src/arch/riscv/include/bits/bigint.h
@@ -0,0 +1,362 @@
+#ifndef _BITS_BIGINT_H
+#define _BITS_BIGINT_H
+
+/** @file
+ *
+ * Big integer support
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+
+/** Element of a big integer */
+typedef unsigned long bigint_element_t;
+
+/**
+ * Initialise big integer
+ *
+ * @v value0		Element 0 of big integer to initialise
+ * @v size		Number of elements
+ * @v data		Raw data
+ * @v len		Length of raw data
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_init_raw ( unsigned long *value0, unsigned int size,
+		  const void *data, size_t len ) {
+	size_t pad_len = ( sizeof ( bigint_t ( size ) ) - len );
+	uint8_t *value_byte = ( ( void * ) value0 );
+	const uint8_t *data_byte = ( data + len );
+
+	/* Copy raw data in reverse order, padding with zeros */
+	while ( len-- )
+		*(value_byte++) = *(--data_byte);
+	while ( pad_len-- )
+		*(value_byte++) = 0;
+}
+
+/**
+ * Add big integers
+ *
+ * @v addend0		Element 0 of big integer to add
+ * @v value0		Element 0 of big integer to be added to
+ * @v size		Number of elements
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_add_raw ( const unsigned long *addend0, unsigned long *value0,
+		 unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
+	unsigned long *valueN = ( value0 + size );
+	unsigned long *discard_addend;
+	unsigned long *discard_value;
+	unsigned long discard_addend_i;
+	unsigned long discard_value_i;
+	unsigned long discard_carry;
+	unsigned long discard_temp;
+
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load addend[i] and value[i] */
+			       LOADN " %2, (%0)\n\t"
+			       LOADN " %3, (%1)\n\t"
+			       /* Add carry flag and addend */
+			       "add %3, %3, %4\n\t"
+			       "sltu %5, %3, %4\n\t"
+			       "add %3, %3, %2\n\t"
+			       "sltu %4, %3, %2\n\t"
+			       "or %4, %4, %5\n\t"
+			       /* Store value[i] */
+			       STOREN " %3, (%1)\n\t"
+			       /* Loop */
+			       "addi %0, %0, %8\n\t"
+			       "addi %1, %1, %8\n\t"
+			       "bne %1, %7, 1b\n\t"
+			       : "=&r" ( discard_addend ),
+				 "=&r" ( discard_value ),
+				 "=&r" ( discard_addend_i ),
+				 "=&r" ( discard_value_i ),
+				 "=&r" ( discard_carry ),
+				 "=&r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "r" ( valueN ),
+				 "i" ( sizeof ( unsigned long ) ),
+				 "0" ( addend0 ), "1" ( value0 ), "4" ( 0 ) );
+}
+
+/**
+ * Subtract big integers
+ *
+ * @v subtrahend0	Element 0 of big integer to subtract
+ * @v value0		Element 0 of big integer to be subtracted from
+ * @v size		Number of elements
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_subtract_raw ( const unsigned long *subtrahend0, unsigned long *value0,
+		      unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
+	unsigned long *valueN = ( value0 + size );
+	unsigned long *discard_subtrahend;
+	unsigned long *discard_value;
+	unsigned long discard_subtrahend_i;
+	unsigned long discard_value_i;
+	unsigned long discard_carry;
+	unsigned long discard_temp;
+
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load subtrahend[i] and value[i] */
+			       LOADN " %2, (%0)\n\t"
+			       LOADN " %3, (%1)\n\t"
+			       /* Subtract carry flag and subtrahend */
+			       "sltu %5, %3, %4\n\t"
+			       "sub %3, %3, %4\n\t"
+			       "sltu %4, %3, %2\n\t"
+			       "sub %3, %3, %2\n\t"
+			       "or %4, %4, %5\n\t"
+			       /* Store value[i] */
+			       STOREN " %3, (%1)\n\t"
+			       /* Loop */
+			       "addi %0, %0, %8\n\t"
+			       "addi %1, %1, %8\n\t"
+			       "bne %1, %7, 1b\n\t"
+			       : "=&r" ( discard_subtrahend ),
+				 "=&r" ( discard_value ),
+				 "=&r" ( discard_subtrahend_i ),
+				 "=&r" ( discard_value_i ),
+				 "=&r" ( discard_carry ),
+				 "=&r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "r" ( valueN ),
+				 "i" ( sizeof ( unsigned long ) ),
+				 "0" ( subtrahend0 ), "1" ( value0 ),
+				 "4" ( 0 ) );
+}
+
+/**
+ * Rotate big integer left
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_rol_raw ( unsigned long *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
+	unsigned long *valueN = ( value0 + size );
+	unsigned long *discard_value;
+	unsigned long discard_value_i;
+	unsigned long discard_carry;
+	unsigned long discard_temp;
+
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load value[i] */
+			       LOADN " %1, (%0)\n\t"
+			       /* Shift left */
+			       "slli %3, %1, 1\n\t"
+			       "or %3, %3, %2\n\t"
+			       "srli %2, %1, %7\n\t"
+			       /* Store value[i] */
+			       STOREN " %3, (%0)\n\t"
+			       /* Loop  */
+			       "addi %0, %0, %6\n\t"
+			       "bne %0, %5, 1b\n\t"
+			       : "=&r" ( discard_value ),
+				 "=&r" ( discard_value_i ),
+				 "=&r" ( discard_carry ),
+				 "=&r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "r" ( valueN ),
+				 "i" ( sizeof ( unsigned long ) ),
+				 "i" ( ( 8 * sizeof ( unsigned long ) - 1 ) ),
+				 "0" ( value0 ), "2" ( 0 ) );
+}
+
+/**
+ * Rotate big integer right
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_ror_raw ( unsigned long *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
+	unsigned long *valueN = ( value0 + size );
+	unsigned long *discard_value;
+	unsigned long discard_value_i;
+	unsigned long discard_carry;
+	unsigned long discard_temp;
+
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load value[i] */
+			       LOADN " %1, %6(%0)\n\t"
+			       /* Shift right */
+			       "srli %3, %1, 1\n\t"
+			       "or %3, %3, %2\n\t"
+			       "slli %2, %1, %7\n\t"
+			       /* Store value[i] */
+			       STOREN " %3, %6(%0)\n\t"
+			       /* Loop  */
+			       "addi %0, %0, %6\n\t"
+			       "bne %0, %5, 1b\n\t"
+			       : "=&r" ( discard_value ),
+				 "=&r" ( discard_value_i ),
+				 "=&r" ( discard_carry ),
+				 "=&r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "r" ( value0 ),
+				 "i" ( -( sizeof ( unsigned long ) ) ),
+				 "i" ( ( 8 * sizeof ( unsigned long ) - 1 ) ),
+				 "0" ( valueN ), "2" ( 0 ) );
+}
+
+/**
+ * Test if big integer is equal to zero
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ * @ret is_zero		Big integer is equal to zero
+ */
+static inline __attribute__ (( always_inline, pure )) int
+bigint_is_zero_raw ( const unsigned long *value0, unsigned int size ) {
+	const unsigned long *value = value0;
+	unsigned long value_i;
+
+	do {
+		value_i = *(value++);
+		if ( value_i )
+			break;
+	} while ( --size );
+
+	return ( value_i == 0 );
+}
+
+/**
+ * Compare big integers
+ *
+ * @v value0		Element 0 of big integer
+ * @v reference0	Element 0 of reference big integer
+ * @v size		Number of elements
+ * @ret geq		Big integer is greater than or equal to the reference
+ */
+static inline __attribute__ (( always_inline, pure )) int
+bigint_is_geq_raw ( const unsigned long *value0,
+		    const unsigned long *reference0, unsigned int size ) {
+	const unsigned long *value = ( value0 + size );
+	const unsigned long *reference = ( reference0 + size );
+	unsigned long value_i;
+	unsigned long reference_i;
+
+	do {
+		value_i = *(--value);
+		reference_i = *(--reference);
+		if ( value_i != reference_i )
+			break;
+	} while ( --size );
+
+	return ( value_i >= reference_i );
+}
+
+/**
+ * Test if bit is set in big integer
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ * @v bit		Bit to test
+ * @ret is_set		Bit is set
+ */
+static inline __attribute__ (( always_inline )) int
+bigint_bit_is_set_raw ( const unsigned long *value0, unsigned int size,
+			unsigned int bit ) {
+	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( const void * ) value0 );
+	unsigned int index = ( bit / ( 8 * sizeof ( *value0 ) ) );
+	unsigned int subindex = ( bit % ( 8 * sizeof ( *value0 ) ) );
+
+	return ( !! ( value->element[index] & ( 1UL << subindex ) ) );
+}
+
+/**
+ * Find highest bit set in big integer
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ * @ret max_bit		Highest bit set + 1 (or 0 if no bits set)
+ */
+static inline __attribute__ (( always_inline )) int
+bigint_max_set_bit_raw ( const unsigned long *value0, unsigned int size ) {
+	const unsigned long *value = ( value0 + size );
+	int max_bit = ( 8 * sizeof ( bigint_t ( size ) ) );
+	unsigned long value_i;
+
+	do {
+		value_i = *(--value);
+		max_bit -= ( ( 8 * sizeof ( *value0 ) ) - fls ( value_i ) );
+		if ( value_i )
+			break;
+	} while ( --size );
+
+	return max_bit;
+}
+
+/**
+ * Grow big integer
+ *
+ * @v source0		Element 0 of source big integer
+ * @v source_size	Number of elements in source big integer
+ * @v dest0		Element 0 of destination big integer
+ * @v dest_size		Number of elements in destination big integer
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_grow_raw ( const unsigned long *source0, unsigned int source_size,
+		  unsigned long *dest0, unsigned int dest_size ) {
+	unsigned int pad_size = ( dest_size - source_size );
+
+	memcpy ( dest0, source0, sizeof ( bigint_t ( source_size ) ) );
+	memset ( ( dest0 + source_size ), 0, sizeof ( bigint_t ( pad_size ) ) );
+}
+
+/**
+ * Shrink big integer
+ *
+ * @v source0		Element 0 of source big integer
+ * @v source_size	Number of elements in source big integer
+ * @v dest0		Element 0 of destination big integer
+ * @v dest_size		Number of elements in destination big integer
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_shrink_raw ( const unsigned long *source0,
+		    unsigned int source_size __unused,
+		    unsigned long *dest0, unsigned int dest_size ) {
+
+	memcpy ( dest0, source0, sizeof ( bigint_t ( dest_size ) ) );
+}
+
+/**
+ * Finalise big integer
+ *
+ * @v value0		Element 0 of big integer to finalise
+ * @v size		Number of elements
+ * @v out		Output buffer
+ * @v len		Length of output buffer
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_done_raw ( const unsigned long *value0, unsigned int size __unused,
+		  void *out, size_t len ) {
+	const uint8_t *value_byte = ( ( const void * ) value0 );
+	uint8_t *out_byte = ( out + len );
+
+	/* Copy raw data in reverse order */
+	while ( len-- )
+		*(--out_byte) = *(value_byte++);
+}
+
+extern void bigint_multiply_raw ( const unsigned long *multiplicand0,
+				  unsigned int multiplicand_size,
+				  const unsigned long *multiplier0,
+				  unsigned int multiplier_size,
+				  unsigned long *value0 );
+
+#endif /* _BITS_BIGINT_H */
diff --git a/src/arch/riscv/include/bits/bitops.h b/src/arch/riscv/include/bits/bitops.h
new file mode 100644
index 000000000..2019db99a
--- /dev/null
+++ b/src/arch/riscv/include/bits/bitops.h
@@ -0,0 +1,82 @@
+#ifndef _BITS_BITOPS_H
+#define _BITS_BITOPS_H
+
+/** @file
+ *
+ * RISC-V bit operations
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/**
+ * Test and set bit atomically
+ *
+ * @v bit		Bit to set
+ * @v bits		Bit field
+ * @ret old		Old value of bit (zero or non-zero)
+ */
+static inline __attribute__ (( always_inline )) int
+test_and_set_bit ( unsigned int bit, volatile void *bits ) {
+	unsigned int index = ( bit / 32 );
+	unsigned int offset = ( bit % 32 );
+	volatile uint32_t *word = ( ( ( volatile uint32_t * ) bits ) + index );
+	uint32_t mask = ( 1U << offset );
+	uint32_t old;
+
+	__asm__ __volatile__ ( "amoor.w %0, %2, %1"
+			       : "=r" ( old ), "+A" ( *word )
+			       : "r" ( mask ) );
+
+	return ( !! ( old & mask ) );
+}
+
+/**
+ * Test and clear bit atomically
+ *
+ * @v bit		Bit to set
+ * @v bits		Bit field
+ * @ret old		Old value of bit (zero or non-zero)
+ */
+static inline __attribute__ (( always_inline )) int
+test_and_clear_bit ( unsigned int bit, volatile void *bits ) {
+	unsigned int index = ( bit / 32 );
+	unsigned int offset = ( bit % 32 );
+	volatile uint32_t *word = ( ( ( volatile uint32_t * ) bits ) + index );
+	uint32_t mask = ( 1U << offset );
+	uint32_t old;
+
+	__asm__ __volatile__ ( "amoand.w %0, %2, %1"
+			       : "=r" ( old ), "+A" ( *word )
+			       : "r" ( ~mask ) );
+
+	return ( !! ( old & mask ) );
+}
+
+/**
+ * Set bit atomically
+ *
+ * @v bit		Bit to set
+ * @v bits		Bit field
+ */
+static inline __attribute__ (( always_inline )) void
+set_bit ( unsigned int bit, volatile void *bits ) {
+
+	test_and_set_bit ( bit, bits );
+}
+
+/**
+ * Clear bit atomically
+ *
+ * @v bit		Bit to set
+ * @v bits		Bit field
+ */
+static inline __attribute__ (( always_inline )) void
+clear_bit ( unsigned int bit, volatile void *bits ) {
+
+	test_and_clear_bit ( bit, bits );
+}
+
+#endif /* _BITS_BITOPS_H */
diff --git a/src/arch/riscv/include/bits/byteswap.h b/src/arch/riscv/include/bits/byteswap.h
new file mode 100644
index 000000000..56d03f64e
--- /dev/null
+++ b/src/arch/riscv/include/bits/byteswap.h
@@ -0,0 +1,48 @@
+#ifndef _BITS_BYTESWAP_H
+#define _BITS_BYTESWAP_H
+
+/** @file
+ *
+ * Byte-order swapping functions
+ *
+ */
+
+#include <stdint.h>
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+extern __asmcall uint64_t riscv_swap_word ( uint64_t x );
+extern __asmcall unsigned long riscv_swap_half ( unsigned long x );
+extern __asmcall unsigned long riscv_swap_byte ( unsigned long x );
+
+static inline __attribute__ (( always_inline, const )) uint16_t
+__bswap_variable_16 ( uint16_t x ) {
+	return riscv_swap_byte ( x );
+}
+
+static inline __attribute__ (( always_inline )) void
+__bswap_16s ( uint16_t *x ) {
+	*x = riscv_swap_byte ( *x );
+}
+
+static inline __attribute__ (( always_inline, const )) uint32_t
+__bswap_variable_32 ( uint32_t x ) {
+	return riscv_swap_half ( x );
+}
+
+static inline __attribute__ (( always_inline )) void
+__bswap_32s ( uint32_t *x ) {
+	*x = riscv_swap_half ( *x );
+}
+
+static inline __attribute__ (( always_inline, const )) uint64_t
+__bswap_variable_64 ( uint64_t x ) {
+	return riscv_swap_word ( x );
+}
+
+static inline __attribute__ (( always_inline )) void
+__bswap_64s ( uint64_t *x ) {
+	*x = riscv_swap_word ( *x );
+}
+
+#endif /* _BITS_BYTESWAP_H */
diff --git a/src/arch/riscv/include/bits/compiler.h b/src/arch/riscv/include/bits/compiler.h
new file mode 100644
index 000000000..624a16108
--- /dev/null
+++ b/src/arch/riscv/include/bits/compiler.h
@@ -0,0 +1,40 @@
+#ifndef _BITS_COMPILER_H
+#define _BITS_COMPILER_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Dummy relocation type */
+#define RELOC_TYPE_NONE R_RISCV_NONE
+
+/* Determine load/store instructions for natural bit width */
+#if __riscv_xlen == 128
+#define NATURAL_SUFFIX q
+#elif __riscv_xlen == 64
+#define NATURAL_SUFFIX d
+#elif __riscv_xlen == 32
+#define NATURAL_SUFFIX w
+#else
+#error "Unsupported bit width"
+#endif
+#ifdef ASSEMBLY
+#define LOADN _C2 ( L, NATURAL_SUFFIX )
+#define STOREN _C2 ( S, NATURAL_SUFFIX )
+#else
+#define LOADN "L" _S2 ( NATURAL_SUFFIX )
+#define STOREN "S" _S2 ( NATURAL_SUFFIX )
+#endif
+
+#ifndef ASSEMBLY
+
+/** Unprefixed constant operand modifier */
+#define ASM_NO_PREFIX ""
+
+/** Declare a function with standard calling conventions */
+#define __asmcall
+
+/** Declare a function with libgcc implicit linkage */
+#define __libgcc
+
+#endif /* ASSEMBLY */
+
+#endif /* _BITS_COMPILER_H */
diff --git a/src/arch/riscv/include/bits/endian.h b/src/arch/riscv/include/bits/endian.h
new file mode 100644
index 000000000..85718cfdd
--- /dev/null
+++ b/src/arch/riscv/include/bits/endian.h
@@ -0,0 +1,8 @@
+#ifndef _BITS_ENDIAN_H
+#define _BITS_ENDIAN_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#define __BYTE_ORDER __LITTLE_ENDIAN
+
+#endif /* _BITS_ENDIAN_H */
diff --git a/src/arch/riscv/include/bits/errfile.h b/src/arch/riscv/include/bits/errfile.h
new file mode 100644
index 000000000..2b1202147
--- /dev/null
+++ b/src/arch/riscv/include/bits/errfile.h
@@ -0,0 +1,19 @@
+#ifndef _BITS_ERRFILE_H
+#define _BITS_ERRFILE_H
+
+/** @file
+ *
+ * RISC-V error file identifiers
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @addtogroup errfile Error file identifiers
+ * @{
+ */
+
+/** @} */
+
+#endif /* _BITS_ERRFILE_H */
diff --git a/src/arch/riscv/include/bits/io.h b/src/arch/riscv/include/bits/io.h
new file mode 100644
index 000000000..4296e318a
--- /dev/null
+++ b/src/arch/riscv/include/bits/io.h
@@ -0,0 +1,17 @@
+#ifndef _BITS_IO_H
+#define _BITS_IO_H
+
+/** @file
+ *
+ * RISCV-specific I/O API implementations
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Page shift */
+#define PAGE_SHIFT 12
+
+#include <ipxe/riscv_io.h>
+
+#endif /* _BITS_IO_H */
diff --git a/src/arch/riscv/include/bits/nap.h b/src/arch/riscv/include/bits/nap.h
new file mode 100644
index 000000000..331399f46
--- /dev/null
+++ b/src/arch/riscv/include/bits/nap.h
@@ -0,0 +1,20 @@
+#ifndef _BITS_NAP_H
+#define _BITS_NAP_H
+
+/** @file
+ *
+ * RISCV-specific CPU sleeping API implementations
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Sleep until next CPU interrupt
+ *
+ */
+static inline __attribute__ (( always_inline )) void cpu_halt ( void ) {
+	__asm__ __volatile__ ( "wfi" );
+}
+
+#endif /* _BITS_NAP_H */
diff --git a/src/arch/riscv/include/bits/setjmp.h b/src/arch/riscv/include/bits/setjmp.h
new file mode 100644
index 000000000..5186fadaf
--- /dev/null
+++ b/src/arch/riscv/include/bits/setjmp.h
@@ -0,0 +1,16 @@
+#ifndef _BITS_SETJMP_H
+#define _BITS_SETJMP_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** A jump buffer */
+typedef struct {
+	/** Return address (ra) */
+	unsigned long ra;
+	/** Stack pointer (sp) */
+	unsigned long sp;
+	/** Callee-saved registers (s0-s11) */
+	unsigned long s[12];
+} jmp_buf[1];
+
+#endif /* _BITS_SETJMP_H */
diff --git a/src/arch/riscv/include/bits/stdint.h b/src/arch/riscv/include/bits/stdint.h
new file mode 100644
index 000000000..fe1f9946a
--- /dev/null
+++ b/src/arch/riscv/include/bits/stdint.h
@@ -0,0 +1,23 @@
+#ifndef _BITS_STDINT_H
+#define _BITS_STDINT_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+typedef __SIZE_TYPE__		size_t;
+typedef signed long		ssize_t;
+typedef signed long		off_t;
+
+typedef unsigned char		uint8_t;
+typedef unsigned short		uint16_t;
+typedef unsigned int		uint32_t;
+typedef unsigned long long	uint64_t;
+
+typedef signed char		int8_t;
+typedef signed short		int16_t;
+typedef signed int		int32_t;
+typedef signed long long	int64_t;
+
+typedef unsigned long		physaddr_t;
+typedef unsigned long		intptr_t;
+
+#endif /* _BITS_STDINT_H */
diff --git a/src/arch/riscv/include/bits/string.h b/src/arch/riscv/include/bits/string.h
new file mode 100644
index 000000000..3f78b351d
--- /dev/null
+++ b/src/arch/riscv/include/bits/string.h
@@ -0,0 +1,89 @@
+#ifndef _BITS_STRING_H
+#define _BITS_STRING_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * String functions
+ *
+ */
+
+extern void riscv_bzero ( void *dest, size_t len );
+extern void riscv_memset ( void *dest, size_t len, int character );
+extern void riscv_memcpy ( void *dest, const void *src, size_t len );
+extern void riscv_memmove_forwards ( void *dest, const void *src, size_t len );
+extern void riscv_memmove_backwards ( void *dest, const void *src, size_t len );
+extern void riscv_memmove ( void *dest, const void *src, size_t len );
+
+/**
+ * Fill memory region
+ *
+ * @v dest		Destination region
+ * @v character		Fill character
+ * @v len		Length
+ * @ret dest		Destination region
+ */
+static inline __attribute__ (( always_inline )) void *
+memset ( void *dest, int character, size_t len ) {
+
+	/* For zeroing larger or non-constant lengths, use the
+	 * optimised variable-length zeroing code.
+	 */
+	if ( __builtin_constant_p ( character ) && ( character == 0 ) ) {
+		riscv_bzero ( dest, len );
+		return dest;
+	}
+
+	/* Not necessarily zeroing: use basic variable-length code */
+	riscv_memset ( dest, len, character );
+	return dest;
+}
+
+/**
+ * Copy memory region
+ *
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ * @ret dest		Destination region
+ */
+static inline __attribute__ (( always_inline )) void *
+memcpy ( void *dest, const void *src, size_t len ) {
+
+	/* Otherwise, use variable-length code */
+	riscv_memcpy ( dest, src, len );
+	return dest;
+}
+
+/**
+ * Copy (possibly overlapping) memory region
+ *
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ * @ret dest		Destination region
+ */
+static inline __attribute__ (( always_inline )) void *
+memmove ( void *dest, const void *src, size_t len ) {
+	ssize_t offset = ( dest - src );
+
+	/* If required direction of copy is known at build time, then
+	 * use the appropriate forwards/backwards copy directly.
+	 */
+	if ( __builtin_constant_p ( offset ) ) {
+		if ( offset <= 0 ) {
+			riscv_memmove_forwards ( dest, src, len );
+			return dest;
+		} else {
+			riscv_memmove_backwards ( dest, src, len );
+			return dest;
+		}
+	}
+
+	/* Otherwise, use ambidirectional copy */
+	riscv_memmove ( dest, src, len );
+	return dest;
+}
+
+#endif /* _BITS_STRING_H */
diff --git a/src/arch/riscv/include/bits/strings.h b/src/arch/riscv/include/bits/strings.h
new file mode 100644
index 000000000..dd6d458b2
--- /dev/null
+++ b/src/arch/riscv/include/bits/strings.h
@@ -0,0 +1,91 @@
+#ifndef _BITS_STRINGS_H
+#define _BITS_STRINGS_H
+
+/** @file
+ *
+ * String functions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+extern __asmcall unsigned long riscv_ffs ( unsigned long value );
+extern __asmcall unsigned long riscv_fls ( unsigned long value );
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsl ( long value ) {
+
+	return riscv_ffs ( value );
+}
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsll ( long long value ){
+	unsigned long low = value;
+	unsigned long high;
+
+	/* Check machine word size */
+	if ( sizeof ( value ) > sizeof ( low ) ) {
+		/* 32-bit */
+		high = ( value >> 32 );
+		if ( low ) {
+			return ( __ffsl ( low ) );
+		} else if ( high ) {
+			return ( 32 + __ffsl ( high ) );
+		} else {
+			return 0;
+		}
+	} else {
+		/* 64-bit */
+		return ( __ffsl ( low ) );
+	}
+}
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v value		Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __flsl ( long value ) {
+
+	return riscv_fls ( value );
+}
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v value		Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __flsll ( long long value ){
+	unsigned long low = value;
+	unsigned long high;
+
+	/* Check machine word size */
+	if ( sizeof ( value ) > sizeof ( low ) ) {
+		/* 32-bit */
+		high = ( value >> 32 );
+		if ( high ) {
+			return ( 32 + __flsl ( high ) );
+		} else if ( low ) {
+			return ( __flsl ( low ) );
+		} else {
+			return 0;
+		}
+	} else {
+		/* 64-bit */
+		return ( __flsl ( low ) );
+	}
+}
+
+#endif /* _BITS_STRINGS_H */
diff --git a/src/arch/riscv/include/ipxe/riscv_io.h b/src/arch/riscv/include/ipxe/riscv_io.h
new file mode 100644
index 000000000..8f9030418
--- /dev/null
+++ b/src/arch/riscv/include/ipxe/riscv_io.h
@@ -0,0 +1,137 @@
+#ifndef _IPXE_RISCV_IO_H
+#define _IPXE_RISCV_IO_H
+
+/** @file
+ *
+ * iPXE I/O API for RISC-V
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#ifdef IOAPI_RISCV
+#define IOAPI_PREFIX_riscv
+#else
+#define IOAPI_PREFIX_riscv __riscv_
+#endif
+
+#include <ipxe/dummy_pio.h>
+
+/*
+ * Memory space mappings
+ *
+ */
+
+/*
+ * Physical<->Bus address mappings
+ *
+ */
+
+static inline __always_inline unsigned long
+IOAPI_INLINE ( riscv, phys_to_bus ) ( unsigned long phys_addr ) {
+	return phys_addr;
+}
+
+static inline __always_inline unsigned long
+IOAPI_INLINE ( riscv, bus_to_phys ) ( unsigned long bus_addr ) {
+	return bus_addr;
+}
+
+/*
+ * MMIO reads and writes
+ *
+ */
+
+/* Single-register read */
+#define RISCV_READX( _suffix, _type, _insn_suffix )			\
+static inline __always_inline _type					\
+IOAPI_INLINE ( riscv, read ## _suffix ) ( volatile _type *io_addr ) {	\
+	unsigned long data;						\
+	__asm__ __volatile__ ( "l" _insn_suffix " %0, %1"		\
+			       : "=r" ( data ) : "m" ( *io_addr ) );	\
+	return data;							\
+}
+
+/* Single-register write */
+#define RISCV_WRITEX( _suffix, _type, _insn_suffix)			\
+static inline __always_inline void					\
+IOAPI_INLINE ( riscv, write ## _suffix ) ( _type data,			\
+					   volatile _type *io_addr ) {	\
+	__asm__ __volatile__ ( "s" _insn_suffix " %0, %1"		\
+			       : : "r" ( data ), "m" ( *io_addr ) );	\
+}
+
+/* Double-register hopefully-fused read */
+#define RISCV_READX_FUSED( _suffix, _type, _insn_suffix )		\
+static inline __always_inline _type					\
+IOAPI_INLINE ( riscv, read ## _suffix ) ( volatile _type *io_addr ) {	\
+	union {								\
+		unsigned long half[2];					\
+	        _type data;						\
+	} u;								\
+	__asm__ __volatile__ ( "l" _insn_suffix " %0, 0(%2)\n\t"	\
+			       "l" _insn_suffix " %1, %3(%2)\n\t"	\
+			       : "=&r" ( u.half[0] ),			\
+				 "=&r" ( u.half[1] )			\
+			       : "r" ( io_addr ),			\
+				 "i" ( sizeof ( u.half[0] ) ) );	\
+	return u.data;							\
+}
+
+/* Double-register hopefully-fused write */
+#define RISCV_WRITEX_FUSED( _suffix, _type, _insn_suffix )		\
+static inline __always_inline void					\
+IOAPI_INLINE ( riscv, write ## _suffix ) ( _type data,			\
+					   volatile _type *io_addr ) {	\
+	union {								\
+		unsigned long half[2];					\
+		_type data;						\
+	} u = { .data = data };						\
+	__asm__ __volatile__ ( "s" _insn_suffix " %0, 0(%2)\n\t"	\
+			       "s" _insn_suffix " %1, %3(%2)\n\t" :	\
+			       : "r" ( u.half[0] ),			\
+				 "r" ( u.half[1] ),			\
+				 "r" ( io_addr ),			\
+				 "i" ( sizeof ( u.half[0] ) ) );	\
+}
+
+RISCV_READX ( b, uint8_t, "bu" );
+RISCV_WRITEX ( b, uint8_t, "b" );
+
+RISCV_READX ( w, uint16_t, "hu" );
+RISCV_WRITEX ( w, uint16_t, "h" );
+
+#if __riscv_xlen > 32
+  RISCV_READX ( l, uint32_t, "wu" );
+  RISCV_WRITEX ( l, uint32_t, "w" );
+#else
+  RISCV_READX ( l, uint32_t, "w" );
+  RISCV_WRITEX ( l, uint32_t, "w" );
+#endif
+
+#if __riscv_xlen >= 64
+  #if __riscv_xlen > 64
+    RISCV_READX ( q, uint64_t, "du" );
+    RISCV_WRITEX ( q, uint64_t, "d" );
+  #else
+    RISCV_READX ( q, uint64_t, "d" );
+    RISCV_WRITEX ( q, uint64_t, "d" );
+  #endif
+#else
+  RISCV_READX_FUSED ( q, uint64_t, "w" );
+  RISCV_WRITEX_FUSED ( q, uint64_t, "w" );
+#endif
+
+/*
+ * Memory barrier
+ *
+ */
+static inline __always_inline void
+IOAPI_INLINE ( riscv, mb ) ( void ) {
+	__asm__ __volatile__ ( "fence rw, rw" );
+}
+
+/* Dummy PIO */
+DUMMY_PIO ( riscv );
+
+#endif /* _IPXE_RISCV_IO_H */
diff --git a/src/arch/riscv32/Makefile b/src/arch/riscv32/Makefile
new file mode 100644
index 000000000..0415f7c2d
--- /dev/null
+++ b/src/arch/riscv32/Makefile
@@ -0,0 +1,20 @@
+# RISCV32-specific directories containing source files
+#
+SRCDIRS		+= arch/riscv32/core
+SRCDIRS		+= arch/riscv32/libgcc
+
+# RISCV32-specific flags
+#
+CFLAGS		+= -march=rv32gc -mabi=ilp32d
+ASFLAGS		+= -march=rv32gc -mabi=ilp32d
+LDFLAGS		+= -m elf32lriscv
+
+# Include common RISCV Makefile
+#
+MAKEDEPS	+= arch/riscv/Makefile
+include arch/riscv/Makefile
+
+# Include platform-specific Makefile
+#
+MAKEDEPS	+= arch/riscv32/Makefile.$(PLATFORM)
+include arch/riscv32/Makefile.$(PLATFORM)
diff --git a/src/arch/riscv32/Makefile.efi b/src/arch/riscv32/Makefile.efi
new file mode 100644
index 000000000..db83c8c14
--- /dev/null
+++ b/src/arch/riscv32/Makefile.efi
@@ -0,0 +1,14 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Specify EFI image builder
+#
+ELF2EFI		= $(ELF2EFI32)
+
+# Specify EFI boot file
+#
+EFI_BOOT_FILE	= bootriscv32.efi
+
+# Include generic EFI Makefile
+#
+MAKEDEPS	+= arch/riscv/Makefile.efi
+include arch/riscv/Makefile.efi
diff --git a/src/arch/riscv32/Makefile.linux b/src/arch/riscv32/Makefile.linux
new file mode 100644
index 000000000..86ac6b8d5
--- /dev/null
+++ b/src/arch/riscv32/Makefile.linux
@@ -0,0 +1,14 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Starting virtual address
+#
+LDFLAGS += -Ttext=0x10000
+
+# Compiler flags for building host API wrapper
+#
+LINUX_CFLAGS += -march=rv32gc -mabi=ilp32d
+
+# Include generic Linux Makefile
+#
+MAKEDEPS += arch/riscv/Makefile.linux
+include arch/riscv/Makefile.linux
diff --git a/src/arch/riscv32/core/riscv32_byteswap.S b/src/arch/riscv32/core/riscv32_byteswap.S
new file mode 100644
index 000000000..571431f35
--- /dev/null
+++ b/src/arch/riscv32/core/riscv32_byteswap.S
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * Byte swapping
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+	.section ".text.riscv_swap", "ax", @progbits
+riscv_swap:
+	.globl	riscv_swap_word
+	.globl	riscv_swap_half
+	.globl	riscv_swap_byte
+riscv_swap_word:
+	/* Swap all bytes in a0 */
+	mv	t0, ra
+	jal	riscv_swap_half
+	mv	ra, t0
+	/* Swap words a0 and a1 */
+	mv	t1, a0
+	mv	a0, a1
+	mv	a1, t1
+riscv_swap_half:
+	/* Swap half-words within a0 */
+	slli	t2, a0, 16
+	srli	a0, a0, 16
+	or	a0, a0, t2
+riscv_swap_byte:
+	/* Swap bytes within each half-word of a0 */
+	li	t3, 0xff00ff00
+	slli	t4, a0, 8
+	and	a0, a0, t3
+	and	t4, t4, t3
+	srli	a0, a0, 8
+	or	a0, a0, t4
+	ret
+	.size	riscv_swap, . - riscv_swap
diff --git a/src/arch/riscv32/include/bits/profile.h b/src/arch/riscv32/include/bits/profile.h
new file mode 100644
index 000000000..cb969077d
--- /dev/null
+++ b/src/arch/riscv32/include/bits/profile.h
@@ -0,0 +1,36 @@
+#ifndef _BITS_PROFILE_H
+#define _BITS_PROFILE_H
+
+/** @file
+ *
+ * Profiling
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/**
+ * Get profiling timestamp
+ *
+ * @ret timestamp	Timestamp
+ */
+static inline __attribute__ (( always_inline )) uint64_t
+profile_timestamp ( void ) {
+	uint32_t cycles_lo;
+	uint32_t cycles_hi;
+	uint32_t tmp;
+
+	/* Read timestamp counter */
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       "rdcycleh %1\n\t"
+			       "rdcycle %0\n\t"
+			       "rdcycleh %2\n\t"
+			       "bne %1, %2, 1b\n\t"
+			       : "=r" ( cycles_lo ), "=r" ( cycles_hi ),
+				 "=r" ( tmp ) );
+	return ( ( ( ( uint64_t ) cycles_hi ) << 32 ) | cycles_lo );
+}
+
+#endif /* _BITS_PROFILE_H */
diff --git a/src/arch/riscv32/include/ipxe/efi/dhcparch.h b/src/arch/riscv32/include/ipxe/efi/dhcparch.h
new file mode 100644
index 000000000..b74df1925
--- /dev/null
+++ b/src/arch/riscv32/include/ipxe/efi/dhcparch.h
@@ -0,0 +1,20 @@
+#ifndef _IPXE_EFI_DHCPARCH_H
+#define _IPXE_EFI_DHCPARCH_H
+
+/** @file
+ *
+ * DHCP client architecture definitions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/dhcp.h>
+
+/** DHCP client architecture */
+#define DHCP_ARCH_CLIENT_ARCHITECTURE DHCP_CLIENT_ARCHITECTURE_RISCV32
+
+/** DHCP client network device interface */
+#define DHCP_ARCH_CLIENT_NDI 1 /* UNDI */ , 3, 10 /* v3.10 */
+
+#endif /* _IPXE_EFI_DHCPARCH_H */
diff --git a/src/arch/riscv32/include/limits.h b/src/arch/riscv32/include/limits.h
new file mode 100644
index 000000000..bb48b75ab
--- /dev/null
+++ b/src/arch/riscv32/include/limits.h
@@ -0,0 +1,61 @@
+#ifndef LIMITS_H
+#define LIMITS_H	1
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/* Number of bits in a `char' */
+#define CHAR_BIT	8
+
+/* Minimum and maximum values a `signed char' can hold */
+#define SCHAR_MIN	(-128)
+#define SCHAR_MAX	127
+
+/* Maximum value an `unsigned char' can hold. (Minimum is 0.) */
+#define UCHAR_MAX	255
+
+/* Minimum and maximum values a `char' can hold */
+#define CHAR_MIN	SCHAR_MIN
+#define CHAR_MAX	SCHAR_MAX
+
+/* Minimum and maximum values a `signed short int' can hold */
+#define SHRT_MIN	(-32768)
+#define SHRT_MAX	32767
+
+/* Maximum value an `unsigned short' can hold. (Minimum is 0.) */
+#define USHRT_MAX	65535
+
+
+/* Minimum and maximum values a `signed int' can hold */
+#define INT_MIN		(-INT_MAX - 1)
+#define INT_MAX		2147483647
+
+/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */
+#define UINT_MAX	4294967295U
+
+
+/* Minimum and maximum values a `signed int' can hold */
+#define INT_MAX		2147483647
+#define INT_MIN		(-INT_MAX - 1)
+
+
+/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */
+#define UINT_MAX	4294967295U
+
+
+/* Minimum and maximum values a `signed long' can hold */
+#define LONG_MAX	2147483647
+#define LONG_MIN	(-LONG_MAX - 1L)
+
+/* Maximum value an `unsigned long' can hold. (Minimum is 0.) */
+#define ULONG_MAX	4294967295UL
+
+/* Minimum and maximum values a `signed long long' can hold */
+#define LLONG_MAX	9223372036854775807LL
+#define LLONG_MIN	(-LONG_MAX - 1LL)
+
+
+/* Maximum value an `unsigned long long' can hold. (Minimum is 0.) */
+#define ULLONG_MAX	18446744073709551615ULL
+
+
+#endif /* LIMITS_H */
diff --git a/src/arch/riscv32/libgcc/llshift.S b/src/arch/riscv32/libgcc/llshift.S
new file mode 100644
index 000000000..8ad6a6daa
--- /dev/null
+++ b/src/arch/riscv32/libgcc/llshift.S
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * Long shifts
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+/**
+ * Shift left
+ *
+ * @v a1:a0		Value to shift
+ * @v a2		Shift amount
+ * @ret a1:a0		Shifted value
+ */
+	.section ".text.__ashldi3", "ax", @progbits
+	.globl	__ashldi3
+__ashldi3:
+	/* Perform shift by 32 bits, if applicable */
+	li	t0, 32
+	sub	t1, t0, a2
+	bgtz	t1, 1f
+	mv	a1, a0
+	mv	a0, zero
+1:	/* Perform shift by modulo-32 bits, if applicable */
+	andi	a2, a2, 0x1f
+	beqz	a2, 2f
+	srl	t2, a0, t1
+	sll	a0, a0, a2
+	sll	a1, a1, a2
+	or	a1, a1, t2
+2:	ret
+	.size	__ashldi3, . - __ashldi3
+
+/**
+ * Logical shift right
+ *
+ * @v a1:a0		Value to shift
+ * @v a2		Shift amount
+ * @ret a1:a0		Shifted value
+ */
+	.section ".text.__lshrdi3", "ax", @progbits
+	.globl	__lshrdi3
+__lshrdi3:
+	/* Perform shift by 32 bits, if applicable */
+	li	t0, 32
+	sub	t1, t0, a2
+	bgtz	t1, 1f
+	mv	a0, a1
+	mv	a1, zero
+1:	/* Perform shift by modulo-32 bits, if applicable */
+	andi	a2, a2, 0x1f
+	beqz	a2, 2f
+	sll	t2, a1, t1
+	srl	a1, a1, a2
+	srl	a0, a0, a2
+	or	a0, a0, t2
+2:	ret
+	.size	__lshrdi3, . - __lshrdi3
+
+/**
+ * Arithmetic shift right
+ *
+ * @v a1:a0		Value to shift
+ * @v a2		Shift amount
+ * @ret a1:a0		Shifted value
+ */
+	.section ".text.__ashrdi3", "ax", @progbits
+	.globl	__ashrdi3
+__ashrdi3:
+	/* Perform shift by 32 bits, if applicable */
+	li	t0, 32
+	sub	t1, t0, a2
+	bgtz	t1, 1f
+	mv	a0, a1
+	srai	a1, a1, 16
+	srai	a1, a1, 16
+1:	/* Perform shift by modulo-32 bits, if applicable */
+	andi	a2, a2, 0x1f
+	beqz	a2, 2f
+	sll	t2, a1, t1
+	sra	a1, a1, a2
+	srl	a0, a0, a2
+	or	a0, a0, t2
+2:	ret
+	.size	__ashrdi3, . - __ashrdi3
diff --git a/src/arch/riscv64/Makefile b/src/arch/riscv64/Makefile
new file mode 100644
index 000000000..2fd18fc13
--- /dev/null
+++ b/src/arch/riscv64/Makefile
@@ -0,0 +1,19 @@
+# RISCV64-specific directories containing source files
+#
+SRCDIRS		+= arch/riscv64/core
+
+# RISCV64-specific flags
+#
+CFLAGS		+= -march=rv64gc -mabi=lp64d
+ASFLAGS		+= -march=rv64gc -mabi=lp64d
+LDFLAGS		+= -m elf64lriscv
+
+# Include common RISCV Makefile
+#
+MAKEDEPS	+= arch/riscv/Makefile
+include arch/riscv/Makefile
+
+# Include platform-specific Makefile
+#
+MAKEDEPS	+= arch/riscv64/Makefile.$(PLATFORM)
+include arch/riscv64/Makefile.$(PLATFORM)
diff --git a/src/arch/riscv64/Makefile.efi b/src/arch/riscv64/Makefile.efi
new file mode 100644
index 000000000..b19e036a3
--- /dev/null
+++ b/src/arch/riscv64/Makefile.efi
@@ -0,0 +1,14 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Specify EFI image builder
+#
+ELF2EFI		= $(ELF2EFI64)
+
+# Specify EFI boot file
+#
+EFI_BOOT_FILE	= bootriscv64.efi
+
+# Include generic EFI Makefile
+#
+MAKEDEPS	+= arch/riscv/Makefile.efi
+include arch/riscv/Makefile.efi
diff --git a/src/arch/riscv64/Makefile.linux b/src/arch/riscv64/Makefile.linux
new file mode 100644
index 000000000..5ceafed8c
--- /dev/null
+++ b/src/arch/riscv64/Makefile.linux
@@ -0,0 +1,10 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Starting virtual address
+#
+LDFLAGS += -Ttext=0x10000
+
+# Include generic Linux Makefile
+#
+MAKEDEPS += arch/riscv/Makefile.linux
+include arch/riscv/Makefile.linux
diff --git a/src/arch/riscv64/core/riscv64_byteswap.S b/src/arch/riscv64/core/riscv64_byteswap.S
new file mode 100644
index 000000000..ec2b0b9dd
--- /dev/null
+++ b/src/arch/riscv64/core/riscv64_byteswap.S
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * Byte swapping
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+	.section ".text.riscv_swap", "ax", @progbits
+riscv_swap:
+	.globl	riscv_swap_word
+	.globl	riscv_swap_half
+	.globl	riscv_swap_byte
+riscv_swap_word:
+	/* Swap low and high words of a0 */
+	slli	t0, a0, 32
+	srli	a0, a0, 32
+	or	a0, a0, t0
+riscv_swap_half:
+	/* Swap half-words within each word of a0 */
+	ld	t1, mask16
+	slli	t2, a0, 16
+	and	a0, a0, t1
+	and	t2, t2, t1
+	srli	a0, a0, 16
+	or	a0, a0, t2
+riscv_swap_byte:
+	/* Swap bytes within each half-word of a0 */
+	ld	t3, mask8
+	slli	t4, a0, 8
+	and	a0, a0, t3
+	and	t4, t4, t3
+	srli	a0, a0, 8
+	or	a0, a0, t4
+	ret
+mask16:	.dword 0xffff0000ffff0000
+mask8:	.dword 0xff00ff00ff00ff00
+	.size	riscv_swap, . - riscv_swap
diff --git a/src/arch/riscv64/include/bits/profile.h b/src/arch/riscv64/include/bits/profile.h
new file mode 100644
index 000000000..2c8e29a22
--- /dev/null
+++ b/src/arch/riscv64/include/bits/profile.h
@@ -0,0 +1,28 @@
+#ifndef _BITS_PROFILE_H
+#define _BITS_PROFILE_H
+
+/** @file
+ *
+ * Profiling
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/**
+ * Get profiling timestamp
+ *
+ * @ret timestamp	Timestamp
+ */
+static inline __attribute__ (( always_inline )) uint64_t
+profile_timestamp ( void ) {
+	uint64_t cycles;
+
+	/* Read timestamp counter */
+	__asm__ __volatile__ ( "rdcycle %0" : "=r" ( cycles ) );
+	return cycles;
+}
+
+#endif /* _BITS_PROFILE_H */
diff --git a/src/arch/riscv64/include/ipxe/efi/dhcparch.h b/src/arch/riscv64/include/ipxe/efi/dhcparch.h
new file mode 100644
index 000000000..33bca044e
--- /dev/null
+++ b/src/arch/riscv64/include/ipxe/efi/dhcparch.h
@@ -0,0 +1,20 @@
+#ifndef _IPXE_EFI_DHCPARCH_H
+#define _IPXE_EFI_DHCPARCH_H
+
+/** @file
+ *
+ * DHCP client architecture definitions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/dhcp.h>
+
+/** DHCP client architecture */
+#define DHCP_ARCH_CLIENT_ARCHITECTURE DHCP_CLIENT_ARCHITECTURE_RISCV64
+
+/** DHCP client network device interface */
+#define DHCP_ARCH_CLIENT_NDI 1 /* UNDI */ , 3, 10 /* v3.10 */
+
+#endif /* _IPXE_EFI_DHCPARCH_H */
diff --git a/src/arch/riscv64/include/limits.h b/src/arch/riscv64/include/limits.h
new file mode 100644
index 000000000..a1374a17f
--- /dev/null
+++ b/src/arch/riscv64/include/limits.h
@@ -0,0 +1,61 @@
+#ifndef LIMITS_H
+#define LIMITS_H	1
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/* Number of bits in a `char' */
+#define CHAR_BIT	8
+
+/* Minimum and maximum values a `signed char' can hold */
+#define SCHAR_MIN	(-128)
+#define SCHAR_MAX	127
+
+/* Maximum value an `unsigned char' can hold. (Minimum is 0.) */
+#define UCHAR_MAX	255
+
+/* Minimum and maximum values a `char' can hold */
+#define CHAR_MIN	SCHAR_MIN
+#define CHAR_MAX	SCHAR_MAX
+
+/* Minimum and maximum values a `signed short int' can hold */
+#define SHRT_MIN	(-32768)
+#define SHRT_MAX	32767
+
+/* Maximum value an `unsigned short' can hold. (Minimum is 0.) */
+#define USHRT_MAX	65535
+
+
+/* Minimum and maximum values a `signed int' can hold */
+#define INT_MIN		(-INT_MAX - 1)
+#define INT_MAX		2147483647
+
+/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */
+#define UINT_MAX	4294967295U
+
+
+/* Minimum and maximum values a `signed int' can hold */
+#define INT_MAX		2147483647
+#define INT_MIN		(-INT_MAX - 1)
+
+
+/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */
+#define UINT_MAX	4294967295U
+
+
+/* Minimum and maximum values a `signed long' can hold */
+#define LONG_MAX	9223372036854775807L
+#define LONG_MIN	(-LONG_MAX - 1L)
+
+/* Maximum value an `unsigned long' can hold. (Minimum is 0.) */
+#define ULONG_MAX	18446744073709551615UL
+
+/* Minimum and maximum values a `signed long long' can hold */
+#define LLONG_MAX	9223372036854775807LL
+#define LLONG_MIN	(-LONG_MAX - 1LL)
+
+
+/* Maximum value an `unsigned long long' can hold. (Minimum is 0.) */
+#define ULLONG_MAX	18446744073709551615ULL
+
+
+#endif /* LIMITS_H */
diff --git a/src/config/defaults/efi.h b/src/config/defaults/efi.h
index 178b448cb..b32381de6 100644
--- a/src/config/defaults/efi.h
+++ b/src/config/defaults/efi.h
@@ -71,4 +71,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define IOAPI_LOONG64
 #endif
 
+#if defined ( __riscv )
+#define IOAPI_RISCV
+#endif
+
 #endif /* CONFIG_DEFAULTS_EFI_H */
diff --git a/src/include/ipxe/efi/ProcessorBind.h b/src/include/ipxe/efi/ProcessorBind.h
index dbccf346d..08854c84f 100644
--- a/src/include/ipxe/efi/ProcessorBind.h
+++ b/src/include/ipxe/efi/ProcessorBind.h
@@ -65,6 +65,10 @@ typedef uint8_t BOOLEAN;
 #include <ipxe/efi/LoongArch64/ProcessorBind.h>
 #endif
 
+#ifdef __riscv
+#include <ipxe/efi/RiscV64/ProcessorBind.h>
+#endif
+
 #endif /* EFI_HOSTONLY */
 
 #endif /* _IPXE_EFI_PROCESSOR_BIND_H */
diff --git a/src/include/ipxe/efi/RiscV64/ProcessorBind.h b/src/include/ipxe/efi/RiscV64/ProcessorBind.h
new file mode 100644
index 000000000..37dd9fc3b
--- /dev/null
+++ b/src/include/ipxe/efi/RiscV64/ProcessorBind.h
@@ -0,0 +1,175 @@
+/** @file
+  Processor or Compiler specific defines and types for RISC-V
+
+  Copyright (c) 2016 - 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#ifndef PROCESSOR_BIND_H__
+#define PROCESSOR_BIND_H__
+
+FILE_LICENCE ( BSD2_PATENT );
+
+///
+/// Define the processor type so other code can make processor based choices
+///
+#define MDE_CPU_RISCV64
+
+//
+// Make sure we are using the correct packing rules per EFI specification
+//
+#if !defined (__GNUC__)
+  #pragma pack()
+#endif
+
+///
+/// 8-byte unsigned value
+///
+typedef unsigned long long UINT64  __attribute__ ((aligned (8)));
+///
+/// 8-byte signed value
+///
+typedef long long INT64  __attribute__ ((aligned (8)));
+///
+/// 4-byte unsigned value
+///
+typedef unsigned int UINT32 __attribute__ ((aligned (4)));
+///
+/// 4-byte signed value
+///
+typedef int INT32  __attribute__ ((aligned (4)));
+///
+/// 2-byte unsigned value
+///
+typedef unsigned short UINT16  __attribute__ ((aligned (2)));
+///
+/// 2-byte Character.  Unless otherwise specified all strings are stored in the
+/// UTF-16 encoding format as defined by Unicode 2.1 and ISO/IEC 10646 standards.
+///
+typedef unsigned short CHAR16  __attribute__ ((aligned (2)));
+///
+/// 2-byte signed value
+///
+typedef short INT16  __attribute__ ((aligned (2)));
+///
+/// Logical Boolean.  1-byte value containing 0 for FALSE or a 1 for TRUE.  Other
+/// values are undefined.
+///
+typedef unsigned char BOOLEAN;
+///
+/// 1-byte unsigned value
+///
+typedef unsigned char UINT8;
+///
+/// 1-byte Character
+///
+typedef char CHAR8;
+///
+/// 1-byte signed value
+///
+typedef signed char INT8;
+///
+/// Unsigned value of native width.  (4 bytes on supported 32-bit processor instructions,
+/// 8 bytes on supported 64-bit processor instructions)
+///
+typedef UINT64 UINTN __attribute__ ((aligned (8)));
+///
+/// Signed value of native width.  (4 bytes on supported 32-bit processor instructions,
+/// 8 bytes on supported 64-bit processor instructions)
+///
+typedef INT64 INTN __attribute__ ((aligned (8)));
+
+//
+// Processor specific defines
+//
+
+///
+/// A value of native width with the highest bit set.
+///
+#define MAX_BIT  0x8000000000000000ULL
+///
+/// A value of native width with the two highest bits set.
+///
+#define MAX_2_BITS  0xC000000000000000ULL
+
+///
+/// Maximum legal RV64 address
+///
+#define MAX_ADDRESS  0xFFFFFFFFFFFFFFFFULL
+
+///
+/// Maximum usable address at boot time (48 bits using 4 KB pages in Supervisor mode)
+///
+#define MAX_ALLOC_ADDRESS  0xFFFFFFFFFFFFULL
+
+///
+/// Maximum legal RISC-V INTN and UINTN values.
+///
+#define MAX_INTN   ((INTN)0x7FFFFFFFFFFFFFFFULL)
+#define MAX_UINTN  ((UINTN)0xFFFFFFFFFFFFFFFFULL)
+
+///
+/// The stack alignment required for RISC-V
+///
+#define CPU_STACK_ALIGNMENT  16
+
+///
+/// Page allocation granularity for RISC-V
+///
+#define DEFAULT_PAGE_ALLOCATION_GRANULARITY  (0x1000)
+#define RUNTIME_PAGE_ALLOCATION_GRANULARITY  (0x1000)
+
+//
+// Modifier to ensure that all protocol member functions and EFI intrinsics
+// use the correct C calling convention. All protocol member functions and
+// EFI intrinsics are required to modify their member functions with EFIAPI.
+//
+#ifdef EFIAPI
+///
+/// If EFIAPI is already defined, then we use that definition.
+///
+#elif defined (__GNUC__)
+///
+/// Define the standard calling convention regardless of optimization level
+/// The GCC support assumes a GCC compiler that supports the EFI ABI. The EFI
+/// ABI is much closer to the x64 Microsoft* ABI than standard x64 (x86-64)
+/// GCC ABI. Thus a standard x64 (x86-64) GCC compiler can not be used for
+/// x64. Warning the assembly code in the MDE x64 does not follow the correct
+/// ABI for the standard x64 (x86-64) GCC.
+///
+#define EFIAPI
+#else
+///
+/// The default for a non Microsoft* or GCC compiler is to assume the EFI ABI
+/// is the standard.
+///
+#define EFIAPI
+#endif
+
+#if defined (__GNUC__)
+///
+/// For GNU assembly code, .global or .globl can declare global symbols.
+/// Define this macro to unify the usage.
+///
+#define ASM_GLOBAL  .globl
+#endif
+
+/**
+  Return the pointer to the first instruction of a function given a function pointer.
+  On x64 CPU architectures, these two pointer values are the same,
+  so the implementation of this macro is very simple.
+
+  @param  FunctionPointer   A pointer to a function.
+
+  @return The pointer to the first instruction of a function given a function pointer.
+
+**/
+#define FUNCTION_ENTRY_POINT(FunctionPointer)  (VOID *)(UINTN)(FunctionPointer)
+
+#ifndef __USER_LABEL_PREFIX__
+#define __USER_LABEL_PREFIX__
+#endif
+
+#endif
diff --git a/src/util/efirom.c b/src/util/efirom.c
index b0334bdb1..1220c187a 100644
--- a/src/util/efirom.c
+++ b/src/util/efirom.c
@@ -91,11 +91,13 @@ static void read_pe_info ( void *pe, uint16_t *machine,
 	switch ( *machine ) {
 	case EFI_IMAGE_MACHINE_IA32:
 	case EFI_IMAGE_MACHINE_ARMTHUMB_MIXED:
+	case EFI_IMAGE_MACHINE_RISCV32:
 		*subsystem = nt->nt32.OptionalHeader.Subsystem;
 		break;
 	case EFI_IMAGE_MACHINE_X64:
 	case EFI_IMAGE_MACHINE_AARCH64:
 	case EFI_IMAGE_MACHINE_LOONGARCH64:
+	case EFI_IMAGE_MACHINE_RISCV64:
 		*subsystem = nt->nt64.OptionalHeader.Subsystem;
 		break;
 	default:
diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 4af587d87..5434e7040 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -83,6 +83,9 @@
 #ifndef EM_AARCH64
 #define EM_AARCH64 183
 #endif
+#ifndef EM_RISCV
+#define EM_RISCV 243
+#endif
 #ifndef EM_LOONGARCH
 #define EM_LOONGARCH 258
 #endif
@@ -167,6 +170,45 @@
 #ifndef R_LARCH_PCREL20_S2
 #define R_LARCH_PCREL20_S2 103
 #endif
+#ifndef R_RISCV_NONE
+#define R_RISCV_NONE 0
+#endif
+#ifndef R_RISCV_32
+#define R_RISCV_32 1
+#endif
+#ifndef R_RISCV_64
+#define R_RISCV_64 2
+#endif
+#ifndef R_RISCV_BRANCH
+#define R_RISCV_BRANCH 16
+#endif
+#ifndef R_RISCV_JAL
+#define R_RISCV_JAL 17
+#endif
+#ifndef R_RISCV_PCREL_HI20
+#define R_RISCV_PCREL_HI20 23
+#endif
+#ifndef R_RISCV_PCREL_LO12_I
+#define R_RISCV_PCREL_LO12_I 24
+#endif
+#ifndef R_RISCV_PCREL_LO12_S
+#define R_RISCV_PCREL_LO12_S 25
+#endif
+#ifndef R_RISCV_ADD32
+#define R_RISCV_ADD32 35
+#endif
+#ifndef R_RISCV_SUB32
+#define R_RISCV_SUB32 39
+#endif
+#ifndef R_RISCV_RVC_BRANCH
+#define R_RISCV_RVC_BRANCH 44
+#endif
+#ifndef R_RISCV_RVC_JUMP
+#define R_RISCV_RVC_JUMP 45
+#endif
+#ifndef R_RISCV_RELAX
+#define R_RISCV_RELAX 51
+#endif
 #ifndef R_X86_64_GOTPCRELX
 #define R_X86_64_GOTPCRELX 41
 #endif
@@ -596,6 +638,11 @@ static void set_machine ( struct elf_file *elf, struct pe_header *pe_header ) {
 	case EM_LOONGARCH:
 		machine = EFI_IMAGE_MACHINE_LOONGARCH64;
 		break;
+	case EM_RISCV:
+		machine = ( ( ELFCLASS == ELFCLASS64 ) ?
+			    EFI_IMAGE_MACHINE_RISCV64 :
+			    EFI_IMAGE_MACHINE_RISCV32 );
+		break;
 	default:
 		eprintf ( "Unknown ELF architecture %d\n", ehdr->e_machine );
 		exit ( 1 );
@@ -828,16 +875,19 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_NONE ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_NULL ) :
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_NONE ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_NONE ) :
 			/* Ignore dummy relocations used by REQUIRE_SYMBOL() */
 			break;
 		case ELF_MREL ( EM_386, R_386_32 ) :
 		case ELF_MREL ( EM_ARM, R_ARM_ABS32 ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_32 ) :
 			/* Generate a 4-byte PE relocation */
 			generate_pe_reloc ( pe_reltab, offset, 4 );
 			break;
 		case ELF_MREL ( EM_X86_64, R_X86_64_64 ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_ABS64 ) :
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_64 ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_64 ) :
 			/* Generate an 8-byte PE relocation */
 			generate_pe_reloc ( pe_reltab, offset, 8 );
 			break;
@@ -869,16 +919,31 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_GOT_PC_HI20 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_GOT_PC_LO12 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCREL20_S2 ):
+		case ELF_MREL ( EM_RISCV, R_RISCV_BRANCH ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_JAL ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_PCREL_HI20 ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_PCREL_LO12_I ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_PCREL_LO12_S ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_RVC_BRANCH ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_RVC_JUMP ) :
 			/* Skip PC-relative relocations; all relative
 			 * offsets remain unaltered when the object is
 			 * loaded.
 			 */
 			break;
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_RELAX ):
+		case ELF_MREL ( EM_RISCV, R_RISCV_RELAX ) :
 			/* Relocation can be relaxed (optimized out).
 			 * Ignore it for now.
 			 */
 			break;
+		case ELF_MREL ( EM_RISCV, R_RISCV_ADD32 ) :
+		case ELF_MREL ( EM_RISCV, R_RISCV_SUB32 ) :
+			/* Ignore label difference relocations since
+			 * we do not perform any relocations that can
+			 * result in altered label differences.
+			 */
+			break;
 		case ELF_MREL ( EM_X86_64, R_X86_64_32 ) :
 			/* Ignore 32-bit relocations in a hybrid
 			 * 32-bit BIOS and 64-bit UEFI binary,
diff --git a/src/util/genfsimg b/src/util/genfsimg
index a981a62d8..23d191692 100755
--- a/src/util/genfsimg
+++ b/src/util/genfsimg
@@ -78,6 +78,12 @@ efi_boot_name() {
 	"64aa" )
 	    echo "BOOTAA64.EFI"
 	    ;;
+	"6450" )
+	    echo "BOOTRISCV64.EFI"
+	    ;;
+	"3250" )
+	    echo "BOOTRISCV32.EFI"
+	    ;;
 	* )
 	    echo "${FILENAME}: unrecognised EFI architecture ${ARCH}" >&2
 	    exit 1

From 9bb20686369abf5166161804c16554d491f53a62 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 16 Sep 2024 11:01:20 +0100
Subject: [PATCH 201/237] [efi] Remove redundant EFI_BOOT_FILE definitions

As of commit 79c0173 ("[build] Create util/genfsimg for building
filesystem-based images"), the EFI boot file name for each CPU
architecture is defined within the genfsimg script itself, rather than
being passed in as a Makefile parameter.

Remove the now-redundant Makefile definitions for EFI_BOOT_FILE.

Reported-by: Christian I. Nilsson <nikize@gmail.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/Makefile.efi   | 4 ----
 src/arch/arm64/Makefile.efi   | 4 ----
 src/arch/i386/Makefile.efi    | 4 ----
 src/arch/loong64/Makefile.efi | 4 ----
 src/arch/riscv32/Makefile.efi | 4 ----
 src/arch/riscv64/Makefile.efi | 4 ----
 src/arch/x86_64/Makefile.efi  | 4 ----
 7 files changed, 28 deletions(-)

diff --git a/src/arch/arm32/Makefile.efi b/src/arch/arm32/Makefile.efi
index d720f34f0..9bd34383d 100644
--- a/src/arch/arm32/Makefile.efi
+++ b/src/arch/arm32/Makefile.efi
@@ -8,10 +8,6 @@ CFLAGS		+= -mfloat-abi=soft
 #
 ELF2EFI		= $(ELF2EFI32)
 
-# Specify EFI boot file
-#
-EFI_BOOT_FILE	= bootarm.efi
-
 # Include generic EFI Makefile
 #
 MAKEDEPS	+= arch/arm/Makefile.efi
diff --git a/src/arch/arm64/Makefile.efi b/src/arch/arm64/Makefile.efi
index 998a64d03..96f2953f4 100644
--- a/src/arch/arm64/Makefile.efi
+++ b/src/arch/arm64/Makefile.efi
@@ -4,10 +4,6 @@
 #
 ELF2EFI		= $(ELF2EFI64)
 
-# Specify EFI boot file
-#
-EFI_BOOT_FILE	= bootaa64.efi
-
 # Include generic EFI Makefile
 #
 MAKEDEPS	+= arch/arm/Makefile.efi
diff --git a/src/arch/i386/Makefile.efi b/src/arch/i386/Makefile.efi
index 37ede65ac..aa809eb5d 100644
--- a/src/arch/i386/Makefile.efi
+++ b/src/arch/i386/Makefile.efi
@@ -8,10 +8,6 @@ ELF2EFI		= $(ELF2EFI32)
 #
 CFLAGS		+= -malign-double
 
-# Specify EFI boot file
-#
-EFI_BOOT_FILE	= bootia32.efi
-
 # Include generic EFI Makefile
 #
 MAKEDEPS	+= arch/x86/Makefile.efi
diff --git a/src/arch/loong64/Makefile.efi b/src/arch/loong64/Makefile.efi
index 1c51bcd67..611c910f2 100644
--- a/src/arch/loong64/Makefile.efi
+++ b/src/arch/loong64/Makefile.efi
@@ -4,10 +4,6 @@
 #
 ELF2EFI		= $(ELF2EFI64)
 
-# Specify EFI boot file
-#
-EFI_BOOT_FILE	= bootloongarch64.efi
-
 # Include generic EFI Makefile
 #
 MAKEDEPS	+= Makefile.efi
diff --git a/src/arch/riscv32/Makefile.efi b/src/arch/riscv32/Makefile.efi
index db83c8c14..2cc1a930f 100644
--- a/src/arch/riscv32/Makefile.efi
+++ b/src/arch/riscv32/Makefile.efi
@@ -4,10 +4,6 @@
 #
 ELF2EFI		= $(ELF2EFI32)
 
-# Specify EFI boot file
-#
-EFI_BOOT_FILE	= bootriscv32.efi
-
 # Include generic EFI Makefile
 #
 MAKEDEPS	+= arch/riscv/Makefile.efi
diff --git a/src/arch/riscv64/Makefile.efi b/src/arch/riscv64/Makefile.efi
index b19e036a3..3948ca15c 100644
--- a/src/arch/riscv64/Makefile.efi
+++ b/src/arch/riscv64/Makefile.efi
@@ -4,10 +4,6 @@
 #
 ELF2EFI		= $(ELF2EFI64)
 
-# Specify EFI boot file
-#
-EFI_BOOT_FILE	= bootriscv64.efi
-
 # Include generic EFI Makefile
 #
 MAKEDEPS	+= arch/riscv/Makefile.efi
diff --git a/src/arch/x86_64/Makefile.efi b/src/arch/x86_64/Makefile.efi
index 0041bb8f0..f38375607 100644
--- a/src/arch/x86_64/Makefile.efi
+++ b/src/arch/x86_64/Makefile.efi
@@ -12,10 +12,6 @@ CFLAGS		+= -mno-red-zone
 #
 ELF2EFI		= $(ELF2EFI64)
 
-# Specify EFI boot file
-#
-EFI_BOOT_FILE	= bootx64.efi
-
 # Include generic EFI Makefile
 #
 MAKEDEPS	+= arch/x86/Makefile.efi

From 59d123658bfe25402c4e89bbaf6eea83140d3c1f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 17 Sep 2024 13:11:43 +0100
Subject: [PATCH 202/237] [gve] Allocate all possible event counters

The admin queue API requires us to tell the device how many event
counters we have provided via the "configure device resources" admin
queue command.  There is, of course, absolutely no documentation
indicating how many event counters actually need to be provided.

We require only two event counters: one for the transmit queue, one
for the receive queue.  (The receive queue doesn't seem to actually
make any use of its event counter, but the "create receive queue"
admin queue command will fail if it doesn't have an available event
counter to choose.)

In the absence of any documentation, we currently make the assumption
that allocating and configuring 16 counters (i.e. one whole cacheline)
will be sufficient to allow for the use of two counters.

This assumption turns out to be incorrect.  On larger instance types
(observed with a c3d-standard-16 instance in europe-west4-a), we find
that creating the transmit or receive queues will each fail with a
probability of around 50% with the "failed precondition" error code.

Experimentation suggests that even though the device has accepted our
"configure device resources" command indicating that we are providing
only 16 event counters, it will attempt to choose any of its potential
32 event counters (and will then fail since the event counter that it
unilaterally chose is outside of the agreed range).

Work around this firmware bug by always allocating the maximum number
of event counters supported by the device.  (This requires deferring
the allocation of the event counters until after issuing the "describe
device" command.)

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/drivers/net/gve.c | 115 ++++++++++++++++++++++++++++--------------
 src/drivers/net/gve.h |  25 ---------
 2 files changed, 76 insertions(+), 64 deletions(-)

diff --git a/src/drivers/net/gve.c b/src/drivers/net/gve.c
index 0193e769f..df10a94c6 100644
--- a/src/drivers/net/gve.c
+++ b/src/drivers/net/gve.c
@@ -191,12 +191,8 @@ static int gve_reset ( struct gve_nic *gve ) {
 static int gve_admin_alloc ( struct gve_nic *gve ) {
 	struct dma_device *dma = gve->dma;
 	struct gve_admin *admin = &gve->admin;
-	struct gve_irqs *irqs = &gve->irqs;
-	struct gve_events *events = &gve->events;
 	struct gve_scratch *scratch = &gve->scratch;
 	size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
-	size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
-	size_t events_len = ( GVE_EVENT_MAX * sizeof ( events->event[0] ) );
 	size_t scratch_len = sizeof ( *scratch->buf );
 	int rc;
 
@@ -207,20 +203,6 @@ static int gve_admin_alloc ( struct gve_nic *gve ) {
 		goto err_admin;
 	}
 
-	/* Allocate interrupt channels */
-	irqs->irq = dma_alloc ( dma, &irqs->map, irqs_len, GVE_ALIGN );
-	if ( ! irqs->irq ) {
-		rc = -ENOMEM;
-		goto err_irqs;
-	}
-
-	/* Allocate event counters */
-	events->event = dma_alloc ( dma, &events->map, events_len, GVE_ALIGN );
-	if ( ! events->event ) {
-		rc = -ENOMEM;
-		goto err_events;
-	}
-
 	/* Allocate scratch buffer */
 	scratch->buf = dma_alloc ( dma, &scratch->map, scratch_len, GVE_ALIGN );
 	if ( ! scratch->buf ) {
@@ -228,17 +210,15 @@ static int gve_admin_alloc ( struct gve_nic *gve ) {
 		goto err_scratch;
 	}
 
-	DBGC ( gve, "GVE %p AQ at [%08lx,%08lx)\n",
+	DBGC ( gve, "GVE %p AQ at [%08lx,%08lx) scratch [%08lx,%08lx)\n",
 	       gve, virt_to_phys ( admin->cmd ),
-	       ( virt_to_phys ( admin->cmd ) + admin_len ) );
+	       ( virt_to_phys ( admin->cmd ) + admin_len ),
+	       virt_to_phys ( scratch->buf ),
+	       ( virt_to_phys ( scratch->buf ) + scratch_len ) );
 	return 0;
 
 	dma_free ( &scratch->map, scratch->buf, scratch_len );
  err_scratch:
-	dma_free ( &events->map, events->event, events_len );
- err_events:
-	dma_free ( &irqs->map, irqs->irq, irqs_len );
- err_irqs:
 	dma_free ( &admin->map, admin->cmd, admin_len );
  err_admin:
 	return rc;
@@ -251,23 +231,13 @@ static int gve_admin_alloc ( struct gve_nic *gve ) {
  */
 static void gve_admin_free ( struct gve_nic *gve ) {
 	struct gve_admin *admin = &gve->admin;
-	struct gve_irqs *irqs = &gve->irqs;
-	struct gve_events *events = &gve->events;
 	struct gve_scratch *scratch = &gve->scratch;
 	size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
-	size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
-	size_t events_len = ( GVE_EVENT_MAX * sizeof ( events->event[0] ) );
 	size_t scratch_len = sizeof ( *scratch->buf );
 
 	/* Free scratch buffer */
 	dma_free ( &scratch->map, scratch->buf, scratch_len );
 
-	/* Free event counter */
-	dma_free ( &events->map, events->event, events_len );
-
-	/* Free interrupt channels */
-	dma_free ( &irqs->map, irqs->irq, irqs_len );
-
 	/* Free admin queue */
 	dma_free ( &admin->map, admin->cmd, admin_len );
 }
@@ -467,13 +437,10 @@ static int gve_describe ( struct gve_nic *gve ) {
 
 	/* Extract queue parameters */
 	gve->events.count = be16_to_cpu ( desc->counters );
-	if ( gve->events.count > GVE_EVENT_MAX )
-		gve->events.count = GVE_EVENT_MAX;
 	gve->tx.count = be16_to_cpu ( desc->tx_count );
 	gve->rx.count = be16_to_cpu ( desc->rx_count );
-	DBGC ( gve, "GVE %p using %d TX, %d RX, %d/%d events\n",
-	       gve, gve->tx.count, gve->rx.count, gve->events.count,
-	       be16_to_cpu ( desc->counters ) );
+	DBGC ( gve, "GVE %p using %d TX, %d RX, %d events\n",
+	       gve, gve->tx.count, gve->rx.count, gve->events.count );
 
 	/* Extract network parameters */
 	build_assert ( sizeof ( desc->mac ) == ETH_ALEN );
@@ -704,6 +671,67 @@ static int gve_destroy_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
  ******************************************************************************
  */
 
+/**
+ * Allocate shared queue resources
+ *
+ * @v gve		GVE device
+ * @ret rc		Return status code
+ */
+static int gve_alloc_shared ( struct gve_nic *gve ) {
+	struct dma_device *dma = gve->dma;
+	struct gve_irqs *irqs = &gve->irqs;
+	struct gve_events *events = &gve->events;
+	size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
+	size_t events_len = ( gve->events.count * sizeof ( events->event[0] ) );
+	int rc;
+
+	/* Allocate interrupt channels */
+	irqs->irq = dma_alloc ( dma, &irqs->map, irqs_len, GVE_ALIGN );
+	if ( ! irqs->irq ) {
+		rc = -ENOMEM;
+		goto err_irqs;
+	}
+	DBGC ( gve, "GVE %p IRQs at [%08lx,%08lx)\n",
+	       gve, virt_to_phys ( irqs->irq ),
+	       ( virt_to_phys ( irqs->irq ) + irqs_len ) );
+
+	/* Allocate event counters */
+	events->event = dma_alloc ( dma, &events->map, events_len, GVE_ALIGN );
+	if ( ! events->event ) {
+		rc = -ENOMEM;
+		goto err_events;
+	}
+	DBGC ( gve, "GVE %p events at [%08lx,%08lx)\n",
+	       gve, virt_to_phys ( events->event ),
+	       ( virt_to_phys ( events->event ) + events_len ) );
+
+	return 0;
+
+	dma_free ( &events->map, events->event, events_len );
+ err_events:
+	dma_free ( &irqs->map, irqs->irq, irqs_len );
+ err_irqs:
+	return rc;
+}
+
+/**
+ * Free shared queue resources
+ *
+ * @v gve		GVE device
+ */
+static void gve_free_shared ( struct gve_nic *gve ) {
+	struct gve_irqs *irqs = &gve->irqs;
+	struct gve_events *events = &gve->events;
+	size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
+	size_t events_len = ( gve->events.count * sizeof ( events->event[0] ) );
+
+	/* Free event counters */
+	dma_free ( &events->map, events->event, events_len );
+
+	/* Free interrupt channels */
+	dma_free ( &irqs->map, irqs->irq, irqs_len );
+}
+
 /**
  * Allocate queue page list
  *
@@ -1117,6 +1145,10 @@ static int gve_open ( struct net_device *netdev ) {
 	struct gve_queue *rx = &gve->rx;
 	int rc;
 
+	/* Allocate shared queue resources */
+	if ( ( rc = gve_alloc_shared ( gve ) ) != 0 )
+		goto err_alloc_shared;
+
 	/* Allocate and prepopulate transmit queue */
 	if ( ( rc = gve_alloc_queue ( gve, tx ) ) != 0 )
 		goto err_alloc_tx;
@@ -1137,6 +1169,8 @@ static int gve_open ( struct net_device *netdev ) {
  err_alloc_rx:
 	gve_free_queue ( gve, tx );
  err_alloc_tx:
+	gve_free_shared ( gve );
+ err_alloc_shared:
 	return rc;
 }
 
@@ -1163,6 +1197,9 @@ static void gve_close ( struct net_device *netdev ) {
 	/* Free queues */
 	gve_free_queue ( gve, rx );
 	gve_free_queue ( gve, tx );
+
+	/* Free shared queue resources */
+	gve_free_shared ( gve );
 }
 
 /**
diff --git a/src/drivers/net/gve.h b/src/drivers/net/gve.h
index 247d6e699..43517cc55 100644
--- a/src/drivers/net/gve.h
+++ b/src/drivers/net/gve.h
@@ -50,15 +50,6 @@ struct google_mac {
  */
 #define GVE_ALIGN GVE_PAGE_SIZE
 
-/**
- * Length alignment
- *
- * All DMA data structure lengths seem to need to be aligned to a
- * multiple of 64 bytes.  (This is not documented anywhere, but is
- * inferred from existing source code and experimentation.)
- */
-#define GVE_LEN_ALIGN 64
-
 /** Configuration BAR */
 #define GVE_CFG_BAR PCI_BASE_ADDRESS_0
 
@@ -350,22 +341,6 @@ struct gve_event {
 	volatile uint32_t count;
 } __attribute__ (( packed ));
 
-/**
- * Maximum number of event counters
- *
- * We tell the device how many event counters we have provided via the
- * "configure device resources" admin queue command.  The device will
- * accept being given only a single counter, but will subsequently
- * fail to create a receive queue.
- *
- * There is, of course, no documentation indicating how may event
- * counters actually need to be provided.  In the absence of evidence
- * to the contrary, assume that 16 counters (i.e. the smallest number
- * we can allocate, given the length alignment constraint on
- * allocations) will be sufficient.
- */
-#define GVE_EVENT_MAX ( GVE_LEN_ALIGN / sizeof ( struct gve_event ) )
-
 /** Event counter array */
 struct gve_events {
 	/** Event counters */

From 3def13265d9475c861eed1a101584b761e97ae33 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 19 Sep 2024 16:23:32 +0100
Subject: [PATCH 203/237] [crypto] Use constant-time big integer multiplication

Big integer multiplication currently performs immediate carry
propagation from each step of the long multiplication, relying on the
fact that the overall result has a known maximum value to minimise the
number of carries performed without ever needing to explicitly check
against the result buffer size.

This is not a constant-time algorithm, since the number of carries
performed will be a function of the input values.  We could make it
constant-time by always continuing to propagate the carry until
reaching the end of the result buffer, but this would introduce a
large number of redundant zero carries.

Require callers of bigint_multiply() to provide a temporary carry
storage buffer, of the same size as the result buffer.  This allows
the carry-out from the accumulation of each double-element product to
be accumulated in the temporary carry space, and then added in via a
single call to bigint_add() after the multiplication is complete.

Since the structure of big integer multiplication is identical across
all current CPU architectures, provide a single shared implementation
of bigint_multiply().  The architecture-specific operation then
becomes the multiplication of two big integer elements and the
accumulation of the double-element product.

Note that any intermediate carry arising from accumulating the lower
half of the double-element product may be added to the upper half of
the double-element product without risk of overflow, since the result
of multiplying two n-bit integers can never have all n bits set in its
upper half.  This simplifies the carry calculations for architectures
such as RISC-V and LoongArch64 that do not have a carry flag.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/core/arm32_bigint.c     | 106 ---------------------
 src/arch/arm32/include/bits/bigint.h   |  34 ++++++-
 src/arch/arm64/core/arm64_bigint.c     | 107 ---------------------
 src/arch/arm64/include/bits/bigint.h   |  35 ++++++-
 src/arch/loong64/core/loong64_bigint.c | 124 -------------------------
 src/arch/loong64/include/bits/bigint.h |  42 ++++++++-
 src/arch/riscv/core/riscv_bigint.c     | 112 ----------------------
 src/arch/riscv/include/bits/bigint.h   |  43 ++++++++-
 src/arch/x86/core/x86_bigint.c         | 100 --------------------
 src/arch/x86/include/bits/bigint.h     |  34 ++++++-
 src/crypto/bigint.c                    | 117 ++++++++++++++++++++++-
 src/crypto/x25519.c                    |  83 +++++++++++------
 src/include/ipxe/bigint.h              |  18 +++-
 src/tests/bigint_test.c                |  12 ++-
 14 files changed, 355 insertions(+), 612 deletions(-)
 delete mode 100644 src/arch/arm32/core/arm32_bigint.c
 delete mode 100644 src/arch/arm64/core/arm64_bigint.c
 delete mode 100644 src/arch/loong64/core/loong64_bigint.c
 delete mode 100644 src/arch/riscv/core/riscv_bigint.c
 delete mode 100644 src/arch/x86/core/x86_bigint.c

diff --git a/src/arch/arm32/core/arm32_bigint.c b/src/arch/arm32/core/arm32_bigint.c
deleted file mode 100644
index 29fb40a7c..000000000
--- a/src/arch/arm32/core/arm32_bigint.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <stdint.h>
-#include <string.h>
-#include <ipxe/bigint.h>
-
-/** @file
- *
- * Big integer support
- */
-
-/**
- * Multiply big integers
- *
- * @v multiplicand0	Element 0 of big integer to be multiplied
- * @v multiplicand_size	Number of elements in multiplicand
- * @v multiplier0	Element 0 of big integer to be multiplied
- * @v multiplier_size	Number of elements in multiplier
- * @v result0		Element 0 of big integer to hold result
- */
-void bigint_multiply_raw ( const uint32_t *multiplicand0,
-			   unsigned int multiplicand_size,
-			   const uint32_t *multiplier0,
-			   unsigned int multiplier_size,
-			   uint32_t *result0 ) {
-	unsigned int result_size = ( multiplicand_size + multiplier_size );
-	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
-		*multiplicand = ( ( const void * ) multiplicand0 );
-	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
-		*multiplier = ( ( const void * ) multiplier0 );
-	bigint_t ( result_size ) __attribute__ (( may_alias ))
-		*result = ( ( void * ) result0 );
-	unsigned int i;
-	unsigned int j;
-	uint32_t multiplicand_element;
-	uint32_t multiplier_element;
-	uint32_t *result_elements;
-	uint32_t discard_low;
-	uint32_t discard_high;
-	uint32_t discard_temp;
-
-	/* Zero result */
-	memset ( result, 0, sizeof ( *result ) );
-
-	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < multiplicand_size ; i++ ) {
-		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < multiplier_size ; j++ ) {
-			multiplier_element = multiplier->element[j];
-			result_elements = &result->element[ i + j ];
-			/* Perform a single multiply, and add the
-			 * resulting double-element into the result,
-			 * carrying as necessary.  The carry can
-			 * never overflow beyond the end of the
-			 * result, since:
-			 *
-			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
-			 */
-			__asm__ __volatile__ ( "umull %1, %2, %5, %6\n\t"
-					       "ldr %3, [%0]\n\t"
-					       "adds %3, %1\n\t"
-					       "stmia %0!, {%3}\n\t"
-					       "ldr %3, [%0]\n\t"
-					       "adcs %3, %2\n\t"
-					       "stmia %0!, {%3}\n\t"
-					       "bcc 2f\n\t"
-					       "\n1:\n\t"
-					       "ldr %3, [%0]\n\t"
-					       "adcs %3, #0\n\t"
-					       "stmia %0!, {%3}\n\t"
-					       "bcs 1b\n\t"
-					       "\n2:\n\t"
-					       : "+l" ( result_elements ),
-						 "=l" ( discard_low ),
-						 "=l" ( discard_high ),
-						 "=l" ( discard_temp ),
-						 "+m" ( *result )
-					       : "l" ( multiplicand_element ),
-						 "l" ( multiplier_element )
-					       : "cc" );
-		}
-	}
-}
diff --git a/src/arch/arm32/include/bits/bigint.h b/src/arch/arm32/include/bits/bigint.h
index e4b511da7..0a368a0c0 100644
--- a/src/arch/arm32/include/bits/bigint.h
+++ b/src/arch/arm32/include/bits/bigint.h
@@ -309,10 +309,34 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused,
 		*(--out_byte) = *(value_byte++);
 }
 
-extern void bigint_multiply_raw ( const uint32_t *multiplicand0,
-				  unsigned int multiplicand_size,
-				  const uint32_t *multiplier0,
-				  unsigned int multiplier_size,
-				  uint32_t *value0 );
+/**
+ * Multiply big integer elements
+ *
+ * @v multiplicand	Multiplicand element
+ * @v multiplier	Multiplier element
+ * @v result		Result element pair
+ * @v carry		Carry element
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_multiply_one ( const uint32_t multiplicand, const uint32_t multiplier,
+		      uint32_t *result, uint32_t *carry ) {
+	uint32_t discard_low;
+	uint32_t discard_high;
+
+	__asm__ __volatile__ ( /* Perform multiplication */
+			       "umull %0, %1, %5, %6\n\t"
+			       /* Accumulate result */
+			       "adds %2, %0\n\t"
+			       "adcs %3, %1\n\t"
+			       /* Accumulate carry (cannot overflow) */
+			       "adc %4, #0\n\t"
+			       : "=r" ( discard_low ),
+				 "=r" ( discard_high ),
+				 "+r" ( result[0] ),
+				 "+r" ( result[1] ),
+				 "+r" ( *carry )
+			       : "r" ( multiplicand ),
+				 "r" ( multiplier ) );
+}
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/arm64/core/arm64_bigint.c b/src/arch/arm64/core/arm64_bigint.c
deleted file mode 100644
index 7740f1aef..000000000
--- a/src/arch/arm64/core/arm64_bigint.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <stdint.h>
-#include <string.h>
-#include <ipxe/bigint.h>
-
-/** @file
- *
- * Big integer support
- */
-
-/**
- * Multiply big integers
- *
- * @v multiplicand0	Element 0 of big integer to be multiplied
- * @v multiplicand_size	Number of elements in multiplicand
- * @v multiplier0	Element 0 of big integer to be multiplied
- * @v multiplier_size	Number of elements in multiplier
- * @v result0		Element 0 of big integer to hold result
- */
-void bigint_multiply_raw ( const uint64_t *multiplicand0,
-			   unsigned int multiplicand_size,
-			   const uint64_t *multiplier0,
-			   unsigned int multiplier_size,
-			   uint64_t *result0 ) {
-	unsigned int result_size = ( multiplicand_size + multiplier_size );
-	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
-		*multiplicand = ( ( const void * ) multiplicand0 );
-	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
-		*multiplier = ( ( const void * ) multiplier0 );
-	bigint_t ( result_size ) __attribute__ (( may_alias ))
-		*result = ( ( void * ) result0 );
-	unsigned int i;
-	unsigned int j;
-	uint64_t multiplicand_element;
-	uint64_t multiplier_element;
-	uint64_t *result_elements;
-	uint64_t discard_low;
-	uint64_t discard_high;
-	uint64_t discard_temp_low;
-	uint64_t discard_temp_high;
-
-	/* Zero result */
-	memset ( result, 0, sizeof ( *result ) );
-
-	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < multiplicand_size ; i++ ) {
-		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < multiplier_size ; j++ ) {
-			multiplier_element = multiplier->element[j];
-			result_elements = &result->element[ i + j ];
-			/* Perform a single multiply, and add the
-			 * resulting double-element into the result,
-			 * carrying as necessary.  The carry can
-			 * never overflow beyond the end of the
-			 * result, since:
-			 *
-			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
-			 */
-			__asm__ __volatile__ ( "mul %1, %6, %7\n\t"
-					       "umulh %2, %6, %7\n\t"
-					       "ldp %3, %4, [%0]\n\t"
-					       "adds %3, %3, %1\n\t"
-					       "adcs %4, %4, %2\n\t"
-					       "stp %3, %4, [%0], #16\n\t"
-					       "bcc 2f\n\t"
-					       "\n1:\n\t"
-					       "ldr %3, [%0]\n\t"
-					       "adcs %3, %3, xzr\n\t"
-					       "str %3, [%0], #8\n\t"
-					       "bcs 1b\n\t"
-					       "\n2:\n\t"
-					       : "+r" ( result_elements ),
-						 "=&r" ( discard_low ),
-						 "=&r" ( discard_high ),
-						 "=r" ( discard_temp_low ),
-						 "=r" ( discard_temp_high ),
-						 "+m" ( *result )
-					       : "r" ( multiplicand_element ),
-						 "r" ( multiplier_element )
-					       : "cc" );
-		}
-	}
-}
diff --git a/src/arch/arm64/include/bits/bigint.h b/src/arch/arm64/include/bits/bigint.h
index 0d08bbd65..ca9feafbe 100644
--- a/src/arch/arm64/include/bits/bigint.h
+++ b/src/arch/arm64/include/bits/bigint.h
@@ -310,10 +310,35 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
 		*(--out_byte) = *(value_byte++);
 }
 
-extern void bigint_multiply_raw ( const uint64_t *multiplicand0,
-				  unsigned int multiplicand_size,
-				  const uint64_t *multiplier0,
-				  unsigned int multiplier_size,
-				  uint64_t *value0 );
+/**
+ * Multiply big integer elements
+ *
+ * @v multiplicand	Multiplicand element
+ * @v multiplier	Multiplier element
+ * @v result		Result element pair
+ * @v carry		Carry element
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_multiply_one ( const uint64_t multiplicand, const uint64_t multiplier,
+		      uint64_t *result, uint64_t *carry ) {
+	uint64_t discard_low;
+	uint64_t discard_high;
+
+	__asm__ __volatile__ ( /* Perform multiplication */
+			       "mul %0, %5, %6\n\t"
+			       "umulh %1, %5, %6\n\t"
+			       /* Accumulate result */
+			       "adds %2, %2, %0\n\t"
+			       "adcs %3, %3, %1\n\t"
+			       /* Accumulate carry (cannot overflow) */
+			       "adc %4, %4, xzr\n\t"
+			       : "=&r" ( discard_low ),
+				 "=r" ( discard_high ),
+				 "+r" ( result[0] ),
+				 "+r" ( result[1] ),
+				 "+r" ( *carry )
+			       : "r" ( multiplicand ),
+				 "r" ( multiplier ) );
+}
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/loong64/core/loong64_bigint.c b/src/arch/loong64/core/loong64_bigint.c
deleted file mode 100644
index b428e22c3..000000000
--- a/src/arch/loong64/core/loong64_bigint.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>.
- * Copyright (c) 2023, Xiaotian Wu <wuxiaotian@loongson.cn>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <stdint.h>
-#include <string.h>
-#include <ipxe/bigint.h>
-
-/** @file
- *
- * Big integer support
- */
-
-/**
- * Multiply big integers
- *
- * @v multiplicand0	Element 0 of big integer to be multiplied
- * @v multiplicand_size	Number of elements in multiplicand
- * @v multiplier0	Element 0 of big integer to be multiplied
- * @v multiplier_size	Number of elements in multiplier
- * @v result0		Element 0 of big integer to hold result
- */
-void bigint_multiply_raw ( const uint64_t *multiplicand0,
-			   unsigned int multiplicand_size,
-			   const uint64_t *multiplier0,
-			   unsigned int multiplier_size,
-			   uint64_t *result0 ) {
-	unsigned int result_size = ( multiplicand_size + multiplier_size );
-	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
-		*multiplicand = ( ( const void * ) multiplicand0 );
-	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
-		*multiplier = ( ( const void * ) multiplier0 );
-	bigint_t ( result_size ) __attribute__ (( may_alias ))
-		*result = ( ( void * ) result0 );
-	unsigned int i;
-	unsigned int j;
-	uint64_t multiplicand_element;
-	uint64_t multiplier_element;
-	uint64_t *result_elements;
-	uint64_t discard_low;
-	uint64_t discard_high;
-	uint64_t discard_temp_low;
-	uint64_t discard_temp_high;
-
-	/* Zero result */
-	memset ( result, 0, sizeof ( *result ) );
-
-	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < multiplicand_size ; i++ ) {
-		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < multiplier_size ; j++ ) {
-			multiplier_element = multiplier->element[j];
-			result_elements = &result->element[ i + j ];
-			/* Perform a single multiply, and add the
-			 * resulting double-element into the result,
-			 * carrying as necessary.  The carry can
-			 * never overflow beyond the end of the
-			 * result, since:
-			 *
-			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
-			 */
-			__asm__ __volatile__ ( "mul.d   %1, %6, %7\n\t"
-					       "mulh.du %2, %6, %7\n\t"
-
-					       "ld.d    %3, %0, 0\n\t"
-					       "ld.d    %4, %0, 8\n\t"
-
-					       "add.d   %3, %3, %1\n\t"
-					       "sltu    $t0, %3, %1\n\t"
-
-					       "add.d   %4, %4, %2\n\t"
-					       "sltu    $t1, %4, %2\n\t"
-
-					       "add.d   %4, %4, $t0\n\t"
-					       "sltu    $t0, %4, $t0\n\t"
-					       "or      $t0, $t0, $t1\n\t"
-
-					       "st.d    %3,  %0, 0\n\t"
-					       "st.d    %4,  %0, 8\n\t"
-
-					       "addi.d  %0,  %0, 16\n\t"
-					       "beqz    $t0, 2f\n"
-					       "1:\n\t"
-					       "ld.d    %3,  %0, 0\n\t"
-					       "add.d   %3,  %3, $t0\n\t"
-					       "sltu    $t0, %3, $t0\n\t"
-					       "st.d    %3,  %0, 0\n\t"
-					       "addi.d  %0, %0, 8\n\t"
-					       "bnez    $t0, 1b\n"
-					       "2:"
-					       : "+r" ( result_elements ),
-						 "=&r" ( discard_low ),
-						 "=&r" ( discard_high ),
-						 "=r" ( discard_temp_low ),
-						 "=r" ( discard_temp_high ),
-						 "+m" ( *result )
-					       : "r" ( multiplicand_element ),
-						 "r" ( multiplier_element )
-					       : "t0", "t1" );
-		}
-	}
-}
diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h
index bec748beb..ec6ca4b89 100644
--- a/src/arch/loong64/include/bits/bigint.h
+++ b/src/arch/loong64/include/bits/bigint.h
@@ -357,10 +357,42 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
 		*(--out_byte) = *(value_byte++);
 }
 
-extern void bigint_multiply_raw ( const uint64_t *multiplicand0,
-				  unsigned int multiplicand_size,
-				  const uint64_t *multiplier0,
-				  unsigned int multiplier_size,
-				  uint64_t *value0 );
+/**
+ * Multiply big integer elements
+ *
+ * @v multiplicand	Multiplicand element
+ * @v multiplier	Multiplier element
+ * @v result		Result element pair
+ * @v carry		Carry element
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_multiply_one ( const uint64_t multiplicand, const uint64_t multiplier,
+		      uint64_t *result, uint64_t *carry ) {
+	uint64_t discard_low;
+	uint64_t discard_high;
+	uint64_t discard_carry;
+
+	__asm__ __volatile__ ( /* Perform multiplication */
+			       "mul.d %0, %6, %7\n\t"
+			       "mulh.du %1, %6, %7\n\t"
+			       /* Accumulate low half */
+			       "add.d %3, %3, %0\n\t"
+			       "sltu %2, %3, %0\n\t"
+			       /* Add carry to high half (cannot overflow) */
+			       "add.d %1, %1, %2\n\t"
+			       /* Accumulate high half */
+			       "add.d %4, %4, %1\n\t"
+			       "sltu %2, %4, %1\n\t"
+			       /* Accumulate carry (cannot overflow) */
+			       "add.d %5, %5, %2\n\t"
+			       : "=&r" ( discard_low ),
+				 "=r" ( discard_high ),
+				 "=r" ( discard_carry ),
+				 "+r" ( result[0] ),
+				 "+r" ( result[1] ),
+				 "+r" ( *carry )
+			       : "r" ( multiplicand ),
+				 "r" ( multiplier ) );
+}
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/riscv/core/riscv_bigint.c b/src/arch/riscv/core/riscv_bigint.c
deleted file mode 100644
index cbc5631a3..000000000
--- a/src/arch/riscv/core/riscv_bigint.c
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <stdint.h>
-#include <string.h>
-#include <ipxe/bigint.h>
-
-/** @file
- *
- * Big integer support
- */
-
-/**
- * Multiply big integers
- *
- * @v multiplicand0	Element 0 of big integer to be multiplied
- * @v multiplicand_size	Number of elements in multiplicand
- * @v multiplier0	Element 0 of big integer to be multiplied
- * @v multiplier_size	Number of elements in multiplier
- * @v result0		Element 0 of big integer to hold result
- */
-void bigint_multiply_raw ( const unsigned long *multiplicand0,
-			   unsigned int multiplicand_size,
-			   const unsigned long *multiplier0,
-			   unsigned int multiplier_size,
-			   unsigned long *result0 ) {
-	unsigned int result_size = ( multiplicand_size + multiplier_size );
-	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
-		*multiplicand = ( ( const void * ) multiplicand0 );
-	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
-		*multiplier = ( ( const void * ) multiplier0 );
-	bigint_t ( result_size ) __attribute__ (( may_alias ))
-		*result = ( ( void * ) result0 );
-	unsigned int i;
-	unsigned int j;
-	unsigned long multiplicand_element;
-	unsigned long multiplier_element;
-	unsigned long *result_elements;
-	unsigned long discard_low;
-	unsigned long discard_high;
-	unsigned long discard_temp;
-	unsigned long discard_carry;
-
-	/* Zero result */
-	memset ( result, 0, sizeof ( *result ) );
-
-	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < multiplicand_size ; i++ ) {
-		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < multiplier_size ; j++ ) {
-			multiplier_element = multiplier->element[j];
-			result_elements = &result->element[ i + j ];
-			/* Perform a single multiply, and add the
-			 * resulting double-element into the result,
-			 * carrying as necessary.  The carry can
-			 * never overflow beyond the end of the
-			 * result, since:
-			 *
-			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
-			 */
-			__asm__ __volatile__ ( /* Perform multiplication */
-					       "mulhu %2, %6, %7\n\t"
-					       "mul %1, %6, %7\n\t"
-					       /* Accumulate low half */
-					       LOADN " %3, (%0)\n\t"
-					       "add %3, %3, %1\n\t"
-					       "sltu %4, %3, %1\n\t"
-					       STOREN " %3, 0(%0)\n\t"
-					       /* Carry into high half */
-					       "add %4, %4, %2\n\t"
-					       /* Propagate as necessary */
-					       "\n1:\n\t"
-					       "addi %0, %0, %8\n\t"
-					       LOADN " %3, 0(%0)\n\t"
-					       "add %3, %3, %4\n\t"
-					       "sltu %4, %3, %4\n\t"
-					       STOREN " %3, 0(%0)\n\t"
-					       "bnez %4, 1b\n\t"
-					       : "+r" ( result_elements ),
-						 "=r" ( discard_low ),
-						 "=r" ( discard_high ),
-						 "=r" ( discard_temp ),
-						 "=r" ( discard_carry ),
-						 "+m" ( *result )
-					       : "r" ( multiplicand_element ),
-						 "r" ( multiplier_element ),
-						 "i" ( sizeof ( *result0 ) ) );
-		}
-	}
-}
diff --git a/src/arch/riscv/include/bits/bigint.h b/src/arch/riscv/include/bits/bigint.h
index fcc0d51c3..c58233469 100644
--- a/src/arch/riscv/include/bits/bigint.h
+++ b/src/arch/riscv/include/bits/bigint.h
@@ -353,10 +353,43 @@ bigint_done_raw ( const unsigned long *value0, unsigned int size __unused,
 		*(--out_byte) = *(value_byte++);
 }
 
-extern void bigint_multiply_raw ( const unsigned long *multiplicand0,
-				  unsigned int multiplicand_size,
-				  const unsigned long *multiplier0,
-				  unsigned int multiplier_size,
-				  unsigned long *value0 );
+/**
+ * Multiply big integer elements
+ *
+ * @v multiplicand	Multiplicand element
+ * @v multiplier	Multiplier element
+ * @v result		Result element pair
+ * @v carry		Carry element
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_multiply_one ( const unsigned long multiplicand,
+		      const unsigned long multiplier,
+		      unsigned long *result, unsigned long *carry ) {
+	unsigned long discard_low;
+	unsigned long discard_high;
+	unsigned long discard_carry;
+
+	__asm__ __volatile__ ( /* Perform multiplication */
+			       "mulhu %1, %6, %7\n\t"
+			       "mul %0, %6, %7\n\t"
+			       /* Accumulate low half */
+			       "add %3, %3, %0\n\t"
+			       "sltu %2, %3, %0\n\t"
+			       /* Add carry to high half (cannot overflow) */
+			       "add %1, %1, %2\n\t"
+			       /* Accumulate high half */
+			       "add %4, %4, %1\n\t"
+			       "sltu %2, %4, %1\n\t"
+			       /* Accumulate carry (cannot overflow) */
+			       "add %5, %5, %2\n\t"
+			       : "=r" ( discard_low ),
+				 "=&r" ( discard_high ),
+				 "=r" ( discard_carry ),
+				 "+r" ( result[0] ),
+				 "+r" ( result[1] ),
+				 "+r" ( *carry )
+			       : "r" ( multiplicand ),
+				 "r" ( multiplier ) );
+}
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/x86/core/x86_bigint.c b/src/arch/x86/core/x86_bigint.c
deleted file mode 100644
index 74e5da9a2..000000000
--- a/src/arch/x86/core/x86_bigint.c
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
- * You can also choose to distribute this program under the terms of
- * the Unmodified Binary Distribution Licence (as given in the file
- * COPYING.UBDL), provided that you have satisfied its requirements.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <stdint.h>
-#include <string.h>
-#include <ipxe/bigint.h>
-
-/** @file
- *
- * Big integer support
- */
-
-/**
- * Multiply big integers
- *
- * @v multiplicand0	Element 0 of big integer to be multiplied
- * @v multiplicand_size	Number of elements in multiplicand
- * @v multiplier0	Element 0 of big integer to be multiplied
- * @v multiplier_size	Number of elements in multiplier
- * @v result0		Element 0 of big integer to hold result
- */
-void bigint_multiply_raw ( const uint32_t *multiplicand0,
-			   unsigned int multiplicand_size,
-			   const uint32_t *multiplier0,
-			   unsigned int multiplier_size,
-			   uint32_t *result0 ) {
-	unsigned int result_size = ( multiplicand_size + multiplier_size );
-	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
-		*multiplicand = ( ( const void * ) multiplicand0 );
-	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
-		*multiplier = ( ( const void * ) multiplier0 );
-	bigint_t ( result_size ) __attribute__ (( may_alias ))
-		*result = ( ( void * ) result0 );
-	unsigned int i;
-	unsigned int j;
-	uint32_t multiplicand_element;
-	uint32_t multiplier_element;
-	uint32_t *result_elements;
-	uint32_t discard_a;
-	uint32_t discard_d;
-	long index;
-
-	/* Zero result */
-	memset ( result, 0, sizeof ( *result ) );
-
-	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < multiplicand_size ; i++ ) {
-		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < multiplier_size ; j++ ) {
-			multiplier_element = multiplier->element[j];
-			result_elements = &result->element[ i + j ];
-			/* Perform a single multiply, and add the
-			 * resulting double-element into the result,
-			 * carrying as necessary.  The carry can
-			 * never overflow beyond the end of the
-			 * result, since:
-			 *
-			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
-			 */
-			__asm__ __volatile__ ( "mull %5\n\t"
-					       "addl %%eax, (%6,%2,4)\n\t"
-					       "adcl %%edx, 4(%6,%2,4)\n\t"
-					       "\n1:\n\t"
-					       "adcl $0, 8(%6,%2,4)\n\t"
-					       "inc %2\n\t"
-						       /* Does not affect CF */
-					       "jc 1b\n\t"
-					       : "=&a" ( discard_a ),
-						 "=&d" ( discard_d ),
-						 "=&r" ( index ),
-						 "+m" ( *result )
-					       : "0" ( multiplicand_element ),
-						 "g" ( multiplier_element ),
-						 "r" ( result_elements ),
-						 "2" ( 0 ) );
-		}
-	}
-}
diff --git a/src/arch/x86/include/bits/bigint.h b/src/arch/x86/include/bits/bigint.h
index a6bc2ca1d..a481e90f7 100644
--- a/src/arch/x86/include/bits/bigint.h
+++ b/src/arch/x86/include/bits/bigint.h
@@ -322,10 +322,34 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused,
 			       : "eax" );
 }
 
-extern void bigint_multiply_raw ( const uint32_t *multiplicand0,
-				  unsigned int multiplicand_size,
-				  const uint32_t *multiplier0,
-				  unsigned int multiplier_size,
-				  uint32_t *value0 );
+/**
+ * Multiply big integer elements
+ *
+ * @v multiplicand	Multiplicand element
+ * @v multiplier	Multiplier element
+ * @v result		Result element pair
+ * @v carry		Carry element
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_multiply_one ( const uint32_t multiplicand, const uint32_t multiplier,
+		      uint32_t *result, uint32_t *carry ) {
+	uint32_t discard_a;
+	uint32_t discard_d;
+
+	__asm__ __volatile__ ( /* Perform multiplication */
+			       "mull %6\n\t"
+			       /* Accumulate result */
+			       "addl %0, %2\n\t"
+			       "adcl %1, %3\n\t"
+			       /* Accumulate carry (cannot overflow) */
+			       "adcl $0, %4\n\t"
+			       : "=a" ( discard_a ),
+				 "=d" ( discard_d ),
+				 "+m" ( result[0] ),
+				 "+m" ( result[1] ),
+				 "+m" ( *carry )
+			       : "0" ( multiplicand ),
+				 "g" ( multiplier ) );
+}
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index 656f979e5..5b7116e28 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -75,6 +75,115 @@ void bigint_swap_raw ( bigint_element_t *first0, bigint_element_t *second0,
 	}
 }
 
+/**
+ * Multiply big integers
+ *
+ * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
+ * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
+ * @v result0		Element 0 of big integer to hold result
+ * @v carry0		Element 0 of big integer to hold temporary carry
+ */
+void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
+			   unsigned int multiplicand_size,
+			   const bigint_element_t *multiplier0,
+			   unsigned int multiplier_size,
+			   bigint_element_t *result0,
+			   bigint_element_t *carry0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*carry = ( ( void * ) carry0 );
+	bigint_element_t multiplicand_element;
+	const bigint_element_t *multiplier_element;
+	bigint_element_t *result_elements;
+	bigint_element_t *carry_element;
+	unsigned int i;
+	unsigned int j;
+
+	/* Zero result and temporary carry space */
+	memset ( result, 0, sizeof ( *result ) );
+	memset ( carry, 0, sizeof ( *carry ) );
+
+	/* Multiply integers one element at a time, adding the double
+	 * element directly into the result and accumulating any
+	 * overall carry out from this double-element addition into
+	 * the temporary carry space.
+	 *
+	 * We could propagate the carry immediately instead of using a
+	 * temporary carry space.  However, this would cause the
+	 * multiplication to run in non-constant time, which is
+	 * undesirable.
+	 *
+	 * The carry elements can never overflow, provided that the
+	 * element size is large enough to accommodate any plausible
+	 * big integer.  The total number of potential carries (across
+	 * all elements) is the sum of the number of elements in the
+	 * multiplicand and multiplier.  With a 16-bit element size,
+	 * this therefore allows for up to a 1Mbit multiplication
+	 * result (e.g. a 512kbit integer multiplied by another
+	 * 512kbit integer), which is around 100x higher than could be
+	 * needed in practice.  With a more realistic 32-bit element
+	 * size, the limit becomes a totally implausible 128Gbit
+	 * multiplication result.
+	 */
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
+		multiplicand_element = multiplicand->element[i];
+		multiplier_element = &multiplier->element[0];
+		result_elements = &result->element[i];
+		carry_element = &carry->element[i];
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
+			bigint_multiply_one ( multiplicand_element,
+					      *(multiplier_element++),
+					      result_elements++,
+					      carry_element++ );
+		}
+	}
+
+	/* Add the temporary carry into the result.  The least
+	 * significant element of the carry represents the carry out
+	 * from multiplying the least significant elements of the
+	 * multiplicand and multiplier, and therefore must be added to
+	 * the third-least significant element of the result (i.e. the
+	 * carry needs to be shifted left by two elements before being
+	 * adding to the result).
+	 *
+	 * The most significant two elements of the carry are
+	 * guaranteed to be zero, since:
+	 *
+	 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
+	 *
+	 * and the overall result of the multiplication (including
+	 * adding in the shifted carries) is therefore guaranteed not
+	 * to overflow beyond the end of the result.
+	 *
+	 * We could avoid this shifting by writing the carry directly
+	 * into the "correct" element during the element-by-element
+	 * multiplication stage above.  However, this would add
+	 * complexity to the loop since we would have to arrange for
+	 * the (provably zero) most significant two carry out results
+	 * to be discarded, in order to avoid writing beyond the end
+	 * of the temporary carry space.
+	 *
+	 * Performing the logical shift is essentially free, since we
+	 * simply adjust the element pointers.
+	 *
+	 * To avoid requiring additional checks in each architecture's
+	 * implementation of bigint_add_raw(), we explicitly avoid
+	 * calling bigint_add_raw() with a size of zero.
+	 */
+	if ( result_size > 2 ) {
+		bigint_add_raw ( &carry->element[0], &result->element[2],
+				 ( result_size - 2 ) );
+	}
+}
+
 /**
  * Perform modular multiplication of big integers
  *
@@ -100,7 +209,10 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 		( ( void * ) result0 );
 	struct {
 		bigint_t ( size * 2 ) result;
-		bigint_t ( size * 2 ) modulus;
+		union {
+			bigint_t ( size * 2 ) modulus;
+			bigint_t ( size * 2 ) carry;
+		};
 	} *temp = tmp;
 	int rotation;
 	int i;
@@ -113,7 +225,8 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 
 	/* Perform multiplication */
 	profile_start ( &bigint_mod_multiply_multiply_profiler );
-	bigint_multiply ( multiplicand, multiplier, &temp->result );
+	bigint_multiply ( multiplicand, multiplier, &temp->result,
+			  &temp->carry );
 	profile_stop ( &bigint_mod_multiply_multiply_profiler );
 
 	/* Rescale modulus to match result */
diff --git a/src/crypto/x25519.c b/src/crypto/x25519.c
index d58f7168c..553f43d34 100644
--- a/src/crypto/x25519.c
+++ b/src/crypto/x25519.c
@@ -43,7 +43,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Storage size of each big integer	     128	      40
  * (in bytes)
  *
- * Stack usage for key exchange		    1144	     360
+ * Stack usage for key exchange		    1144	     424
  * (in bytes, large objects only)
  *
  * Cost of big integer addition		      16	       5
@@ -207,35 +207,60 @@ union x25519_multiply_step3 {
  * We overlap the buffers used by each step of the multiplication
  * calculation to reduce the total stack space required:
  *
- * |--------------------------------------------------------|
- * | <- pad -> | <------------ step 1 result -------------> |
- * |           | <- low 256 bits -> | <-- high 260 bits --> |
- * | <------- step 2 result ------> | <-- step 3 result --> |
- * |--------------------------------------------------------|
+ * |--------------------------------------------------------------------------|
+ * | <------- step 1 carry ------> | <----------- step 1 result ------------> |
+ * |                               | <- low 256 bits -> | <- high 260 bits -> |
+ * | <- step 2 carry -> | <-- step 2 result --> | <pad> |                     |
+ * | <- s3 carry -> | <--------- pad ---------> | <- step 3 result -> |       |
+ * |--------------------------------------------------------------------------|
  */
 union x25519_multiply_workspace {
-	/** Step 1 result */
+	/** Step 1 */
 	struct {
-		/** Padding to avoid collision between steps 1 and 2
-		 *
-		 * The step 2 multiplication consumes the high 260
-		 * bits of step 1, and so the step 2 multiplication
-		 * result must not overlap this portion of the step 1
-		 * result.
-		 */
-		uint8_t pad[ sizeof ( union x25519_multiply_step2 ) -
-			     offsetof ( union x25519_multiply_step1,
-					parts.high_260bit ) ];
+		/** Step 1 temporary carry workspace */
+		union x25519_multiply_step1 carry;
 		/** Step 1 result */
-		union x25519_multiply_step1 step1;
-	} __attribute__ (( packed ));
-	/** Steps 2 and 3 results */
+		union x25519_multiply_step1 result;
+	} __attribute__ (( packed )) step1;
+	/** Step 2
+	 *
+	 * The step 2 multiplication consumes the high 260 bits of
+	 * step 1, and so the step 2 multiplication result (and
+	 * temporary carry workspace) must not overlap this portion of
+	 * the step 1 result.
+	 */
 	struct {
+		/** Step 2 temporary carry workspace */
+		union x25519_multiply_step2 carry;
 		/** Step 2 result */
-		union x25519_multiply_step2 step2;
+		union x25519_multiply_step2 result;
+		/** Avoid collision between step 1 result and step 2 result */
+		uint8_t pad[ ( int )
+			     ( sizeof ( union x25519_multiply_step1 ) +
+			       offsetof ( union x25519_multiply_step1,
+					  parts.high_260bit ) -
+			       sizeof ( union x25519_multiply_step2 ) -
+			       sizeof ( union x25519_multiply_step2 ) ) ];
+	} __attribute__ (( packed )) step2;
+	/** Step 3
+	 *
+	 * The step 3 multiplication consumes the high 11 bits of step
+	 * 2, and so the step 3 multiplication result (and temporary
+	 * carry workspace) must not overlap this portion of the step
+	 * 2 result.
+	 */
+	struct {
+		/** Step 3 temporary carry workspace */
+		union x25519_multiply_step3 carry;
+		/** Avoid collision between step 2 result and step 3 carry */
+		uint8_t pad1[ ( int )
+			      ( sizeof ( union x25519_multiply_step2 ) -
+				sizeof ( union x25519_multiply_step3 ) ) ];
+		/** Avoid collision between step 2 result and step 3 result */
+		uint8_t pad2[ sizeof ( union x25519_multiply_step2 ) ];
 		/** Step 3 result */
-		union x25519_multiply_step3 step3;
-	} __attribute__ (( packed ));
+		union x25519_multiply_step3 result;
+	} __attribute__ (( packed )) step3;
 };
 
 /** An X25519 elliptic curve point in projective coordinates
@@ -426,9 +451,9 @@ void x25519_multiply ( const union x25519_oct258 *multiplicand,
 		       const union x25519_oct258 *multiplier,
 		       union x25519_quad257 *result ) {
 	union x25519_multiply_workspace tmp;
-	union x25519_multiply_step1 *step1 = &tmp.step1;
-	union x25519_multiply_step2 *step2 = &tmp.step2;
-	union x25519_multiply_step3 *step3 = &tmp.step3;
+	union x25519_multiply_step1 *step1 = &tmp.step1.result;
+	union x25519_multiply_step2 *step2 = &tmp.step2.result;
+	union x25519_multiply_step3 *step3 = &tmp.step3.result;
 
 	/* Step 1: perform raw multiplication
 	 *
@@ -439,7 +464,7 @@ void x25519_multiply ( const union x25519_oct258 *multiplicand,
 	 */
 	static_assert ( sizeof ( step1->product ) >= sizeof ( step1->parts ) );
 	bigint_multiply ( &multiplicand->value, &multiplier->value,
-			  &step1->product );
+			  &step1->product, &tmp.step1.carry.product );
 
 	/* Step 2: reduce high-order 516-256=260 bits of step 1 result
 	 *
@@ -465,7 +490,7 @@ void x25519_multiply ( const union x25519_oct258 *multiplicand,
 	static_assert ( sizeof ( step2->product ) >= sizeof ( step2->parts ) );
 	bigint_grow ( &step1->parts.low_256bit, &result->value );
 	bigint_multiply ( &step1->parts.high_260bit, &x25519_reduce_256,
-			  &step2->product );
+			  &step2->product, &tmp.step2.carry.product );
 	bigint_add ( &result->value, &step2->value );
 
 	/* Step 3: reduce high-order 267-256=11 bits of step 2 result
@@ -503,7 +528,7 @@ void x25519_multiply ( const union x25519_oct258 *multiplicand,
 	memset ( &step3->value, 0, sizeof ( step3->value ) );
 	bigint_grow ( &step2->parts.low_256bit, &result->value );
 	bigint_multiply ( &step2->parts.high_11bit, &x25519_reduce_256,
-			  &step3->product );
+			  &step3->product, &tmp.step3.carry.product );
 	bigint_add ( &step3->value, &result->value );
 
 	/* Step 1 calculates the product of the input operands, and
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 3dc344dff..efe156596 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -208,13 +208,15 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v multiplicand	Big integer to be multiplied
  * @v multiplier	Big integer to be multiplied
  * @v result		Big integer to hold result
+ * @v carry		Big integer to hold temporary carry space
  */
-#define bigint_multiply( multiplicand, multiplier, result ) do {	\
+#define bigint_multiply( multiplicand, multiplier, result, carry ) do {	\
 	unsigned int multiplicand_size = bigint_size (multiplicand);	\
 	unsigned int multiplier_size = bigint_size (multiplier);	\
 	bigint_multiply_raw ( (multiplicand)->element,			\
 			      multiplicand_size, (multiplier)->element,	\
-			      multiplier_size, (result)->element );	\
+			      multiplier_size, (result)->element,	\
+			      (carry)->element );			\
 	} while ( 0 )
 
 /**
@@ -245,7 +247,10 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	unsigned int size = bigint_size (modulus);			\
 	sizeof ( struct {						\
 		bigint_t ( size * 2 ) temp_result;			\
-		bigint_t ( size * 2 ) temp_modulus;			\
+		union {							\
+			bigint_t ( size * 2 ) temp_modulus;		\
+			bigint_t ( size * 2 ) temp_carry;		\
+		};							\
 	} ); } )
 
 /**
@@ -311,11 +316,16 @@ void bigint_shrink_raw ( const bigint_element_t *source0,
 			 unsigned int dest_size );
 void bigint_swap_raw ( bigint_element_t *first0, bigint_element_t *second0,
 		       unsigned int size, int swap );
+void bigint_multiply_one ( const bigint_element_t multiplicand,
+			   const bigint_element_t multiplier,
+			   bigint_element_t *result,
+			   bigint_element_t *carry );
 void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 			   unsigned int multiplicand_size,
 			   const bigint_element_t *multiplier0,
 			   unsigned int multiplier_size,
-			   bigint_element_t *result0 );
+			   bigint_element_t *result0,
+			   bigint_element_t *carry0 );
 void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 			       const bigint_element_t *multiplier0,
 			       const bigint_element_t *modulus0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 76aca1059..fcc77f25f 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -173,7 +173,8 @@ void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
 			      unsigned int multiplicand_size,
 			      const bigint_element_t *multiplier0,
 			      unsigned int multiplier_size,
-			      bigint_element_t *result0 ) {
+			      bigint_element_t *result0,
+			      bigint_element_t *carry0 ) {
 	unsigned int result_size = ( multiplicand_size + multiplier_size );
 	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
 		*multiplicand = ( ( const void * ) multiplicand0 );
@@ -181,8 +182,10 @@ void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
 		*multiplier = ( ( const void * ) multiplier0 );
 	bigint_t ( result_size ) __attribute__ (( may_alias ))
 		*result = ( ( void * ) result0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*carry = ( ( void * ) carry0 );
 
-	bigint_multiply ( multiplicand, multiplier, result );
+	bigint_multiply ( multiplicand, multiplier, result, carry );
 }
 
 void bigint_mod_multiply_sample ( const bigint_element_t *multiplicand0,
@@ -495,11 +498,14 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	bigint_t ( multiplicand_size ) multiplicand_temp;		\
 	bigint_t ( multiplier_size ) multiplier_temp;			\
 	bigint_t ( multiplicand_size + multiplier_size ) result_temp;	\
+	bigint_t ( multiplicand_size + multiplier_size ) carry_temp;	\
 	{} /* Fix emacs alignment */					\
 									\
 	assert ( bigint_size ( &result_temp ) ==			\
 		 ( bigint_size ( &multiplicand_temp ) +			\
 		   bigint_size ( &multiplier_temp ) ) );		\
+	assert ( bigint_size ( &carry_temp ) ==				\
+		 bigint_size ( &result_temp ) );			\
 	bigint_init ( &multiplicand_temp, multiplicand_raw,		\
 		      sizeof ( multiplicand_raw ) );			\
 	bigint_init ( &multiplier_temp, multiplier_raw,			\
@@ -508,7 +514,7 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	DBG_HDA ( 0, &multiplicand_temp, sizeof ( multiplicand_temp ) );\
 	DBG_HDA ( 0, &multiplier_temp, sizeof ( multiplier_temp ) );	\
 	bigint_multiply ( &multiplicand_temp, &multiplier_temp,		\
-			  &result_temp );				\
+			  &result_temp, &carry_temp );			\
 	DBG_HDA ( 0, &result_temp, sizeof ( result_temp ) );		\
 	bigint_done ( &result_temp, result_raw, sizeof ( result_raw ) );\
 									\

From 5f7c6bd95bd6089473db3ba4f033584f5de0ee8a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 24 Sep 2024 14:49:32 +0100
Subject: [PATCH 204/237] [profile] Standardise return type of
 profile_timestamp()

All consumers of profile_timestamp() currently treat the value as an
unsigned long.  Only the elapsed number of ticks is ever relevant: the
absolute value of the timestamp is not used.  Profiling is used to
measure short durations that are generally fewer than a million CPU
cycles, for which an unsigned long is easily large enough.

Standardise the return type of profile_timestamp() as unsigned long
across all CPU architectures.  This allows 32-bit architectures such
as i386 and riscv32 to omit all logic associated with retrieving the
upper 32 bits of the 64-bit hardware counter, which simplifies the
code and allows riscv32 and riscv64 to share the same implementation.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/include/bits/profile.h         |  2 +-
 src/arch/arm64/include/bits/profile.h         |  2 +-
 src/arch/i386/include/bits/profile.h          |  6 ++--
 src/arch/loong64/include/bits/profile.h       |  2 +-
 .../{riscv64 => riscv}/include/bits/profile.h |  4 +--
 src/arch/riscv32/include/bits/profile.h       | 36 -------------------
 src/arch/x86_64/include/bits/profile.h        |  2 +-
 src/include/ipxe/profile.h                    |  2 ++
 8 files changed, 11 insertions(+), 45 deletions(-)
 rename src/arch/{riscv64 => riscv}/include/bits/profile.h (81%)
 delete mode 100644 src/arch/riscv32/include/bits/profile.h

diff --git a/src/arch/arm32/include/bits/profile.h b/src/arch/arm32/include/bits/profile.h
index 2b15d1604..4e4bf48a3 100644
--- a/src/arch/arm32/include/bits/profile.h
+++ b/src/arch/arm32/include/bits/profile.h
@@ -16,7 +16,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @ret timestamp	Timestamp
  */
-static inline __attribute__ (( always_inline )) uint64_t
+static inline __attribute__ (( always_inline )) unsigned long
 profile_timestamp ( void ) {
 	uint32_t cycles;
 
diff --git a/src/arch/arm64/include/bits/profile.h b/src/arch/arm64/include/bits/profile.h
index 62ffa3772..4a5b3f7a1 100644
--- a/src/arch/arm64/include/bits/profile.h
+++ b/src/arch/arm64/include/bits/profile.h
@@ -16,7 +16,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @ret timestamp	Timestamp
  */
-static inline __attribute__ (( always_inline )) uint64_t
+static inline __attribute__ (( always_inline )) unsigned long
 profile_timestamp ( void ) {
 	uint64_t cycles;
 
diff --git a/src/arch/i386/include/bits/profile.h b/src/arch/i386/include/bits/profile.h
index e184d7b51..21c216a81 100644
--- a/src/arch/i386/include/bits/profile.h
+++ b/src/arch/i386/include/bits/profile.h
@@ -16,12 +16,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @ret timestamp	Timestamp
  */
-static inline __attribute__ (( always_inline )) uint64_t
+static inline __attribute__ (( always_inline )) unsigned long
 profile_timestamp ( void ) {
-	uint64_t tsc;
+	uint32_t tsc;
 
 	/* Read timestamp counter */
-	__asm__ __volatile__ ( "rdtsc" : "=A" ( tsc ) );
+	__asm__ __volatile__ ( "rdtsc" : "=a" ( tsc ) : : "edx" );
 	return tsc;
 }
 
diff --git a/src/arch/loong64/include/bits/profile.h b/src/arch/loong64/include/bits/profile.h
index 9f597ce2d..02f8d4b7c 100644
--- a/src/arch/loong64/include/bits/profile.h
+++ b/src/arch/loong64/include/bits/profile.h
@@ -16,7 +16,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @ret timestamp	Timestamp
  */
-static inline __attribute__ (( always_inline )) uint64_t
+static inline __attribute__ (( always_inline )) unsigned long
 profile_timestamp ( void ) {
 	uint64_t cycles;
 
diff --git a/src/arch/riscv64/include/bits/profile.h b/src/arch/riscv/include/bits/profile.h
similarity index 81%
rename from src/arch/riscv64/include/bits/profile.h
rename to src/arch/riscv/include/bits/profile.h
index 2c8e29a22..e9e003dab 100644
--- a/src/arch/riscv64/include/bits/profile.h
+++ b/src/arch/riscv/include/bits/profile.h
@@ -16,9 +16,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @ret timestamp	Timestamp
  */
-static inline __attribute__ (( always_inline )) uint64_t
+static inline __attribute__ (( always_inline )) unsigned long
 profile_timestamp ( void ) {
-	uint64_t cycles;
+	unsigned long cycles;
 
 	/* Read timestamp counter */
 	__asm__ __volatile__ ( "rdcycle %0" : "=r" ( cycles ) );
diff --git a/src/arch/riscv32/include/bits/profile.h b/src/arch/riscv32/include/bits/profile.h
deleted file mode 100644
index cb969077d..000000000
--- a/src/arch/riscv32/include/bits/profile.h
+++ /dev/null
@@ -1,36 +0,0 @@
-#ifndef _BITS_PROFILE_H
-#define _BITS_PROFILE_H
-
-/** @file
- *
- * Profiling
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <stdint.h>
-
-/**
- * Get profiling timestamp
- *
- * @ret timestamp	Timestamp
- */
-static inline __attribute__ (( always_inline )) uint64_t
-profile_timestamp ( void ) {
-	uint32_t cycles_lo;
-	uint32_t cycles_hi;
-	uint32_t tmp;
-
-	/* Read timestamp counter */
-	__asm__ __volatile__ ( "\n1:\n\t"
-			       "rdcycleh %1\n\t"
-			       "rdcycle %0\n\t"
-			       "rdcycleh %2\n\t"
-			       "bne %1, %2, 1b\n\t"
-			       : "=r" ( cycles_lo ), "=r" ( cycles_hi ),
-				 "=r" ( tmp ) );
-	return ( ( ( ( uint64_t ) cycles_hi ) << 32 ) | cycles_lo );
-}
-
-#endif /* _BITS_PROFILE_H */
diff --git a/src/arch/x86_64/include/bits/profile.h b/src/arch/x86_64/include/bits/profile.h
index b7c74fbe7..c85b6fe5c 100644
--- a/src/arch/x86_64/include/bits/profile.h
+++ b/src/arch/x86_64/include/bits/profile.h
@@ -16,7 +16,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @ret timestamp	Timestamp
  */
-static inline __attribute__ (( always_inline )) uint64_t
+static inline __attribute__ (( always_inline )) unsigned long
 profile_timestamp ( void ) {
 	uint32_t eax;
 	uint32_t edx;
diff --git a/src/include/ipxe/profile.h b/src/include/ipxe/profile.h
index 2c69e1208..fd45b3cdc 100644
--- a/src/include/ipxe/profile.h
+++ b/src/include/ipxe/profile.h
@@ -60,6 +60,8 @@ struct profiler {
 #define __profiler
 #endif
 
+unsigned long profile_timestamp ( void );
+
 extern unsigned long profile_excluded;
 
 extern void profile_update ( struct profiler *profiler, unsigned long sample );

From e0282688c1d9795a1400718b8c9e3f468530ae97 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 24 Sep 2024 11:52:16 +0100
Subject: [PATCH 205/237] [arm] Check PMCCNTR availability before use for
 profiling

Reading from PMCCNTR causes an undefined instruction exception when
running in PL0 (e.g. as a Linux userspace binary), unless the
PMUSERENR.EN bit is set.

Restructure profile_timestamp() for 32-bit ARM to perform an
availability check on the first invocation, with subsequent
invocations returning zero if PMCCNTR could not be enabled.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/core/pmccntr.S         | 85 +++++++++++++++++++++++++++
 src/arch/arm32/include/bits/profile.h | 17 +++++-
 2 files changed, 99 insertions(+), 3 deletions(-)
 create mode 100644 src/arch/arm32/core/pmccntr.S

diff --git a/src/arch/arm32/core/pmccntr.S b/src/arch/arm32/core/pmccntr.S
new file mode 100644
index 000000000..c2fbeff4f
--- /dev/null
+++ b/src/arch/arm32/core/pmccntr.S
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * Performance Monitor Cycle Counter (PMCCNTR)
+ *
+ */
+
+	.section ".note.GNU-stack", "", %progbits
+	.text
+	.arm
+
+/*
+ * PMCCNTR status
+ *
+ * bit 31 	set if PMCCNTR availability is not yet determined
+ * bit 0	set if PMCCNTR is available
+ *
+ */
+	.section ".data.pmccntr_status", "aw", %progbits
+	.globl	pmccntr_status
+pmccntr_status:
+	.word	0x80000000
+
+/*
+ * Check PMCCNTR availability
+ *
+ * Must preserve all registers, and return with either PMCCNTR enabled
+ * or the Z flag set to indicate unavailability.
+ *
+ */
+	.section ".text.pmccntr_check", "ax", %progbits
+	.globl	pmccntr_check
+	.type	pmccntr_check, %function
+pmccntr_check:
+	/* Save registers */
+	stmfd	sp!, { r0, r1 }
+	/* Read CPSR.M (bits 3:0, always permitted in PL0) */
+	mrs	r0, cpsr
+	and	r0, r0, #0x0000000f
+	/* Read PMUSERENR.EN (bit 0, always permitted in PL0) */
+	mrc	p15, 0, r1, c9, c14, 0
+	and	r1, r1, #0x00000001
+	/* Check if we are in PL1+ or in PL0 with PMUSERENR.EN set */
+	orrs	r0, r0, r1
+	/* If PMCCNTR is unavailable, exit with status=0 and ZF set */
+	beq	1f
+	/* Set PMCR.E (bit 0), set exit status=1 and ZF clear */
+	movs	r0, #0x00000001
+	mcr	p15, 0, r0, c9, c12, 0
+	/* Set PMCNTENSET.C (bit 31) */
+	mov	r1, #0x80000000
+	mcr	p15, 0, r1, c9, c12, 1
+1:	/* Store PMCCNTR status */
+	ldr	r1, pmccntr_status_ptr
+	str	r0, [r1]
+	/* Restore registers and return */
+	ldmfd	sp!, { r0, r1 }
+	bx	lr
+pmccntr_status_ptr:
+	.word pmccntr_status
+	.size	pmccntr_check, . - pmccntr_check
diff --git a/src/arch/arm32/include/bits/profile.h b/src/arch/arm32/include/bits/profile.h
index 4e4bf48a3..31c321884 100644
--- a/src/arch/arm32/include/bits/profile.h
+++ b/src/arch/arm32/include/bits/profile.h
@@ -11,6 +11,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
+extern uint32_t pmccntr_status;
+
 /**
  * Get profiling timestamp
  *
@@ -21,9 +23,18 @@ profile_timestamp ( void ) {
 	uint32_t cycles;
 
 	/* Read cycle counter */
-	__asm__ __volatile__ ( "mcr p15, 0, %1, c9, c12, 0\n\t"
-			       "mrc p15, 0, %0, c9, c13, 0\n\t"
-			       : "=r" ( cycles ) : "r" ( 1 ) );
+	__asm__ __volatile__ ( /* Check PMCCNTR status */
+			       "tst %0, %0\n\t"
+			       /* Check availability if not yet known */
+			       "it mi\n\t"
+			       "blxmi pmccntr_check\n\t"
+			       /* Read PMCCNTR if available */
+			       "it ne\n\t"
+			       "mrcne p15, 0, %0, c9, c13, 0\n\t"
+			       "\n1:\n\t"
+			       : "=r" ( cycles )
+			       : "0" ( pmccntr_status )
+			       : "cc", "lr" );
 	return cycles;
 }
 

From 8844a3d5464c4632a1374cbb9304aeea61d0166f Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 24 Sep 2024 19:15:11 +0100
Subject: [PATCH 206/237] [arm] Support building as a Linux userspace binary
 for AArch32

Add support for building as a Linux userspace binary for AArch32.
This allows the self-test suite to be more easily run for the 32-bit
ARM code.  For example:

  make CROSS=arm-linux-gnu- bin-arm32-linux/tests.linux

  qemu-arm -L /usr/arm-linux-gnu/sys-root/ \
           ./bin-arm32-linux/tests.linux

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/Makefile.linux | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 src/arch/arm32/Makefile.linux

diff --git a/src/arch/arm32/Makefile.linux b/src/arch/arm32/Makefile.linux
new file mode 100644
index 000000000..289118f29
--- /dev/null
+++ b/src/arch/arm32/Makefile.linux
@@ -0,0 +1,25 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# The number of different ABIs for 32-bit ARM is insane.  It is
+# unclear whether or not unaligned accesses ought to work in a 32-bit
+# Linux userspace binary.  When running in QEMU, unaligned accesses
+# result in a SIGBUS.  Since this is likely to be the most common use
+# case (for running self-tests on an x86 build machine), and since we
+# don't particularly care about performance for Linux userspace
+# binaries, force the compiler to never generate an unaligned access.
+#
+CFLAGS += -mno-unaligned-access
+
+# Inhibit the harmless warning about wchar_t size mismatch between the
+# linux_api.o helper object and the rest of iPXE.
+#
+LINUX_CFLAGS += -Wl,--no-wchar-size-warning
+
+# Starting virtual address
+#
+LDFLAGS += -Ttext=0x10000
+
+# Include generic Linux Makefile
+#
+MAKEDEPS += arch/arm/Makefile.linux
+include arch/arm/Makefile.linux

From 3f4f843920afdc1d808a8b20354cf3eca481401a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 26 Sep 2024 16:24:57 +0100
Subject: [PATCH 207/237] [crypto] Eliminate temporary carry space for big
 integer multiplication

An n-bit multiplication product may be added to up to two n-bit
integers without exceeding the range of a (2n)-bit integer:

  (2^n - 1)*(2^n - 1) + (2^n - 1) + (2^n - 1) = 2^(2n) - 1

Exploit this to perform big integer multiplication in constant time
without requiring the caller to provide temporary carry space.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/include/bits/bigint.h   |  15 ++--
 src/arch/arm64/include/bits/bigint.h   |  17 ++--
 src/arch/loong64/include/bits/bigint.h |  17 ++--
 src/arch/riscv/include/bits/bigint.h   |  17 ++--
 src/arch/x86/include/bits/bigint.h     |  25 +++---
 src/crypto/bigint.c                    | 108 ++++++++-----------------
 src/crypto/x25519.c                    |  83 +++++++------------
 src/include/ipxe/bigint.h              |  14 +---
 src/tests/bigint_test.c                |  12 +--
 9 files changed, 111 insertions(+), 197 deletions(-)

diff --git a/src/arch/arm32/include/bits/bigint.h b/src/arch/arm32/include/bits/bigint.h
index 0a368a0c0..c148d6be2 100644
--- a/src/arch/arm32/include/bits/bigint.h
+++ b/src/arch/arm32/include/bits/bigint.h
@@ -314,7 +314,7 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused,
  *
  * @v multiplicand	Multiplicand element
  * @v multiplier	Multiplier element
- * @v result		Result element pair
+ * @v result		Result element
  * @v carry		Carry element
  */
 static inline __attribute__ (( always_inline )) void
@@ -324,19 +324,20 @@ bigint_multiply_one ( const uint32_t multiplicand, const uint32_t multiplier,
 	uint32_t discard_high;
 
 	__asm__ __volatile__ ( /* Perform multiplication */
-			       "umull %0, %1, %5, %6\n\t"
+			       "umull %0, %1, %4, %5\n\t"
 			       /* Accumulate result */
 			       "adds %2, %0\n\t"
-			       "adcs %3, %1\n\t"
+			       "adc %1, #0\n\t"
 			       /* Accumulate carry (cannot overflow) */
-			       "adc %4, #0\n\t"
+			       "adds %2, %3\n\t"
+			       "adc %3, %1, #0\n\t"
 			       : "=r" ( discard_low ),
 				 "=r" ( discard_high ),
-				 "+r" ( result[0] ),
-				 "+r" ( result[1] ),
+				 "+r" ( *result ),
 				 "+r" ( *carry )
 			       : "r" ( multiplicand ),
-				 "r" ( multiplier ) );
+				 "r" ( multiplier )
+			       : "cc" );
 }
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/arm64/include/bits/bigint.h b/src/arch/arm64/include/bits/bigint.h
index ca9feafbe..6b62c809e 100644
--- a/src/arch/arm64/include/bits/bigint.h
+++ b/src/arch/arm64/include/bits/bigint.h
@@ -315,7 +315,7 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
  *
  * @v multiplicand	Multiplicand element
  * @v multiplier	Multiplier element
- * @v result		Result element pair
+ * @v result		Result element
  * @v carry		Carry element
  */
 static inline __attribute__ (( always_inline )) void
@@ -325,20 +325,21 @@ bigint_multiply_one ( const uint64_t multiplicand, const uint64_t multiplier,
 	uint64_t discard_high;
 
 	__asm__ __volatile__ ( /* Perform multiplication */
-			       "mul %0, %5, %6\n\t"
-			       "umulh %1, %5, %6\n\t"
+			       "mul %0, %4, %5\n\t"
+			       "umulh %1, %4, %5\n\t"
 			       /* Accumulate result */
 			       "adds %2, %2, %0\n\t"
-			       "adcs %3, %3, %1\n\t"
+			       "adc %1, %1, xzr\n\t"
 			       /* Accumulate carry (cannot overflow) */
-			       "adc %4, %4, xzr\n\t"
+			       "adds %2, %2, %3\n\t"
+			       "adc %3, %1, xzr\n\t"
 			       : "=&r" ( discard_low ),
 				 "=r" ( discard_high ),
-				 "+r" ( result[0] ),
-				 "+r" ( result[1] ),
+				 "+r" ( *result ),
 				 "+r" ( *carry )
 			       : "r" ( multiplicand ),
-				 "r" ( multiplier ) );
+				 "r" ( multiplier )
+			       : "cc" );
 }
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h
index ec6ca4b89..23edaeea5 100644
--- a/src/arch/loong64/include/bits/bigint.h
+++ b/src/arch/loong64/include/bits/bigint.h
@@ -362,7 +362,7 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
  *
  * @v multiplicand	Multiplicand element
  * @v multiplier	Multiplier element
- * @v result		Result element pair
+ * @v result		Result element
  * @v carry		Carry element
  */
 static inline __attribute__ (( always_inline )) void
@@ -373,23 +373,20 @@ bigint_multiply_one ( const uint64_t multiplicand, const uint64_t multiplier,
 	uint64_t discard_carry;
 
 	__asm__ __volatile__ ( /* Perform multiplication */
-			       "mul.d %0, %6, %7\n\t"
-			       "mulh.du %1, %6, %7\n\t"
+			       "mul.d %0, %5, %6\n\t"
+			       "mulh.du %1, %5, %6\n\t"
 			       /* Accumulate low half */
 			       "add.d %3, %3, %0\n\t"
 			       "sltu %2, %3, %0\n\t"
-			       /* Add carry to high half (cannot overflow) */
 			       "add.d %1, %1, %2\n\t"
-			       /* Accumulate high half */
-			       "add.d %4, %4, %1\n\t"
-			       "sltu %2, %4, %1\n\t"
 			       /* Accumulate carry (cannot overflow) */
-			       "add.d %5, %5, %2\n\t"
+			       "add.d %3, %3, %4\n\t"
+			       "sltu %2, %3, %4\n\t"
+			       "add.d %4, %1, %2\n\t"
 			       : "=&r" ( discard_low ),
 				 "=r" ( discard_high ),
 				 "=r" ( discard_carry ),
-				 "+r" ( result[0] ),
-				 "+r" ( result[1] ),
+				 "+r" ( *result ),
 				 "+r" ( *carry )
 			       : "r" ( multiplicand ),
 				 "r" ( multiplier ) );
diff --git a/src/arch/riscv/include/bits/bigint.h b/src/arch/riscv/include/bits/bigint.h
index c58233469..13fd16759 100644
--- a/src/arch/riscv/include/bits/bigint.h
+++ b/src/arch/riscv/include/bits/bigint.h
@@ -358,7 +358,7 @@ bigint_done_raw ( const unsigned long *value0, unsigned int size __unused,
  *
  * @v multiplicand	Multiplicand element
  * @v multiplier	Multiplier element
- * @v result		Result element pair
+ * @v result		Result element
  * @v carry		Carry element
  */
 static inline __attribute__ (( always_inline )) void
@@ -370,23 +370,20 @@ bigint_multiply_one ( const unsigned long multiplicand,
 	unsigned long discard_carry;
 
 	__asm__ __volatile__ ( /* Perform multiplication */
-			       "mulhu %1, %6, %7\n\t"
-			       "mul %0, %6, %7\n\t"
+			       "mulhu %1, %5, %6\n\t"
+			       "mul %0, %5, %6\n\t"
 			       /* Accumulate low half */
 			       "add %3, %3, %0\n\t"
 			       "sltu %2, %3, %0\n\t"
-			       /* Add carry to high half (cannot overflow) */
 			       "add %1, %1, %2\n\t"
-			       /* Accumulate high half */
-			       "add %4, %4, %1\n\t"
-			       "sltu %2, %4, %1\n\t"
 			       /* Accumulate carry (cannot overflow) */
-			       "add %5, %5, %2\n\t"
+			       "add %3, %3, %4\n\t"
+			       "sltu %2, %3, %4\n\t"
+			       "add %4, %1, %2\n\t"
 			       : "=r" ( discard_low ),
 				 "=&r" ( discard_high ),
 				 "=r" ( discard_carry ),
-				 "+r" ( result[0] ),
-				 "+r" ( result[1] ),
+				 "+r" ( *result ),
 				 "+r" ( *carry )
 			       : "r" ( multiplicand ),
 				 "r" ( multiplier ) );
diff --git a/src/arch/x86/include/bits/bigint.h b/src/arch/x86/include/bits/bigint.h
index a481e90f7..320d49498 100644
--- a/src/arch/x86/include/bits/bigint.h
+++ b/src/arch/x86/include/bits/bigint.h
@@ -327,29 +327,28 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused,
  *
  * @v multiplicand	Multiplicand element
  * @v multiplier	Multiplier element
- * @v result		Result element pair
+ * @v result		Result element
  * @v carry		Carry element
  */
 static inline __attribute__ (( always_inline )) void
 bigint_multiply_one ( const uint32_t multiplicand, const uint32_t multiplier,
 		      uint32_t *result, uint32_t *carry ) {
 	uint32_t discard_a;
-	uint32_t discard_d;
 
 	__asm__ __volatile__ ( /* Perform multiplication */
-			       "mull %6\n\t"
+			       "mull %3\n\t"
+			       /* Accumulate carry */
+			       "addl %5, %0\n\t"
+			       "adcl $0, %1\n\t"
 			       /* Accumulate result */
 			       "addl %0, %2\n\t"
-			       "adcl %1, %3\n\t"
-			       /* Accumulate carry (cannot overflow) */
-			       "adcl $0, %4\n\t"
-			       : "=a" ( discard_a ),
-				 "=d" ( discard_d ),
-				 "+m" ( result[0] ),
-				 "+m" ( result[1] ),
-				 "+m" ( *carry )
-			       : "0" ( multiplicand ),
-				 "g" ( multiplier ) );
+			       "adcl $0, %1\n\t"
+			       : "=&a" ( discard_a ),
+				 "=&d" ( *carry ),
+				 "+m" ( *result )
+			       : "g" ( multiplicand ),
+				 "0" ( multiplier ),
+				 "r" ( *carry ) );
 }
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index 5b7116e28..b63f7ccc1 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -83,14 +83,12 @@ void bigint_swap_raw ( bigint_element_t *first0, bigint_element_t *second0,
  * @v multiplier0	Element 0 of big integer to be multiplied
  * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v carry0		Element 0 of big integer to hold temporary carry
  */
 void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 			   unsigned int multiplicand_size,
 			   const bigint_element_t *multiplier0,
 			   unsigned int multiplier_size,
-			   bigint_element_t *result0,
-			   bigint_element_t *carry0 ) {
+			   bigint_element_t *result0 ) {
 	unsigned int result_size = ( multiplicand_size + multiplier_size );
 	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
 		*multiplicand = ( ( const void * ) multiplicand0 );
@@ -98,89 +96,51 @@ void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 		*multiplier = ( ( const void * ) multiplier0 );
 	bigint_t ( result_size ) __attribute__ (( may_alias ))
 		*result = ( ( void * ) result0 );
-	bigint_t ( result_size ) __attribute__ (( may_alias ))
-		*carry = ( ( void * ) carry0 );
 	bigint_element_t multiplicand_element;
 	const bigint_element_t *multiplier_element;
-	bigint_element_t *result_elements;
-	bigint_element_t *carry_element;
+	bigint_element_t *result_element;
+	bigint_element_t carry_element;
 	unsigned int i;
 	unsigned int j;
 
-	/* Zero result and temporary carry space */
-	memset ( result, 0, sizeof ( *result ) );
-	memset ( carry, 0, sizeof ( *carry ) );
+	/* Zero required portion of result
+	 *
+	 * All elements beyond the length of the multiplier will be
+	 * written before they are read, and so do not need to be
+	 * zeroed in advance.
+	 */
+	memset ( result, 0, sizeof ( *multiplier ) );
 
-	/* Multiply integers one element at a time, adding the double
-	 * element directly into the result and accumulating any
-	 * overall carry out from this double-element addition into
-	 * the temporary carry space.
+	/* Multiply integers one element at a time, adding the low
+	 * half of the double-element product directly into the
+	 * result, and maintaining a running single-element carry.
 	 *
-	 * We could propagate the carry immediately instead of using a
-	 * temporary carry space.  However, this would cause the
-	 * multiplication to run in non-constant time, which is
-	 * undesirable.
+	 * The running carry can never overflow beyond a single
+	 * element.  At each step, the calculation we perform is:
 	 *
-	 * The carry elements can never overflow, provided that the
-	 * element size is large enough to accommodate any plausible
-	 * big integer.  The total number of potential carries (across
-	 * all elements) is the sum of the number of elements in the
-	 * multiplicand and multiplier.  With a 16-bit element size,
-	 * this therefore allows for up to a 1Mbit multiplication
-	 * result (e.g. a 512kbit integer multiplied by another
-	 * 512kbit integer), which is around 100x higher than could be
-	 * needed in practice.  With a more realistic 32-bit element
-	 * size, the limit becomes a totally implausible 128Gbit
-	 * multiplication result.
+	 *   carry:result[i+j] := ( ( multiplicand[i] * multiplier[j] )
+	 *                          + result[i+j] + carry )
+	 *
+	 * The maximum value (for n-bit elements) is therefore:
+	 *
+	 *   (2^n - 1)*(2^n - 1) + (2^n - 1) + (2^n - 1) = 2^(2n) - 1
+	 *
+	 * This is precisely the maximum value for a 2n-bit integer,
+	 * and so the carry out remains within the range of an n-bit
+	 * integer, i.e. a single element.
 	 */
 	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
 		multiplier_element = &multiplier->element[0];
-		result_elements = &result->element[i];
-		carry_element = &carry->element[i];
+		result_element = &result->element[i];
+		carry_element = 0;
 		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			bigint_multiply_one ( multiplicand_element,
 					      *(multiplier_element++),
-					      result_elements++,
-					      carry_element++ );
+					      result_element++,
+					      &carry_element );
 		}
-	}
-
-	/* Add the temporary carry into the result.  The least
-	 * significant element of the carry represents the carry out
-	 * from multiplying the least significant elements of the
-	 * multiplicand and multiplier, and therefore must be added to
-	 * the third-least significant element of the result (i.e. the
-	 * carry needs to be shifted left by two elements before being
-	 * adding to the result).
-	 *
-	 * The most significant two elements of the carry are
-	 * guaranteed to be zero, since:
-	 *
-	 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
-	 *
-	 * and the overall result of the multiplication (including
-	 * adding in the shifted carries) is therefore guaranteed not
-	 * to overflow beyond the end of the result.
-	 *
-	 * We could avoid this shifting by writing the carry directly
-	 * into the "correct" element during the element-by-element
-	 * multiplication stage above.  However, this would add
-	 * complexity to the loop since we would have to arrange for
-	 * the (provably zero) most significant two carry out results
-	 * to be discarded, in order to avoid writing beyond the end
-	 * of the temporary carry space.
-	 *
-	 * Performing the logical shift is essentially free, since we
-	 * simply adjust the element pointers.
-	 *
-	 * To avoid requiring additional checks in each architecture's
-	 * implementation of bigint_add_raw(), we explicitly avoid
-	 * calling bigint_add_raw() with a size of zero.
-	 */
-	if ( result_size > 2 ) {
-		bigint_add_raw ( &carry->element[0], &result->element[2],
-				 ( result_size - 2 ) );
+		*result_element = carry_element;
 	}
 }
 
@@ -209,10 +169,7 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 		( ( void * ) result0 );
 	struct {
 		bigint_t ( size * 2 ) result;
-		union {
-			bigint_t ( size * 2 ) modulus;
-			bigint_t ( size * 2 ) carry;
-		};
+		bigint_t ( size * 2 ) modulus;
 	} *temp = tmp;
 	int rotation;
 	int i;
@@ -225,8 +182,7 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 
 	/* Perform multiplication */
 	profile_start ( &bigint_mod_multiply_multiply_profiler );
-	bigint_multiply ( multiplicand, multiplier, &temp->result,
-			  &temp->carry );
+	bigint_multiply ( multiplicand, multiplier, &temp->result );
 	profile_stop ( &bigint_mod_multiply_multiply_profiler );
 
 	/* Rescale modulus to match result */
diff --git a/src/crypto/x25519.c b/src/crypto/x25519.c
index 553f43d34..d58f7168c 100644
--- a/src/crypto/x25519.c
+++ b/src/crypto/x25519.c
@@ -43,7 +43,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Storage size of each big integer	     128	      40
  * (in bytes)
  *
- * Stack usage for key exchange		    1144	     424
+ * Stack usage for key exchange		    1144	     360
  * (in bytes, large objects only)
  *
  * Cost of big integer addition		      16	       5
@@ -207,60 +207,35 @@ union x25519_multiply_step3 {
  * We overlap the buffers used by each step of the multiplication
  * calculation to reduce the total stack space required:
  *
- * |--------------------------------------------------------------------------|
- * | <------- step 1 carry ------> | <----------- step 1 result ------------> |
- * |                               | <- low 256 bits -> | <- high 260 bits -> |
- * | <- step 2 carry -> | <-- step 2 result --> | <pad> |                     |
- * | <- s3 carry -> | <--------- pad ---------> | <- step 3 result -> |       |
- * |--------------------------------------------------------------------------|
+ * |--------------------------------------------------------|
+ * | <- pad -> | <------------ step 1 result -------------> |
+ * |           | <- low 256 bits -> | <-- high 260 bits --> |
+ * | <------- step 2 result ------> | <-- step 3 result --> |
+ * |--------------------------------------------------------|
  */
 union x25519_multiply_workspace {
-	/** Step 1 */
+	/** Step 1 result */
 	struct {
-		/** Step 1 temporary carry workspace */
-		union x25519_multiply_step1 carry;
+		/** Padding to avoid collision between steps 1 and 2
+		 *
+		 * The step 2 multiplication consumes the high 260
+		 * bits of step 1, and so the step 2 multiplication
+		 * result must not overlap this portion of the step 1
+		 * result.
+		 */
+		uint8_t pad[ sizeof ( union x25519_multiply_step2 ) -
+			     offsetof ( union x25519_multiply_step1,
+					parts.high_260bit ) ];
 		/** Step 1 result */
-		union x25519_multiply_step1 result;
-	} __attribute__ (( packed )) step1;
-	/** Step 2
-	 *
-	 * The step 2 multiplication consumes the high 260 bits of
-	 * step 1, and so the step 2 multiplication result (and
-	 * temporary carry workspace) must not overlap this portion of
-	 * the step 1 result.
-	 */
+		union x25519_multiply_step1 step1;
+	} __attribute__ (( packed ));
+	/** Steps 2 and 3 results */
 	struct {
-		/** Step 2 temporary carry workspace */
-		union x25519_multiply_step2 carry;
 		/** Step 2 result */
-		union x25519_multiply_step2 result;
-		/** Avoid collision between step 1 result and step 2 result */
-		uint8_t pad[ ( int )
-			     ( sizeof ( union x25519_multiply_step1 ) +
-			       offsetof ( union x25519_multiply_step1,
-					  parts.high_260bit ) -
-			       sizeof ( union x25519_multiply_step2 ) -
-			       sizeof ( union x25519_multiply_step2 ) ) ];
-	} __attribute__ (( packed )) step2;
-	/** Step 3
-	 *
-	 * The step 3 multiplication consumes the high 11 bits of step
-	 * 2, and so the step 3 multiplication result (and temporary
-	 * carry workspace) must not overlap this portion of the step
-	 * 2 result.
-	 */
-	struct {
-		/** Step 3 temporary carry workspace */
-		union x25519_multiply_step3 carry;
-		/** Avoid collision between step 2 result and step 3 carry */
-		uint8_t pad1[ ( int )
-			      ( sizeof ( union x25519_multiply_step2 ) -
-				sizeof ( union x25519_multiply_step3 ) ) ];
-		/** Avoid collision between step 2 result and step 3 result */
-		uint8_t pad2[ sizeof ( union x25519_multiply_step2 ) ];
+		union x25519_multiply_step2 step2;
 		/** Step 3 result */
-		union x25519_multiply_step3 result;
-	} __attribute__ (( packed )) step3;
+		union x25519_multiply_step3 step3;
+	} __attribute__ (( packed ));
 };
 
 /** An X25519 elliptic curve point in projective coordinates
@@ -451,9 +426,9 @@ void x25519_multiply ( const union x25519_oct258 *multiplicand,
 		       const union x25519_oct258 *multiplier,
 		       union x25519_quad257 *result ) {
 	union x25519_multiply_workspace tmp;
-	union x25519_multiply_step1 *step1 = &tmp.step1.result;
-	union x25519_multiply_step2 *step2 = &tmp.step2.result;
-	union x25519_multiply_step3 *step3 = &tmp.step3.result;
+	union x25519_multiply_step1 *step1 = &tmp.step1;
+	union x25519_multiply_step2 *step2 = &tmp.step2;
+	union x25519_multiply_step3 *step3 = &tmp.step3;
 
 	/* Step 1: perform raw multiplication
 	 *
@@ -464,7 +439,7 @@ void x25519_multiply ( const union x25519_oct258 *multiplicand,
 	 */
 	static_assert ( sizeof ( step1->product ) >= sizeof ( step1->parts ) );
 	bigint_multiply ( &multiplicand->value, &multiplier->value,
-			  &step1->product, &tmp.step1.carry.product );
+			  &step1->product );
 
 	/* Step 2: reduce high-order 516-256=260 bits of step 1 result
 	 *
@@ -490,7 +465,7 @@ void x25519_multiply ( const union x25519_oct258 *multiplicand,
 	static_assert ( sizeof ( step2->product ) >= sizeof ( step2->parts ) );
 	bigint_grow ( &step1->parts.low_256bit, &result->value );
 	bigint_multiply ( &step1->parts.high_260bit, &x25519_reduce_256,
-			  &step2->product, &tmp.step2.carry.product );
+			  &step2->product );
 	bigint_add ( &result->value, &step2->value );
 
 	/* Step 3: reduce high-order 267-256=11 bits of step 2 result
@@ -528,7 +503,7 @@ void x25519_multiply ( const union x25519_oct258 *multiplicand,
 	memset ( &step3->value, 0, sizeof ( step3->value ) );
 	bigint_grow ( &step2->parts.low_256bit, &result->value );
 	bigint_multiply ( &step2->parts.high_11bit, &x25519_reduce_256,
-			  &step3->product, &tmp.step3.carry.product );
+			  &step3->product );
 	bigint_add ( &step3->value, &result->value );
 
 	/* Step 1 calculates the product of the input operands, and
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index efe156596..bcb7af5ec 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -208,15 +208,13 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v multiplicand	Big integer to be multiplied
  * @v multiplier	Big integer to be multiplied
  * @v result		Big integer to hold result
- * @v carry		Big integer to hold temporary carry space
  */
-#define bigint_multiply( multiplicand, multiplier, result, carry ) do {	\
+#define bigint_multiply( multiplicand, multiplier, result ) do {	\
 	unsigned int multiplicand_size = bigint_size (multiplicand);	\
 	unsigned int multiplier_size = bigint_size (multiplier);	\
 	bigint_multiply_raw ( (multiplicand)->element,			\
 			      multiplicand_size, (multiplier)->element,	\
-			      multiplier_size, (result)->element,	\
-			      (carry)->element );			\
+			      multiplier_size, (result)->element );	\
 	} while ( 0 )
 
 /**
@@ -247,10 +245,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	unsigned int size = bigint_size (modulus);			\
 	sizeof ( struct {						\
 		bigint_t ( size * 2 ) temp_result;			\
-		union {							\
-			bigint_t ( size * 2 ) temp_modulus;		\
-			bigint_t ( size * 2 ) temp_carry;		\
-		};							\
+		bigint_t ( size * 2 ) temp_modulus;			\
 	} ); } )
 
 /**
@@ -324,8 +319,7 @@ void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 			   unsigned int multiplicand_size,
 			   const bigint_element_t *multiplier0,
 			   unsigned int multiplier_size,
-			   bigint_element_t *result0,
-			   bigint_element_t *carry0 );
+			   bigint_element_t *result0 );
 void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 			       const bigint_element_t *multiplier0,
 			       const bigint_element_t *modulus0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index fcc77f25f..76aca1059 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -173,8 +173,7 @@ void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
 			      unsigned int multiplicand_size,
 			      const bigint_element_t *multiplier0,
 			      unsigned int multiplier_size,
-			      bigint_element_t *result0,
-			      bigint_element_t *carry0 ) {
+			      bigint_element_t *result0 ) {
 	unsigned int result_size = ( multiplicand_size + multiplier_size );
 	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
 		*multiplicand = ( ( const void * ) multiplicand0 );
@@ -182,10 +181,8 @@ void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
 		*multiplier = ( ( const void * ) multiplier0 );
 	bigint_t ( result_size ) __attribute__ (( may_alias ))
 		*result = ( ( void * ) result0 );
-	bigint_t ( result_size ) __attribute__ (( may_alias ))
-		*carry = ( ( void * ) carry0 );
 
-	bigint_multiply ( multiplicand, multiplier, result, carry );
+	bigint_multiply ( multiplicand, multiplier, result );
 }
 
 void bigint_mod_multiply_sample ( const bigint_element_t *multiplicand0,
@@ -498,14 +495,11 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	bigint_t ( multiplicand_size ) multiplicand_temp;		\
 	bigint_t ( multiplier_size ) multiplier_temp;			\
 	bigint_t ( multiplicand_size + multiplier_size ) result_temp;	\
-	bigint_t ( multiplicand_size + multiplier_size ) carry_temp;	\
 	{} /* Fix emacs alignment */					\
 									\
 	assert ( bigint_size ( &result_temp ) ==			\
 		 ( bigint_size ( &multiplicand_temp ) +			\
 		   bigint_size ( &multiplier_temp ) ) );		\
-	assert ( bigint_size ( &carry_temp ) ==				\
-		 bigint_size ( &result_temp ) );			\
 	bigint_init ( &multiplicand_temp, multiplicand_raw,		\
 		      sizeof ( multiplicand_raw ) );			\
 	bigint_init ( &multiplier_temp, multiplier_raw,			\
@@ -514,7 +508,7 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	DBG_HDA ( 0, &multiplicand_temp, sizeof ( multiplicand_temp ) );\
 	DBG_HDA ( 0, &multiplier_temp, sizeof ( multiplier_temp ) );	\
 	bigint_multiply ( &multiplicand_temp, &multiplier_temp,		\
-			  &result_temp, &carry_temp );			\
+			  &result_temp );				\
 	DBG_HDA ( 0, &result_temp, sizeof ( result_temp ) );		\
 	bigint_done ( &result_temp, result_raw, sizeof ( result_raw ) );\
 									\

From 7e0bf4ec5cb3dd608d97735575e3f62252455878 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 7 Oct 2024 12:13:42 +0100
Subject: [PATCH 208/237] [crypto] Rename bigint_rol()/bigint_ror() to
 bigint_shl()/bigint_shr()

The big integer shift operations are misleadingly described as
rotations since the original x86 implementations are essentially
trivial loops around the relevant rotate-through-carry instruction.

The overall operation performed is a shift rather than a rotation.
Update the function names and descriptions to reflect this.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/include/bits/bigint.h   |  8 ++---
 src/arch/arm64/include/bits/bigint.h   |  8 ++---
 src/arch/loong64/include/bits/bigint.h |  8 ++---
 src/arch/riscv/include/bits/bigint.h   |  8 ++---
 src/arch/x86/include/bits/bigint.h     |  8 ++---
 src/crypto/bigint.c                    | 16 ++++-----
 src/include/ipxe/bigint.h              | 16 ++++-----
 src/tests/bigint_test.c                | 48 +++++++++++++-------------
 8 files changed, 60 insertions(+), 60 deletions(-)

diff --git a/src/arch/arm32/include/bits/bigint.h b/src/arch/arm32/include/bits/bigint.h
index c148d6be2..f11ec906b 100644
--- a/src/arch/arm32/include/bits/bigint.h
+++ b/src/arch/arm32/include/bits/bigint.h
@@ -112,13 +112,13 @@ bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0,
 }
 
 /**
- * Rotate big integer left
+ * Shift big integer left
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_rol_raw ( uint32_t *value0, unsigned int size ) {
+bigint_shl_raw ( uint32_t *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	uint32_t *discard_value;
@@ -141,13 +141,13 @@ bigint_rol_raw ( uint32_t *value0, unsigned int size ) {
 }
 
 /**
- * Rotate big integer right
+ * Shift big integer right
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_ror_raw ( uint32_t *value0, unsigned int size ) {
+bigint_shr_raw ( uint32_t *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	uint32_t *discard_value;
diff --git a/src/arch/arm64/include/bits/bigint.h b/src/arch/arm64/include/bits/bigint.h
index 6b62c809e..05095413b 100644
--- a/src/arch/arm64/include/bits/bigint.h
+++ b/src/arch/arm64/include/bits/bigint.h
@@ -111,13 +111,13 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 }
 
 /**
- * Rotate big integer left
+ * Shift big integer left
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
+bigint_shl_raw ( uint64_t *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	uint64_t *discard_value;
@@ -140,13 +140,13 @@ bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
 }
 
 /**
- * Rotate big integer right
+ * Shift big integer right
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_ror_raw ( uint64_t *value0, unsigned int size ) {
+bigint_shr_raw ( uint64_t *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	uint64_t *discard_value;
diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h
index 23edaeea5..237275851 100644
--- a/src/arch/loong64/include/bits/bigint.h
+++ b/src/arch/loong64/include/bits/bigint.h
@@ -136,13 +136,13 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 }
 
 /**
- * Rotate big integer left
+ * Shift big integer left
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
+bigint_shl_raw ( uint64_t *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	uint64_t *discard_value;
@@ -177,13 +177,13 @@ bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
 }
 
 /**
- * Rotate big integer right
+ * Shift big integer right
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_ror_raw ( uint64_t *value0, unsigned int size ) {
+bigint_shr_raw ( uint64_t *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	uint64_t *discard_value;
diff --git a/src/arch/riscv/include/bits/bigint.h b/src/arch/riscv/include/bits/bigint.h
index 13fd16759..58351ac07 100644
--- a/src/arch/riscv/include/bits/bigint.h
+++ b/src/arch/riscv/include/bits/bigint.h
@@ -135,13 +135,13 @@ bigint_subtract_raw ( const unsigned long *subtrahend0, unsigned long *value0,
 }
 
 /**
- * Rotate big integer left
+ * Shift big integer left
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_rol_raw ( unsigned long *value0, unsigned int size ) {
+bigint_shl_raw ( unsigned long *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	unsigned long *valueN = ( value0 + size );
@@ -174,13 +174,13 @@ bigint_rol_raw ( unsigned long *value0, unsigned int size ) {
 }
 
 /**
- * Rotate big integer right
+ * Shift big integer right
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_ror_raw ( unsigned long *value0, unsigned int size ) {
+bigint_shr_raw ( unsigned long *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	unsigned long *valueN = ( value0 + size );
diff --git a/src/arch/x86/include/bits/bigint.h b/src/arch/x86/include/bits/bigint.h
index 320d49498..512b2724d 100644
--- a/src/arch/x86/include/bits/bigint.h
+++ b/src/arch/x86/include/bits/bigint.h
@@ -104,13 +104,13 @@ bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0,
 }
 
 /**
- * Rotate big integer left
+ * Shift big integer left
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_rol_raw ( uint32_t *value0, unsigned int size ) {
+bigint_shl_raw ( uint32_t *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	long index;
@@ -127,13 +127,13 @@ bigint_rol_raw ( uint32_t *value0, unsigned int size ) {
 }
 
 /**
- * Rotate big integer right
+ * Shift big integer right
  *
  * @v value0		Element 0 of big integer
  * @v size		Number of elements
  */
 static inline __attribute__ (( always_inline )) void
-bigint_ror_raw ( uint32_t *value0, unsigned int size ) {
+bigint_shr_raw ( uint32_t *value0, unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
 		( ( void * ) value0 );
 	long discard_c;
diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index b63f7ccc1..c7b6dafc9 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -171,7 +171,7 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 		bigint_t ( size * 2 ) result;
 		bigint_t ( size * 2 ) modulus;
 	} *temp = tmp;
-	int rotation;
+	int shift;
 	int i;
 
 	/* Start profiling */
@@ -188,18 +188,18 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 	/* Rescale modulus to match result */
 	profile_start ( &bigint_mod_multiply_rescale_profiler );
 	bigint_grow ( modulus, &temp->modulus );
-	rotation = ( bigint_max_set_bit ( &temp->result ) -
-		     bigint_max_set_bit ( &temp->modulus ) );
-	for ( i = 0 ; i < rotation ; i++ )
-		bigint_rol ( &temp->modulus );
+	shift = ( bigint_max_set_bit ( &temp->result ) -
+		  bigint_max_set_bit ( &temp->modulus ) );
+	for ( i = 0 ; i < shift ; i++ )
+		bigint_shl ( &temp->modulus );
 	profile_stop ( &bigint_mod_multiply_rescale_profiler );
 
 	/* Subtract multiples of modulus */
 	profile_start ( &bigint_mod_multiply_subtract_profiler );
-	for ( i = 0 ; i <= rotation ; i++ ) {
+	for ( i = 0 ; i <= shift ; i++ ) {
 		if ( bigint_is_geq ( &temp->result, &temp->modulus ) )
 			bigint_subtract ( &temp->modulus, &temp->result );
-		bigint_ror ( &temp->modulus );
+		bigint_shr ( &temp->modulus );
 	}
 	profile_stop ( &bigint_mod_multiply_subtract_profiler );
 
@@ -255,7 +255,7 @@ void bigint_mod_exp_raw ( const bigint_element_t *base0,
 			bigint_mod_multiply ( result, &temp->base, modulus,
 					      result, temp->mod_multiply );
 		}
-		bigint_ror ( &temp->exponent );
+		bigint_shr ( &temp->exponent );
 		bigint_mod_multiply ( &temp->base, &temp->base, modulus,
 				      &temp->base, temp->mod_multiply );
 	}
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index bcb7af5ec..19cd4e3dd 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -89,23 +89,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	} while ( 0 )
 
 /**
- * Rotate big integer left
+ * Shift big integer left
  *
  * @v value		Big integer
  */
-#define bigint_rol( value ) do {					\
+#define bigint_shl( value ) do {					\
 	unsigned int size = bigint_size (value);			\
-	bigint_rol_raw ( (value)->element, size );			\
+	bigint_shl_raw ( (value)->element, size );			\
 	} while ( 0 )
 
 /**
- * Rotate big integer right
+ * Shift big integer right
  *
  * @v value		Big integer
  */
-#define bigint_ror( value ) do {					\
+#define bigint_shr( value ) do {					\
 	unsigned int size = bigint_size (value);			\
-	bigint_ror_raw ( (value)->element, size );			\
+	bigint_shr_raw ( (value)->element, size );			\
 	} while ( 0 )
 
 /**
@@ -293,8 +293,8 @@ void bigint_add_raw ( const bigint_element_t *addend0,
 		      bigint_element_t *value0, unsigned int size );
 void bigint_subtract_raw ( const bigint_element_t *subtrahend0,
 			   bigint_element_t *value0, unsigned int size );
-void bigint_rol_raw ( bigint_element_t *value0, unsigned int size );
-void bigint_ror_raw ( bigint_element_t *value0, unsigned int size );
+void bigint_shl_raw ( bigint_element_t *value0, unsigned int size );
+void bigint_shr_raw ( bigint_element_t *value0, unsigned int size );
 int bigint_is_zero_raw ( const bigint_element_t *value0, unsigned int size );
 int bigint_is_geq_raw ( const bigint_element_t *value0,
 			const bigint_element_t *reference0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 76aca1059..65f124f24 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -78,18 +78,18 @@ void bigint_subtract_sample ( const bigint_element_t *subtrahend0,
 	bigint_subtract ( subtrahend, value );
 }
 
-void bigint_rol_sample ( bigint_element_t *value0, unsigned int size ) {
+void bigint_shl_sample ( bigint_element_t *value0, unsigned int size ) {
 	bigint_t ( size ) *value __attribute__ (( may_alias ))
 		= ( ( void * ) value0 );
 
-	bigint_rol ( value );
+	bigint_shl ( value );
 }
 
-void bigint_ror_sample ( bigint_element_t *value0, unsigned int size ) {
+void bigint_shr_sample ( bigint_element_t *value0, unsigned int size ) {
 	bigint_t ( size ) *value __attribute__ (( may_alias ))
 		= ( ( void * ) value0 );
 
-	bigint_ror ( value );
+	bigint_shr ( value );
 }
 
 int bigint_is_zero_sample ( const bigint_element_t *value0,
@@ -290,12 +290,12 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	} while ( 0 )
 
 /**
- * Report result of big integer left rotation test
+ * Report result of big integer left shift test
  *
  * @v value		Big integer
  * @v expected		Big integer expected result
  */
-#define bigint_rol_ok( value, expected ) do {				\
+#define bigint_shl_ok( value, expected ) do {				\
 	static const uint8_t value_raw[] = value;			\
 	static const uint8_t expected_raw[] = expected;			\
 	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
@@ -305,9 +305,9 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	{} /* Fix emacs alignment */					\
 									\
 	bigint_init ( &value_temp, value_raw, sizeof ( value_raw ) );	\
-	DBG ( "Rotate left:\n" );					\
+	DBG ( "Shift left:\n" );					\
 	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
-	bigint_rol ( &value_temp );					\
+	bigint_shl ( &value_temp );					\
 	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
 	bigint_done ( &value_temp, result_raw, sizeof ( result_raw ) );	\
 									\
@@ -316,12 +316,12 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	} while ( 0 )
 
 /**
- * Report result of big integer right rotation test
+ * Report result of big integer right shift test
  *
  * @v value		Big integer
  * @v expected		Big integer expected result
  */
-#define bigint_ror_ok( value, expected ) do {				\
+#define bigint_shr_ok( value, expected ) do {				\
 	static const uint8_t value_raw[] = value;			\
 	static const uint8_t expected_raw[] = expected;			\
 	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
@@ -331,9 +331,9 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	{} /* Fix emacs alignment */					\
 									\
 	bigint_init ( &value_temp, value_raw, sizeof ( value_raw ) );	\
-	DBG ( "Rotate right:\n" );					\
+	DBG ( "Shift right:\n" );					\
 	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
-	bigint_ror ( &value_temp );					\
+	bigint_shr ( &value_temp );					\
 	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
 	bigint_done ( &value_temp, result_raw, sizeof ( result_raw ) );	\
 									\
@@ -801,15 +801,15 @@ static void bigint_test_exec ( void ) {
 				      0xda, 0xc8, 0x8c, 0x71, 0x86, 0x97,
 				      0x7f, 0xcb, 0x94, 0x31, 0x1d, 0xbc,
 				      0x44, 0x1a ) );
-	bigint_rol_ok ( BIGINT ( 0xe0 ),
+	bigint_shl_ok ( BIGINT ( 0xe0 ),
 			BIGINT ( 0xc0 ) );
-	bigint_rol_ok ( BIGINT ( 0x43, 0x1d ),
+	bigint_shl_ok ( BIGINT ( 0x43, 0x1d ),
 			BIGINT ( 0x86, 0x3a ) );
-	bigint_rol_ok ( BIGINT ( 0xac, 0xed, 0x9b ),
+	bigint_shl_ok ( BIGINT ( 0xac, 0xed, 0x9b ),
 			BIGINT ( 0x59, 0xdb, 0x36 ) );
-	bigint_rol_ok ( BIGINT ( 0x2c, 0xe8, 0x3a, 0x22 ),
+	bigint_shl_ok ( BIGINT ( 0x2c, 0xe8, 0x3a, 0x22 ),
 			BIGINT ( 0x59, 0xd0, 0x74, 0x44 ) );
-	bigint_rol_ok ( BIGINT ( 0x4e, 0x88, 0x4a, 0x05, 0x5e, 0x10, 0xee,
+	bigint_shl_ok ( BIGINT ( 0x4e, 0x88, 0x4a, 0x05, 0x5e, 0x10, 0xee,
 				 0x5b, 0xc6, 0x40, 0x0e, 0x03, 0xd7, 0x0d,
 				 0x28, 0xa5, 0x55, 0xb2, 0x50, 0xef, 0x69,
 				 0xd1, 0x1d ),
@@ -817,7 +817,7 @@ static void bigint_test_exec ( void ) {
 				 0xb7, 0x8c, 0x80, 0x1c, 0x07, 0xae, 0x1a,
 				 0x51, 0x4a, 0xab, 0x64, 0xa1, 0xde, 0xd3,
 				 0xa2, 0x3a ) );
-	bigint_rol_ok ( BIGINT ( 0x07, 0x62, 0x78, 0x70, 0x2e, 0xd4, 0x41,
+	bigint_shl_ok ( BIGINT ( 0x07, 0x62, 0x78, 0x70, 0x2e, 0xd4, 0x41,
 				 0xaa, 0x9b, 0x50, 0xb1, 0x9a, 0x71, 0xf5,
 				 0x1c, 0x2f, 0xe7, 0x0d, 0xf1, 0x46, 0x57,
 				 0x04, 0x99, 0x78, 0x4e, 0x84, 0x78, 0xba,
@@ -855,15 +855,15 @@ static void bigint_test_exec ( void ) {
 				 0x49, 0x7c, 0x1e, 0xdb, 0xc7, 0x65, 0xa6,
 				 0x0e, 0xd1, 0xd2, 0x00, 0xb3, 0x41, 0xc9,
 				 0x3c, 0xbc ) );
-	bigint_ror_ok ( BIGINT ( 0x8f ),
+	bigint_shr_ok ( BIGINT ( 0x8f ),
 			BIGINT ( 0x47 ) );
-	bigint_ror_ok ( BIGINT ( 0xaa, 0x1d ),
+	bigint_shr_ok ( BIGINT ( 0xaa, 0x1d ),
 			BIGINT ( 0x55, 0x0e ) );
-	bigint_ror_ok ( BIGINT ( 0xf0, 0xbd, 0x68 ),
+	bigint_shr_ok ( BIGINT ( 0xf0, 0xbd, 0x68 ),
 			BIGINT ( 0x78, 0x5e, 0xb4 ) );
-	bigint_ror_ok ( BIGINT ( 0x33, 0xa0, 0x3d, 0x95 ),
+	bigint_shr_ok ( BIGINT ( 0x33, 0xa0, 0x3d, 0x95 ),
 			BIGINT ( 0x19, 0xd0, 0x1e, 0xca ) );
-	bigint_ror_ok ( BIGINT ( 0xa1, 0xf4, 0xb9, 0x64, 0x91, 0x99, 0xa1,
+	bigint_shr_ok ( BIGINT ( 0xa1, 0xf4, 0xb9, 0x64, 0x91, 0x99, 0xa1,
 				 0xf4, 0xae, 0xeb, 0x71, 0x97, 0x1b, 0x71,
 				 0x09, 0x38, 0x3f, 0x8f, 0xc5, 0x3a, 0xb9,
 				 0x75, 0x94 ),
@@ -871,7 +871,7 @@ static void bigint_test_exec ( void ) {
 				 0xfa, 0x57, 0x75, 0xb8, 0xcb, 0x8d, 0xb8,
 				 0x84, 0x9c, 0x1f, 0xc7, 0xe2, 0x9d, 0x5c,
 				 0xba, 0xca ) );
-	bigint_ror_ok ( BIGINT ( 0xc0, 0xb3, 0x78, 0x46, 0x69, 0x6e, 0x35,
+	bigint_shr_ok ( BIGINT ( 0xc0, 0xb3, 0x78, 0x46, 0x69, 0x6e, 0x35,
 				 0x94, 0xed, 0x28, 0xdc, 0xfd, 0xf6, 0xdb,
 				 0x2d, 0x24, 0xcb, 0xa4, 0x6f, 0x0e, 0x58,
 				 0x89, 0x04, 0xec, 0xc8, 0x0c, 0x2d, 0xb3,

From f78c5a763cc7bb2e2b7b437e7cc74a3efb876960 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 8 Oct 2024 11:52:07 +0100
Subject: [PATCH 209/237] [crypto] Use architecture-independent bigint_is_set()

Every architecture uses the same implementation for bigint_is_set(),
and there is no reason to suspect that a future CPU architecture will
provide a more efficient way to implement this operation.

Simplify the code by providing a single architecture-independent
implementation of bigint_is_set().

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/include/bits/bigint.h   | 19 -------------------
 src/arch/arm64/include/bits/bigint.h   | 19 -------------------
 src/arch/loong64/include/bits/bigint.h | 19 -------------------
 src/arch/riscv/include/bits/bigint.h   | 19 -------------------
 src/arch/x86/include/bits/bigint.h     | 19 -------------------
 src/include/ipxe/bigint.h              | 19 +++++++++++++++++++
 6 files changed, 19 insertions(+), 95 deletions(-)

diff --git a/src/arch/arm32/include/bits/bigint.h b/src/arch/arm32/include/bits/bigint.h
index f11ec906b..39d3dc347 100644
--- a/src/arch/arm32/include/bits/bigint.h
+++ b/src/arch/arm32/include/bits/bigint.h
@@ -216,25 +216,6 @@ bigint_is_geq_raw ( const uint32_t *value0, const uint32_t *reference0,
 	return ( value_i >= reference_i );
 }
 
-/**
- * Test if bit is set in big integer
- *
- * @v value0		Element 0 of big integer
- * @v size		Number of elements
- * @v bit		Bit to test
- * @ret is_set		Bit is set
- */
-static inline __attribute__ (( always_inline )) int
-bigint_bit_is_set_raw ( const uint32_t *value0, unsigned int size,
-			unsigned int bit ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
-		( ( const void * ) value0 );
-	unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) );
-	unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) );
-
-	return ( value->element[index] & ( 1 << subindex ) );
-}
-
 /**
  * Find highest bit set in big integer
  *
diff --git a/src/arch/arm64/include/bits/bigint.h b/src/arch/arm64/include/bits/bigint.h
index 05095413b..50493499e 100644
--- a/src/arch/arm64/include/bits/bigint.h
+++ b/src/arch/arm64/include/bits/bigint.h
@@ -217,25 +217,6 @@ bigint_is_geq_raw ( const uint64_t *value0, const uint64_t *reference0,
 	return ( value_i >= reference_i );
 }
 
-/**
- * Test if bit is set in big integer
- *
- * @v value0		Element 0 of big integer
- * @v size		Number of elements
- * @v bit		Bit to test
- * @ret is_set		Bit is set
- */
-static inline __attribute__ (( always_inline )) int
-bigint_bit_is_set_raw ( const uint64_t *value0, unsigned int size,
-			unsigned int bit ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
-		( ( const void * ) value0 );
-	unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) );
-	unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) );
-
-	return ( !! ( value->element[index] & ( 1UL << subindex ) ) );
-}
-
 /**
  * Find highest bit set in big integer
  *
diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h
index 237275851..234d8dfa7 100644
--- a/src/arch/loong64/include/bits/bigint.h
+++ b/src/arch/loong64/include/bits/bigint.h
@@ -264,25 +264,6 @@ bigint_is_geq_raw ( const uint64_t *value0, const uint64_t *reference0,
 	return ( value_i >= reference_i );
 }
 
-/**
- * Test if bit is set in big integer
- *
- * @v value0		Element 0 of big integer
- * @v size		Number of elements
- * @v bit		Bit to test
- * @ret is_set		Bit is set
- */
-static inline __attribute__ (( always_inline )) int
-bigint_bit_is_set_raw ( const uint64_t *value0, unsigned int size,
-			unsigned int bit ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
-		( ( const void * ) value0 );
-	unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) );
-	unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) );
-
-	return ( !! ( value->element[index] & ( 1UL << subindex ) ) );
-}
-
 /**
  * Find highest bit set in big integer
  *
diff --git a/src/arch/riscv/include/bits/bigint.h b/src/arch/riscv/include/bits/bigint.h
index 58351ac07..5b22b9ac7 100644
--- a/src/arch/riscv/include/bits/bigint.h
+++ b/src/arch/riscv/include/bits/bigint.h
@@ -259,25 +259,6 @@ bigint_is_geq_raw ( const unsigned long *value0,
 	return ( value_i >= reference_i );
 }
 
-/**
- * Test if bit is set in big integer
- *
- * @v value0		Element 0 of big integer
- * @v size		Number of elements
- * @v bit		Bit to test
- * @ret is_set		Bit is set
- */
-static inline __attribute__ (( always_inline )) int
-bigint_bit_is_set_raw ( const unsigned long *value0, unsigned int size,
-			unsigned int bit ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
-		( ( const void * ) value0 );
-	unsigned int index = ( bit / ( 8 * sizeof ( *value0 ) ) );
-	unsigned int subindex = ( bit % ( 8 * sizeof ( *value0 ) ) );
-
-	return ( !! ( value->element[index] & ( 1UL << subindex ) ) );
-}
-
 /**
  * Find highest bit set in big integer
  *
diff --git a/src/arch/x86/include/bits/bigint.h b/src/arch/x86/include/bits/bigint.h
index 512b2724d..5580030ee 100644
--- a/src/arch/x86/include/bits/bigint.h
+++ b/src/arch/x86/include/bits/bigint.h
@@ -195,25 +195,6 @@ bigint_is_geq_raw ( const uint32_t *value0, const uint32_t *reference0,
 	return result;
 }
 
-/**
- * Test if bit is set in big integer
- *
- * @v value0		Element 0 of big integer
- * @v size		Number of elements
- * @v bit		Bit to test
- * @ret is_set		Bit is set
- */
-static inline __attribute__ (( always_inline )) int
-bigint_bit_is_set_raw ( const uint32_t *value0, unsigned int size,
-			unsigned int bit ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
-		( ( const void * ) value0 );
-	unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) );
-	unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) );
-
-	return ( value->element[index] & ( 1 << subindex ) );
-}
-
 /**
  * Find highest bit set in big integer
  *
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 19cd4e3dd..c556afbc1 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -285,6 +285,25 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <bits/bigint.h>
 
+/**
+ * Test if bit is set in big integer
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ * @v bit		Bit to test
+ * @ret is_set		Bit is set
+ */
+static inline __attribute__ (( always_inline )) int
+bigint_bit_is_set_raw ( const bigint_element_t *value0, unsigned int size,
+			unsigned int bit ) {
+	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( const void * ) value0 );
+	unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) );
+	unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) );
+
+	return ( !! ( value->element[index] & ( 1UL << subindex ) ) );
+}
+
 void bigint_init_raw ( bigint_element_t *value0, unsigned int size,
 		       const void *data, size_t len );
 void bigint_done_raw ( const bigint_element_t *value0, unsigned int size,

From 2bf16c6ffca1e294bb8233d19c9c36e43b31f041 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 15 Oct 2024 13:50:51 +0100
Subject: [PATCH 210/237] [crypto] Separate out bigint_reduce() from
 bigint_mod_multiply()

Faster modular multiplication algorithms such as Montgomery
multiplication will still require the ability to perform a single
direct modular reduction.

Neaten up the implementation of direct reduction and split it out into
a separate bigint_reduce() function, complete with its own unit tests.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       | 213 +++++++++++++++++++++++++++++++-------
 src/include/ipxe/bigint.h |  34 ++++++
 src/tests/bigint_test.c   |  86 +++++++++++++++
 3 files changed, 296 insertions(+), 37 deletions(-)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index c7b6dafc9..a8b99ec3c 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -34,22 +34,14 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Big integer support
  */
 
+/** Modular direct reduction profiler */
+static struct profiler bigint_mod_profiler __profiler =
+	{ .name = "bigint_mod" };
+
 /** Modular multiplication overall profiler */
 static struct profiler bigint_mod_multiply_profiler __profiler =
 	{ .name = "bigint_mod_multiply" };
 
-/** Modular multiplication multiply step profiler */
-static struct profiler bigint_mod_multiply_multiply_profiler __profiler =
-	{ .name = "bigint_mod_multiply.multiply" };
-
-/** Modular multiplication rescale step profiler */
-static struct profiler bigint_mod_multiply_rescale_profiler __profiler =
-	{ .name = "bigint_mod_multiply.rescale" };
-
-/** Modular multiplication subtract step profiler */
-static struct profiler bigint_mod_multiply_subtract_profiler __profiler =
-	{ .name = "bigint_mod_multiply.subtract" };
-
 /**
  * Conditionally swap big integers (in constant time)
  *
@@ -144,6 +136,175 @@ void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 	}
 }
 
+/**
+ * Reduce big integer
+ *
+ * @v minuend0		Element 0 of big integer to be reduced
+ * @v minuend_size	Number of elements in minuend
+ * @v modulus0		Element 0 of big integer modulus
+ * @v modulus_size	Number of elements in modulus and result
+ * @v result0		Element 0 of big integer to hold result
+ * @v tmp		Temporary working space
+ */
+void bigint_reduce_raw ( const bigint_element_t *minuend0,
+			 unsigned int minuend_size,
+			 const bigint_element_t *modulus0,
+			 unsigned int modulus_size,
+			 bigint_element_t *result0, void *tmp ) {
+	const bigint_t ( minuend_size ) __attribute__ (( may_alias ))
+		*minuend = ( ( const void * ) minuend0 );
+	const bigint_t ( modulus_size ) __attribute__ (( may_alias ))
+		*modulus = ( ( const void * ) modulus0 );
+	bigint_t ( modulus_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
+	struct {
+		bigint_t ( minuend_size ) minuend;
+		bigint_t ( minuend_size ) modulus;
+	} *temp = tmp;
+	const unsigned int width = ( 8 * sizeof ( bigint_element_t ) );
+	const bigint_element_t msb_mask = ( 1UL << ( width - 1 ) );
+	bigint_element_t *element;
+	unsigned int minuend_max;
+	unsigned int modulus_max;
+	unsigned int subshift;
+	bigint_element_t msb;
+	int offset;
+	int shift;
+	int i;
+
+	/* Start profiling */
+	profile_start ( &bigint_mod_profiler );
+
+	/* Sanity check */
+	assert ( minuend_size >= modulus_size );
+	assert ( sizeof ( *temp ) == bigint_reduce_tmp_len ( minuend ) );
+
+	/* Copy minuend and modulus to temporary working space */
+	bigint_shrink ( minuend, &temp->minuend );
+	bigint_grow ( modulus, &temp->modulus );
+
+	/* Normalise the modulus
+	 *
+	 * Scale the modulus by shifting left such that both modulus
+	 * "m" and minuend "x" have the same most significant set bit.
+	 * (If this is not possible, then the minuend is already less
+	 * than the modulus, and we may therefore skip reduction
+	 * completely.)
+	 */
+	minuend_max = bigint_max_set_bit ( minuend );
+	modulus_max = bigint_max_set_bit ( modulus );
+	shift = ( minuend_max - modulus_max );
+	if ( shift < 0 )
+		goto skip;
+	subshift = ( shift & ( width - 1 ) );
+	offset = ( shift / width );
+	element = temp->modulus.element;
+	for ( i = ( ( minuend_max - 1 ) / width ) ; ; i-- ) {
+		element[i] = ( element[ i - offset ] << subshift );
+		if ( i <= offset )
+			break;
+		if ( subshift ) {
+			element[i] |= ( element[ i - offset - 1 ]
+					>> ( width - subshift ) );
+		}
+	}
+	for ( i-- ; i >= 0 ; i-- )
+		element[i] = 0;
+
+	/* Reduce the minuend "x" by iteratively adding or subtracting
+	 * the scaled modulus "m".
+	 *
+	 * On each loop iteration, we maintain the invariant:
+	 *
+	 *    -2m <= x < 2m
+	 *
+	 * If x is positive, we obtain the new minuend x' by
+	 * subtracting m, otherwise we add m:
+	 *
+	 *      0 <= x < 2m   =>   x' := x - m   =>   -m <= x' < m
+	 *    -2m <= x < 0    =>   x' := x + m   =>   -m <= x' < m
+	 *
+	 * and then halve the modulus (by shifting right):
+	 *
+	 *      m' = m/2
+	 *
+	 * We therefore end up with:
+	 *
+	 *     -m <= x' < m   =>   -2m' <= x' < 2m'
+	 *
+	 * i.e. we have preseved the invariant while reducing the
+	 * bounds on x' by one power of two.
+	 *
+	 * The issue remains of how to determine on each iteration
+	 * whether or not x is currently positive, given that both
+	 * input values are unsigned big integers that may use all
+	 * available bits (including the MSB).
+	 *
+	 * On the first loop iteration, we may simply assume that x is
+	 * positive, since it is unmodified from the input value and
+	 * so is positive by definition (even if the MSB is set).  We
+	 * therefore unconditionally perform a subtraction on the
+	 * first loop iteration.
+	 *
+	 * Let k be the MSB after normalisation.  We then have:
+	 *
+	 *    2^k <= m < 2^(k+1)
+	 *    2^k <= x < 2^(k+1)
+	 *
+	 * On the first loop iteration, we therefore have:
+	 *
+	 *     x' = (x - m)
+	 *        < 2^(k+1) - 2^k
+	 *        < 2^k
+	 *
+	 * Any positive value of x' therefore has its MSB set to zero,
+	 * and so we may validly treat the MSB of x' as a sign bit at
+	 * the end of the first loop iteration.
+	 *
+	 * On all subsequent loop iterations, the starting value m is
+	 * guaranteed to have its MSB set to zero (since it has
+	 * already been shifted right at least once).  Since we know
+	 * from above that we preserve the loop invariant:
+	 *
+	 *     -m <= x' < m
+	 *
+	 * we immediately know that any positive value of x' also has
+	 * its MSB set to zero, and so we may validly treat the MSB of
+	 * x' as a sign bit at the end of all subsequent loop
+	 * iterations.
+	 *
+	 * After the last loop iteration (when m' has been shifted
+	 * back down to the original value of the modulus), we may
+	 * need to add a single multiple of m' to ensure that x' is
+	 * positive, i.e. lies within the range 0 <= x' < m'.  To
+	 * allow for reusing the (inlined) expansion of
+	 * bigint_subtract(), we achieve this via a potential
+	 * additional loop iteration that performs the addition and is
+	 * then guaranteed to terminate (since the result will be
+	 * positive).
+	 */
+	for ( msb = 0 ; ( msb || ( shift >= 0 ) ) ; shift-- ) {
+		if ( msb ) {
+			bigint_add ( &temp->modulus, &temp->minuend );
+		} else {
+			bigint_subtract ( &temp->modulus, &temp->minuend );
+		}
+		msb = ( temp->minuend.element[ minuend_size - 1 ] & msb_mask );
+		if ( shift > 0 )
+			bigint_shr ( &temp->modulus );
+	}
+
+ skip:
+	/* Sanity check */
+	assert ( ! bigint_is_geq ( &temp->minuend, &temp->modulus ) );
+
+	/* Copy result */
+	bigint_shrink ( &temp->minuend, result );
+
+	/* Stop profiling */
+	profile_stop ( &bigint_mod_profiler );
+}
+
 /**
  * Perform modular multiplication of big integers
  *
@@ -171,8 +332,6 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 		bigint_t ( size * 2 ) result;
 		bigint_t ( size * 2 ) modulus;
 	} *temp = tmp;
-	int shift;
-	int i;
 
 	/* Start profiling */
 	profile_start ( &bigint_mod_multiply_profiler );
@@ -181,33 +340,13 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 	assert ( sizeof ( *temp ) == bigint_mod_multiply_tmp_len ( modulus ) );
 
 	/* Perform multiplication */
-	profile_start ( &bigint_mod_multiply_multiply_profiler );
 	bigint_multiply ( multiplicand, multiplier, &temp->result );
-	profile_stop ( &bigint_mod_multiply_multiply_profiler );
 
-	/* Rescale modulus to match result */
-	profile_start ( &bigint_mod_multiply_rescale_profiler );
-	bigint_grow ( modulus, &temp->modulus );
-	shift = ( bigint_max_set_bit ( &temp->result ) -
-		  bigint_max_set_bit ( &temp->modulus ) );
-	for ( i = 0 ; i < shift ; i++ )
-		bigint_shl ( &temp->modulus );
-	profile_stop ( &bigint_mod_multiply_rescale_profiler );
-
-	/* Subtract multiples of modulus */
-	profile_start ( &bigint_mod_multiply_subtract_profiler );
-	for ( i = 0 ; i <= shift ; i++ ) {
-		if ( bigint_is_geq ( &temp->result, &temp->modulus ) )
-			bigint_subtract ( &temp->modulus, &temp->result );
-		bigint_shr ( &temp->modulus );
-	}
-	profile_stop ( &bigint_mod_multiply_subtract_profiler );
-
-	/* Resize result */
-	bigint_shrink ( &temp->result, result );
+	/* Reduce result */
+	bigint_reduce ( &temp->result, modulus, result, temp );
 
 	/* Sanity check */
-	assert ( bigint_is_geq ( modulus, result ) );
+	assert ( ! bigint_is_geq ( result, modulus ) );
 
 	/* Stop profiling */
 	profile_stop ( &bigint_mod_multiply_profiler );
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index c556afbc1..c56b2155f 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -217,6 +217,35 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 			      multiplier_size, (result)->element );	\
 	} while ( 0 )
 
+/**
+ * Reduce big integer
+ *
+ * @v minuend		Big integer to be reduced
+ * @v modulus		Big integer modulus
+ * @v result		Big integer to hold result
+ * @v tmp		Temporary working space
+ */
+#define bigint_reduce( minuend, modulus, result, tmp ) do {		\
+	unsigned int minuend_size = bigint_size (minuend);		\
+	unsigned int modulus_size = bigint_size (modulus);		\
+	bigint_reduce_raw ( (minuend)->element, minuend_size,		\
+			    (modulus)->element, modulus_size,		\
+			    (result)->element, tmp );			\
+	} while ( 0 )
+
+/**
+ * Calculate temporary working space required for reduction
+ *
+ * @v minuend		Big integer to be reduced
+ * @ret len		Length of temporary working space
+ */
+#define bigint_reduce_tmp_len( minuend ) ( {				\
+	unsigned int size = bigint_size (minuend);			\
+	sizeof ( struct {						\
+		bigint_t ( size ) temp_minuend;				\
+		bigint_t ( size ) temp_modulus;				\
+	} ); } )
+
 /**
  * Perform modular multiplication of big integers
  *
@@ -339,6 +368,11 @@ void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 			   const bigint_element_t *multiplier0,
 			   unsigned int multiplier_size,
 			   bigint_element_t *result0 );
+void bigint_reduce_raw ( const bigint_element_t *minuend0,
+			 unsigned int minuend_size,
+			 const bigint_element_t *modulus0,
+			 unsigned int modulus_size,
+			 bigint_element_t *result0, void *tmp );
 void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 			       const bigint_element_t *multiplier0,
 			       const bigint_element_t *modulus0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 65f124f24..104e1f362 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -185,6 +185,21 @@ void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
 	bigint_multiply ( multiplicand, multiplier, result );
 }
 
+void bigint_reduce_sample ( const bigint_element_t *minuend0,
+			    unsigned int minuend_size,
+			    const bigint_element_t *modulus0,
+			    unsigned int modulus_size,
+			    bigint_element_t *result0, void *tmp ) {
+	const bigint_t ( minuend_size ) __attribute__ (( may_alias ))
+		*minuend = ( ( const void * ) minuend0 );
+	const bigint_t ( modulus_size ) __attribute__ (( may_alias ))
+		*modulus = ( ( const void * ) modulus0 );
+	bigint_t ( modulus_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
+
+	bigint_reduce ( minuend, modulus, result, tmp );
+}
+
 void bigint_mod_multiply_sample ( const bigint_element_t *multiplicand0,
 				  const bigint_element_t *multiplier0,
 				  const bigint_element_t *modulus0,
@@ -516,6 +531,48 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 		      sizeof ( result_raw ) ) == 0 );			\
 	} while ( 0 )
 
+/**
+ * Report result of big integer modular direct reduction test
+ *
+ * @v minuend		Big integer to be reduced
+ * @v modulus		Big integer modulus
+ * @v expected		Big integer expected result
+ */
+#define bigint_reduce_ok( minuend, modulus, expected ) do {		\
+	static const uint8_t minuend_raw[] = minuend;			\
+	static const uint8_t modulus_raw[] = modulus;			\
+	static const uint8_t expected_raw[] = expected;			\
+	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
+	unsigned int minuend_size =					\
+		bigint_required_size ( sizeof ( minuend_raw ) );	\
+	unsigned int modulus_size =					\
+		bigint_required_size ( sizeof ( modulus_raw ) );	\
+	bigint_t ( minuend_size ) minuend_temp;				\
+	bigint_t ( modulus_size ) modulus_temp;				\
+	bigint_t ( modulus_size ) result_temp;				\
+	size_t tmp_len = bigint_reduce_tmp_len ( &minuend_temp );	\
+	uint8_t tmp[tmp_len];						\
+	{} /* Fix emacs alignment */					\
+									\
+	assert ( bigint_size ( &result_temp ) ==			\
+		 bigint_size ( &modulus_temp ) );			\
+	bigint_init ( &minuend_temp, minuend_raw,			\
+		      sizeof ( minuend_raw ) );				\
+	bigint_init ( &modulus_temp, modulus_raw,			\
+		      sizeof ( modulus_raw ) );				\
+	DBG ( "Modular reduce:\n" );					\
+	DBG_HDA ( 0, &minuend_temp, sizeof ( minuend_temp ) );		\
+	DBG_HDA ( 0, &modulus_temp, sizeof ( modulus_temp ) );		\
+	bigint_reduce ( &minuend_temp, &modulus_temp, &result_temp,	\
+			tmp );						\
+	DBG_HDA ( 0, &result_temp, sizeof ( result_temp ) );		\
+	bigint_done ( &result_temp, result_raw,				\
+		      sizeof ( result_raw ) );				\
+									\
+	ok ( memcmp ( result_raw, expected_raw,				\
+		      sizeof ( result_raw ) ) == 0 );			\
+	} while ( 0 )
+
 /**
  * Report result of big integer modular multiplication test
  *
@@ -1674,6 +1731,35 @@ static void bigint_test_exec ( void ) {
 				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 				      0x00, 0x00, 0x00, 0x01 ) );
+	bigint_reduce_ok ( BIGINT ( 0x00 ),
+			   BIGINT ( 0xaf ),
+			   BIGINT ( 0x00 ) );
+	bigint_reduce_ok ( BIGINT ( 0xab ),
+			   BIGINT ( 0xab ),
+			   BIGINT ( 0x00 ) );
+	bigint_reduce_ok ( BIGINT ( 0x1d, 0x97, 0x63, 0xc9, 0x97, 0xcd, 0x43,
+				    0xcb, 0x8e, 0x71, 0xac, 0x41, 0xdd ),
+			   BIGINT ( 0xcc, 0x9d, 0xa0, 0x79, 0x96, 0x6a, 0x46,
+				    0xd5, 0xb4, 0x30, 0xd2, 0x2b, 0xbf ),
+			   BIGINT ( 0x1d, 0x97, 0x63, 0xc9, 0x97, 0xcd, 0x43,
+				    0xcb, 0x8e, 0x71, 0xac, 0x41, 0xdd ) );
+	bigint_reduce_ok ( BIGINT ( 0x21, 0xfa, 0x4f, 0xce, 0x0f, 0x0f, 0x4d,
+				    0x43, 0xaa, 0xad, 0x21, 0x30, 0xe5 ),
+			   BIGINT ( 0x21, 0xfa, 0x4f, 0xce, 0x0f, 0x0f, 0x4d,
+				    0x43, 0xaa, 0xad, 0x21, 0x30, 0xe5 ),
+			   BIGINT ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				    0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ) );
+	bigint_reduce_ok ( BIGINT ( 0xf9, 0x78, 0x96, 0x39, 0xee, 0x98, 0x42,
+				    0x6a, 0xb8, 0x74, 0x0b, 0xe8, 0x5c, 0x76,
+				    0x34, 0xaf ),
+			   BIGINT ( 0xf3, 0x65, 0x35, 0x41, 0x66, 0x65 ),
+			   BIGINT ( 0xb3, 0x07, 0xe8, 0xb7, 0x01, 0xf6 ) );
+	bigint_reduce_ok ( BIGINT ( 0xfe, 0x30, 0xe1, 0xc6, 0x65, 0x97, 0x48,
+				    0x2e, 0x94, 0xd4 ),
+			   BIGINT ( 0x47, 0xaa, 0x88, 0x00, 0xd0, 0x30, 0x62,
+				    0xfb, 0x5d, 0x55 ),
+			   BIGINT ( 0x27, 0x31, 0x49, 0xc3, 0xf5, 0x06, 0x1f,
+				    0x3c, 0x7c, 0xd5 ) );
 	bigint_mod_multiply_ok ( BIGINT ( 0x37 ),
 				 BIGINT ( 0x67 ),
 				 BIGINT ( 0x3f ),

From c219b5d8a9536cc87e9c53e8389e59c706ba3f8e Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 17 Oct 2024 14:05:25 +0100
Subject: [PATCH 211/237] [usb] Add "usbscan" command for iterating over USB
 devices

Implement a "usbscan" command as a direct analogy of the existing
"pciscan" command, allowing scripts to iterate over all detected USB
devices.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config.c        |   3 +
 src/config/general.h       |   1 +
 src/drivers/bus/usb.c      |  75 ++++++++++++++++++++++-
 src/hci/commands/usb_cmd.c | 122 +++++++++++++++++++++++++++++++++++++
 src/include/ipxe/errfile.h |   1 +
 src/include/ipxe/usb.h     |  25 ++++++++
 6 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 src/hci/commands/usb_cmd.c

diff --git a/src/config/config.c b/src/config/config.c
index f532c1d2d..0f950eb9d 100644
--- a/src/config/config.c
+++ b/src/config/config.c
@@ -302,6 +302,9 @@ REQUIRE_OBJECT ( shim_cmd );
 #ifdef IMAGE_CRYPT_CMD
 REQUIRE_OBJECT ( image_crypt_cmd );
 #endif
+#ifdef USB_CMD
+REQUIRE_OBJECT ( usb_cmd );
+#endif
 
 /*
  * Drag in miscellaneous objects
diff --git a/src/config/general.h b/src/config/general.h
index b82e5602c..763a34aa0 100644
--- a/src/config/general.h
+++ b/src/config/general.h
@@ -171,6 +171,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 //#define IMAGE_MEM_CMD		/* Read memory command */
 #define IMAGE_ARCHIVE_CMD	/* Archive image management commands */
 #define SHIM_CMD		/* EFI shim command (or dummy command) */
+//#define USB_CMD		/* USB commands */
 
 /*
  * ROM-specific options
diff --git a/src/drivers/bus/usb.c b/src/drivers/bus/usb.c
index 428ae26c1..e1e7d51f7 100644
--- a/src/drivers/bus/usb.c
+++ b/src/drivers/bus/usb.c
@@ -1323,7 +1323,8 @@ usb_probe_all ( struct usb_device *usb,
 		func->name = func->dev.name;
 		func->usb = usb;
 		func->dev.desc.bus_type = BUS_TYPE_USB;
-		func->dev.desc.location = usb->address;
+		func->dev.desc.location =
+			USB_BUSDEV ( bus->address, usb->address );
 		func->dev.desc.vendor = le16_to_cpu ( usb->device.vendor );
 		func->dev.desc.device = le16_to_cpu ( usb->device.product );
 		snprintf ( func->dev.name, sizeof ( func->dev.name ),
@@ -1725,6 +1726,25 @@ static void free_usb ( struct usb_device *usb ) {
 	free ( usb );
 }
 
+/**
+ * Find USB device by address
+ *
+ * @v bus		USB bus
+ * @v address		Device address
+ * @ret usb		USB device, or NULL if not found
+ */
+struct usb_device * find_usb ( struct usb_bus *bus, unsigned int address ) {
+	struct usb_device *usb;
+
+	/* Search for a matching non-zero address */
+	list_for_each_entry ( usb, &bus->devices, list ) {
+		if ( address && ( usb->address == address ) )
+			return usb;
+	}
+
+	return NULL;
+}
+
 /******************************************************************************
  *
  * USB device hotplug event handling
@@ -2115,6 +2135,11 @@ int register_usb_bus ( struct usb_bus *bus ) {
 	/* Sanity checks */
 	assert ( bus->hub != NULL );
 
+	/* Assign the first available bus address */
+	bus->address = 0;
+	while ( find_usb_bus ( bus->address ) != NULL )
+		bus->address++;
+
 	/* Open bus */
 	if ( ( rc = bus->host->open ( bus ) ) != 0 )
 		goto err_open;
@@ -2187,6 +2212,23 @@ void free_usb_bus ( struct usb_bus *bus ) {
 	free ( bus );
 }
 
+/**
+ * Find USB bus by address
+ *
+ * @v address		Bus address
+ * @ret bus		USB bus, or NULL
+ */
+struct usb_bus * find_usb_bus ( unsigned int address ) {
+	struct usb_bus *bus;
+
+	for_each_usb_bus ( bus ) {
+		if ( bus->address == address )
+			return bus;
+	}
+
+	return NULL;
+}
+
 /**
  * Find USB bus by device location
  *
@@ -2209,7 +2251,7 @@ struct usb_bus * find_usb_bus_by_location ( unsigned int bus_type,
 
 /******************************************************************************
  *
- * USB address assignment
+ * USB device addressing
  *
  ******************************************************************************
  */
@@ -2250,6 +2292,35 @@ void usb_free_address ( struct usb_bus *bus, unsigned int address ) {
 	bus->addresses &= ~( 1ULL << ( address - 1 ) );
 }
 
+/**
+ * Find next USB device
+ *
+ * @v usb		USB device to fill in
+ * @v busdev		Starting bus:dev address
+ * @ret busdev		Bus:dev address of next USB device
+ * @ret rc		Return status code
+ */
+int usb_find_next ( struct usb_device **usb, uint16_t *busdev ) {
+	struct usb_bus *bus;
+
+	do {
+		/* Find USB bus, if any */
+		bus = find_usb_bus ( USB_BUS ( *busdev ) );
+		if ( ! bus ) {
+			*busdev |= ( USB_BUS ( 1 ) - 1 );
+			continue;
+		}
+
+		/* Find USB device, if any */
+		*usb = find_usb ( bus, USB_DEV ( *busdev ) );
+		if ( *usb )
+			return 0;
+
+	} while ( ++(*busdev) );
+
+	return -ENODEV;
+}
+
 /******************************************************************************
  *
  * USB bus topology
diff --git a/src/hci/commands/usb_cmd.c b/src/hci/commands/usb_cmd.c
new file mode 100644
index 000000000..d1086fd7e
--- /dev/null
+++ b/src/hci/commands/usb_cmd.c
@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <getopt.h>
+#include <ipxe/usb.h>
+#include <ipxe/command.h>
+#include <ipxe/parseopt.h>
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * USB commands
+ *
+ */
+
+/** "usbscan" options */
+struct usbscan_options {};
+
+/** "usbscan" option list */
+static struct option_descriptor usbscan_opts[] = {};
+
+/** "usbscan" command descriptor */
+static struct command_descriptor usbscan_cmd =
+	COMMAND_DESC ( struct usbscan_options, usbscan_opts, 1, 1,
+		       "<setting>" );
+
+/**
+ * "usbscan" command
+ *
+ * @v argc		Argument count
+ * @v argv		Argument list
+ * @ret rc		Return status code
+ */
+static int usbscan_exec ( int argc, char **argv ) {
+	struct usbscan_options opts;
+	struct named_setting setting;
+	struct usb_device *usb;
+	unsigned long prev;
+	uint16_t busdev;
+	int len;
+	int rc;
+
+	/* Parse options */
+	if ( ( rc = parse_options ( argc, argv, &usbscan_cmd, &opts ) ) != 0 )
+		goto err_parse_options;
+
+	/* Parse setting name */
+	if ( ( rc = parse_autovivified_setting ( argv[optind],
+						 &setting ) ) != 0 )
+		goto err_parse_setting;
+
+	/* Determine starting bus:dev.fn address */
+	if ( ( len = fetchn_setting ( setting.settings, &setting.setting,
+				      NULL, &setting.setting, &prev ) ) < 0 ) {
+		/* Setting not yet defined: start searching from 00:00 */
+		busdev = 0;
+	} else {
+		/* Setting is defined: start searching from next location */
+		busdev = ( prev + 1 );
+		if ( ! busdev ) {
+			rc = -ENOENT;
+			goto err_end;
+		}
+	}
+
+	/* Find next existent USB device */
+	if ( ( rc = usb_find_next ( &usb, &busdev ) ) != 0 )
+		goto err_find_next;
+
+	/* Apply default type if necessary.  Use ":uint16" rather than
+	 * ":hex" to allow for easy inclusion within a
+	 * "${usb/${location}....}" constructed setting.
+	 */
+	if ( ! setting.setting.type )
+		setting.setting.type = &setting_type_uint16;
+
+	/* Store setting */
+	if ( ( rc = storen_setting ( setting.settings, &setting.setting,
+				     busdev ) ) != 0 ) {
+		printf ( "Could not store \"%s\": %s\n",
+			 setting.setting.name, strerror ( rc ) );
+		goto err_store;
+	}
+
+ err_store:
+ err_end:
+ err_find_next:
+ err_parse_setting:
+ err_parse_options:
+	return rc;
+}
+
+/** USB commands */
+struct command usb_commands[] __command = {
+	{
+		.name = "usbscan",
+		.exec = usbscan_exec,
+	},
+};
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 42bc17789..01a2be654 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -423,6 +423,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_editstring	      ( ERRFILE_OTHER | 0x00610000 )
 #define ERRFILE_widget_ui	      ( ERRFILE_OTHER | 0x00620000 )
 #define ERRFILE_form_ui		      ( ERRFILE_OTHER | 0x00630000 )
+#define ERRFILE_usb_cmd		      ( ERRFILE_OTHER | 0x00640000 )
 
 /** @} */
 
diff --git a/src/include/ipxe/usb.h b/src/include/ipxe/usb.h
index 911247ede..d9891b757 100644
--- a/src/include/ipxe/usb.h
+++ b/src/include/ipxe/usb.h
@@ -54,6 +54,20 @@ enum usb_speed {
 	USB_SPEED_SUPER = USB_SPEED ( 5, 3 ),
 };
 
+/** Define a USB bus:device address
+ *
+ * @v bus		Bus address
+ * @v dev		Device address
+ * @ret busdev		Bus:device address
+ */
+#define USB_BUSDEV( bus, dev ) ( ( (bus) << 8 ) | (dev) )
+
+/** Extract USB bus address */
+#define USB_BUS( busdev ) ( (busdev) >> 8 )
+
+/** Extract USB device address */
+#define USB_DEV( busdev ) ( (busdev) & 0xff )
+
 /** USB packet IDs */
 enum usb_pid {
 	/** IN PID */
@@ -956,6 +970,12 @@ struct usb_bus {
 	/** Host controller operations set */
 	struct usb_host_operations *op;
 
+	/** Bus address
+	 *
+	 * This is an internal index used only to allow a USB device
+	 * to be identified via a nominal bus:device address.
+	 */
+	unsigned int address;
 	/** Largest transfer allowed on the bus */
 	size_t mtu;
 	/** Address in-use mask
@@ -1269,6 +1289,9 @@ extern struct usb_endpoint_companion_descriptor *
 usb_endpoint_companion_descriptor ( struct usb_configuration_descriptor *config,
 				    struct usb_endpoint_descriptor *desc );
 
+extern struct usb_device * find_usb ( struct usb_bus *bus,
+				      unsigned int address );
+
 extern struct usb_hub * alloc_usb_hub ( struct usb_bus *bus,
 					struct usb_device *usb,
 					unsigned int ports,
@@ -1285,11 +1308,13 @@ extern struct usb_bus * alloc_usb_bus ( struct device *dev,
 extern int register_usb_bus ( struct usb_bus *bus );
 extern void unregister_usb_bus ( struct usb_bus *bus );
 extern void free_usb_bus ( struct usb_bus *bus );
+extern struct usb_bus * find_usb_bus ( unsigned int address );
 extern struct usb_bus * find_usb_bus_by_location ( unsigned int bus_type,
 						   unsigned int location );
 
 extern int usb_alloc_address ( struct usb_bus *bus );
 extern void usb_free_address ( struct usb_bus *bus, unsigned int address );
+extern int usb_find_next ( struct usb_device **usb, uint16_t *busdev );
 extern unsigned int usb_route_string ( struct usb_device *usb );
 extern struct usb_port * usb_root_hub_port ( struct usb_device *usb );
 extern struct usb_port * usb_transaction_translator ( struct usb_device *usb );

From c69f9589cc7543baba08dbabdb5c30104fadaa34 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 18 Oct 2024 13:13:28 +0100
Subject: [PATCH 212/237] [usb] Expose USB device descriptor and strings via
 settings

Allow scripts to read basic information from USB device descriptors
via the settings mechanism.  For example:

  echo USB vendor ID: ${usb/${busloc}.8.2}
  echo USB device ID: ${usb/${busloc}.10.2}
  echo USB manufacturer name: ${usb/${busloc}.14.0}

The general syntax is

  usb/<bus:dev>.<offset>.<length>

where bus:dev is the USB bus:device address (as obtained via the
"usbscan" command, or from e.g. ${net0/busloc} for a USB network
device), and <offset> and <length> select the required portion of the
USB device descriptor.

Following the usage of SMBIOS settings tags, a <length> of zero may be
used to indicate that the byte at <offset> contains a USB string
descriptor index, and an <offset> of zero may be used to indicate that
the <length> contains a literal USB string descriptor index.

Since the byte at offset zero can never contain a string index, and a
literal string index can never be zero, the combination of both
<length> and <offset> being zero may be used to indicate that the
entire device descriptor is to be read as a raw hex dump.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config_usb.c        |   8 ++
 src/config/settings.h          |   1 +
 src/drivers/bus/usb.c          |   8 +-
 src/drivers/bus/usb_settings.c | 176 +++++++++++++++++++++++++++++++++
 src/drivers/net/ecm.c          |   3 +-
 src/include/ipxe/errfile.h     |   1 +
 6 files changed, 191 insertions(+), 6 deletions(-)
 create mode 100644 src/drivers/bus/usb_settings.c

diff --git a/src/config/config_usb.c b/src/config/config_usb.c
index b679aeb27..10dec221a 100644
--- a/src/config/config_usb.c
+++ b/src/config/config_usb.c
@@ -22,6 +22,7 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/usb.h>
+#include <config/settings.h>
 
 /** @file
  *
@@ -63,3 +64,10 @@ REQUIRE_OBJECT ( usbblk );
 #ifdef USB_EFI
 REQUIRE_OBJECT ( efi_usb );
 #endif
+
+/*
+ * Drag in USB settings mechanism
+ */
+#ifdef USB_SETTINGS
+REQUIRE_OBJECT ( usb_settings );
+#endif
diff --git a/src/config/settings.h b/src/config/settings.h
index d7f787d38..7b4af4fdf 100644
--- a/src/config/settings.h
+++ b/src/config/settings.h
@@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <config/defaults.h>
 
 #define	PCI_SETTINGS	/* PCI device settings */
+#define	USB_SETTINGS	/* USB device settings */
 //#define	CPUID_SETTINGS	/* CPUID settings */
 //#define	MEMMAP_SETTINGS	/* Memory map settings */
 //#define	VMWARE_SETTINGS	/* VMware GuestInfo settings */
diff --git a/src/drivers/bus/usb.c b/src/drivers/bus/usb.c
index e1e7d51f7..b3b361b0d 100644
--- a/src/drivers/bus/usb.c
+++ b/src/drivers/bus/usb.c
@@ -914,10 +914,9 @@ static unsigned int usb_get_default_language ( struct usb_device *usb ) {
  */
 int usb_get_string_descriptor ( struct usb_device *usb, unsigned int index,
 				unsigned int language, char *buf, size_t len ) {
-	size_t max = ( len ? ( len - 1 /* NUL */ ) : 0 );
 	struct {
 		struct usb_descriptor_header header;
-		uint16_t character[max];
+		uint16_t character[len];
 	} __attribute__ (( packed )) *desc;
 	unsigned int actual;
 	unsigned int i;
@@ -952,10 +951,9 @@ int usb_get_string_descriptor ( struct usb_device *usb, unsigned int index,
 		   sizeof ( desc->character[0] ) );
 
 	/* Copy to buffer */
-	for ( i = 0 ; ( ( i < actual ) && ( i < max ) ) ; i++ )
+	memset ( buf, 0, len );
+	for ( i = 0 ; ( ( i < actual ) && ( i < len ) ) ; i++ )
 		buf[i] = le16_to_cpu ( desc->character[i] );
-	if ( len )
-		buf[i] = '\0';
 
 	/* Free buffer */
 	free ( desc );
diff --git a/src/drivers/bus/usb_settings.c b/src/drivers/bus/usb_settings.c
new file mode 100644
index 000000000..db6f94d8a
--- /dev/null
+++ b/src/drivers/bus/usb_settings.c
@@ -0,0 +1,176 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdio.h>
+#include <errno.h>
+#include <ipxe/usb.h>
+#include <ipxe/settings.h>
+#include <ipxe/init.h>
+
+/** @file
+ *
+ * USB device settings
+ *
+ */
+
+/** USB device settings scope */
+static const struct settings_scope usb_settings_scope;
+
+/**
+ * Check applicability of USB device setting
+ *
+ * @v settings		Settings block
+ * @v setting		Setting
+ * @ret applies		Setting applies within this settings block
+ */
+static int usb_settings_applies ( struct settings *settings __unused,
+				  const struct setting *setting ) {
+
+	return ( setting->scope == &usb_settings_scope );
+}
+
+/**
+ * Fetch value of USB device setting
+ *
+ * @v settings		Settings block
+ * @v setting		Setting to fetch
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int usb_settings_fetch ( struct settings *settings __unused,
+				struct setting *setting,
+				void *data, size_t len ) {
+	uint8_t *dst = data;
+	const uint8_t *src;
+	const uint8_t *desc;
+	struct usb_bus *bus;
+	struct usb_device *usb;
+	int tag_direction;
+	unsigned int tag_busdev;
+	unsigned int tag_offset;
+	unsigned int tag_len;
+	unsigned int index;
+	int rc;
+
+	/* Extract parameters from tag */
+	tag_direction = ( ( setting->tag & ( 1 << 31 ) ) ? +1 : -1 );
+	tag_busdev = ( ( setting->tag >> 16 ) & 0x7fff );
+	tag_offset = ( ( setting->tag >> 8 ) & 0xff );
+	tag_len = ( ( setting->tag >> 0 ) & 0xff );
+
+	/* Locate USB device */
+	bus = find_usb_bus ( USB_BUS ( tag_busdev ) );
+	if ( ! bus )
+		return -ENODEV;
+	usb = find_usb ( bus, USB_DEV ( tag_busdev ) );
+	if ( ! usb )
+		return -ENODEV;
+	desc = ( ( const uint8_t * ) &usb->device );
+
+	/* Following the usage of SMBIOS settings tags, a <length> of
+	 * zero indicates that the byte at <offset> contains a string
+	 * index.  An <offset> of zero indicates that the <length>
+	 * contains a literal string index.
+	 *
+	 * Since the byte at offset zero can never contain a string
+	 * index, and a literal string index can never be zero, the
+	 * combination of both <length> and <offset> being zero
+	 * indicates that the entire structure is to be read.
+	 *
+	 * By default we reverse the byte direction since USB values
+	 * are little-endian and iPXE settings are big-endian.  We
+	 * invert this default when reading the entire structure.
+	 */
+	if ( ( tag_len == 0 ) && ( tag_offset == 0 ) ) {
+		tag_len = sizeof ( usb->device );
+		tag_direction = -tag_direction;
+	} else if ( ( tag_len == 0 ) || ( tag_offset == 0 ) ) {
+		index = tag_len;
+		if ( ( ! index ) && ( tag_offset < sizeof ( usb->device ) ) )
+			index = desc[tag_offset];
+		if ( ( rc = usb_get_string_descriptor ( usb, index, 0, data,
+							len ) ) < 0 ) {
+			return rc;
+		}
+		if ( ! setting->type )
+			setting->type = &setting_type_string;
+		return rc;
+	}
+
+	/* Limit length */
+	if ( tag_offset > sizeof ( usb->device ) ) {
+		tag_len = 0;
+	} else if ( ( tag_offset + tag_len ) > sizeof ( usb->device ) ) {
+		tag_len = ( sizeof ( usb->device ) - tag_offset );
+	}
+
+	/* Copy data, reversing endianness if applicable */
+	dst = data;
+	src = ( desc + tag_offset );
+	if ( tag_direction < 0 )
+		src += ( tag_len - 1 );
+	if ( len > tag_len )
+		len = tag_len;
+	for ( ; len-- ; src += tag_direction, dst++ )
+		*dst = *src;
+
+	/* Set type to ":hexraw" if not already specified */
+	if ( ! setting->type )
+		setting->type = &setting_type_hexraw;
+
+	return tag_len;
+}
+
+/** USB device settings operations */
+static struct settings_operations usb_settings_operations = {
+	.applies = usb_settings_applies,
+	.fetch = usb_settings_fetch,
+};
+
+/** USB device settings */
+static struct settings usb_settings = {
+	.refcnt = NULL,
+	.siblings = LIST_HEAD_INIT ( usb_settings.siblings ),
+	.children = LIST_HEAD_INIT ( usb_settings.children ),
+	.op = &usb_settings_operations,
+	.default_scope = &usb_settings_scope,
+};
+
+/** Initialise USB device settings */
+static void usb_settings_init ( void ) {
+	int rc;
+
+	if ( ( rc = register_settings ( &usb_settings, NULL, "usb" ) ) != 0 ) {
+		DBG ( "USB could not register settings: %s\n",
+		      strerror ( rc ) );
+		return;
+	}
+}
+
+/** USB device settings initialiser */
+struct init_fn usb_settings_init_fn __init_fn ( INIT_NORMAL ) = {
+	.initialise = usb_settings_init,
+};
diff --git a/src/drivers/net/ecm.c b/src/drivers/net/ecm.c
index ab1f98370..7b3e92b9b 100644
--- a/src/drivers/net/ecm.c
+++ b/src/drivers/net/ecm.c
@@ -97,8 +97,9 @@ int ecm_fetch_mac ( struct usb_function *func,
 	int rc;
 
 	/* Fetch MAC address string */
+	buf[ sizeof ( buf ) - 1 ] = '\0';
 	len = usb_get_string_descriptor ( usb, desc->mac, 0, buf,
-					  sizeof ( buf ) );
+					  ( sizeof ( buf ) - 1 ) );
 	if ( len < 0 ) {
 		rc = len;
 		return rc;
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 01a2be654..6183e5b7e 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -424,6 +424,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_widget_ui	      ( ERRFILE_OTHER | 0x00620000 )
 #define ERRFILE_form_ui		      ( ERRFILE_OTHER | 0x00630000 )
 #define ERRFILE_usb_cmd		      ( ERRFILE_OTHER | 0x00640000 )
+#define ERRFILE_usb_settings	      ( ERRFILE_OTHER | 0x00650000 )
 
 /** @} */
 

From fa1c24d14baf903c549fdb4f30a55b115eccad7d Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 21 Oct 2024 16:09:44 +0100
Subject: [PATCH 213/237] [crypto] Add bigint_mod_invert() to calculate inverse
 modulo a power of two
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Montgomery multiplication requires calculating the inverse of the
modulus modulo a larger power of two.

Add bigint_mod_invert() to calculate the inverse of any (odd) big
integer modulo an arbitrary power of two, using a lightly modified
version of the algorithm presented in "A New Algorithm for Inversion
mod p^k (Koç, 2017)".

The power of two is taken to be 2^k, where k is the number of bits
available in the big integer representation of the invertend.  The
inverse modulo any smaller power of two may be obtained simply by
masking off the relevant bits in the inverse.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       | 54 ++++++++++++++++++++++++++++++++++
 src/include/ipxe/bigint.h | 28 ++++++++++++++++++
 src/tests/bigint_test.c   | 61 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 143 insertions(+)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index a8b99ec3c..3f5db8517 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -305,6 +305,60 @@ void bigint_reduce_raw ( const bigint_element_t *minuend0,
 	profile_stop ( &bigint_mod_profiler );
 }
 
+/**
+ * Compute inverse of odd big integer modulo any power of two
+ *
+ * @v invertend0	Element 0 of odd big integer to be inverted
+ * @v inverse0		Element 0 of big integer to hold result
+ * @v size		Number of elements in invertend and result
+ * @v tmp		Temporary working space
+ */
+void bigint_mod_invert_raw ( const bigint_element_t *invertend0,
+			     bigint_element_t *inverse0,
+			     unsigned int size, void *tmp ) {
+	const bigint_t ( size ) __attribute__ (( may_alias ))
+		*invertend = ( ( const void * ) invertend0 );
+	bigint_t ( size ) __attribute__ (( may_alias ))
+		*inverse = ( ( void * ) inverse0 );
+	struct {
+		bigint_t ( size ) residue;
+	} *temp = tmp;
+	const unsigned int width = ( 8 * sizeof ( bigint_element_t ) );
+	unsigned int i;
+
+	/* Sanity check */
+	assert ( invertend->element[0] & 1 );
+
+	/* Initialise temporary working space and output value */
+	memset ( &temp->residue, 0xff, sizeof ( temp->residue ) );
+	memset ( inverse, 0, sizeof ( *inverse ) );
+
+	/* Compute inverse modulo 2^(width)
+	 *
+	 * This method is a lightly modified version of the pseudocode
+	 * presented in "A New Algorithm for Inversion mod p^k (Koç,
+	 * 2017)".
+	 *
+	 * Each loop iteration calculates one bit of the inverse.  The
+	 * residue value is the two's complement negation of the value
+	 * "b" as used by Koç, to allow for division by two using a
+	 * logical right shift (since we have no arithmetic right
+	 * shift operation for big integers).
+	 *
+	 * Due to the suffix property of inverses mod 2^k, the result
+	 * represents the least significant bits of the inverse modulo
+	 * an arbitrarily large 2^k.
+	 */
+	for ( i = 0 ; i < ( 8 * sizeof ( *inverse ) ) ; i++ ) {
+		if ( temp->residue.element[0] & 1 ) {
+			inverse->element[ i / width ] |=
+				( 1UL << ( i % width ) );
+			bigint_add ( invertend, &temp->residue );
+		}
+		bigint_shr ( &temp->residue );
+	}
+}
+
 /**
  * Perform modular multiplication of big integers
  *
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index c56b2155f..f7900e0e6 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -246,6 +246,31 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 		bigint_t ( size ) temp_modulus;				\
 	} ); } )
 
+/**
+ * Compute inverse of odd big integer modulo its own size
+ *
+ * @v invertend		Odd big integer to be inverted
+ * @v inverse		Big integer to hold result
+ * @v tmp		Temporary working space
+ */
+#define bigint_mod_invert( invertend, inverse, tmp ) do {		\
+	unsigned int size = bigint_size (invertend);			\
+	bigint_mod_invert_raw ( (invertend)->element,			\
+				(inverse)->element, size, tmp );	\
+	} while ( 0 )
+
+/**
+ * Calculate temporary working space required for modular inversion
+ *
+ * @v invertend		Odd big integer to be inverted
+ * @ret len		Length of temporary working space
+ */
+#define bigint_mod_invert_tmp_len( invertend ) ( {			\
+	unsigned int size = bigint_size (invertend);			\
+	sizeof ( struct {						\
+		bigint_t ( size ) temp_residue;				\
+	} ); } )
+
 /**
  * Perform modular multiplication of big integers
  *
@@ -373,6 +398,9 @@ void bigint_reduce_raw ( const bigint_element_t *minuend0,
 			 const bigint_element_t *modulus0,
 			 unsigned int modulus_size,
 			 bigint_element_t *result0, void *tmp );
+void bigint_mod_invert_raw ( const bigint_element_t *invertend0,
+			     bigint_element_t *inverse0,
+			     unsigned int size, void *tmp );
 void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 			       const bigint_element_t *multiplier0,
 			       const bigint_element_t *modulus0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 104e1f362..166c771b9 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -200,6 +200,17 @@ void bigint_reduce_sample ( const bigint_element_t *minuend0,
 	bigint_reduce ( minuend, modulus, result, tmp );
 }
 
+void bigint_mod_invert_sample ( const bigint_element_t *invertend0,
+				bigint_element_t *inverse0,
+				unsigned int size, void *tmp ) {
+	const bigint_t ( size ) __attribute__ (( may_alias ))
+		*invertend = ( ( const void * ) invertend0 );
+	bigint_t ( size ) __attribute__ (( may_alias ))
+		*inverse = ( ( void * ) inverse0 );
+
+	bigint_mod_invert ( invertend, inverse, tmp );
+}
+
 void bigint_mod_multiply_sample ( const bigint_element_t *multiplicand0,
 				  const bigint_element_t *multiplier0,
 				  const bigint_element_t *modulus0,
@@ -573,6 +584,39 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 		      sizeof ( result_raw ) ) == 0 );			\
 	} while ( 0 )
 
+/**
+ * Report result of big integer modular inversion test
+ *
+ * @v invertend		Big integer to be inverted
+ * @v expected		Big integer expected result
+ */
+#define bigint_mod_invert_ok( invertend, expected ) do {		\
+	static const uint8_t invertend_raw[] = invertend;		\
+	static const uint8_t expected_raw[] = expected;			\
+	uint8_t inverse_raw[ sizeof ( expected_raw ) ];			\
+	unsigned int size =						\
+		bigint_required_size ( sizeof ( invertend_raw ) );	\
+	bigint_t ( size ) invertend_temp;				\
+	bigint_t ( size ) inverse_temp;					\
+	size_t tmp_len = bigint_mod_invert_tmp_len ( &invertend_temp );	\
+	uint8_t tmp[tmp_len];						\
+	{} /* Fix emacs alignment */					\
+									\
+	assert ( bigint_size ( &invertend_temp ) ==			\
+		 bigint_size ( &inverse_temp ) );			\
+	bigint_init ( &invertend_temp, invertend_raw,			\
+		      sizeof ( invertend_raw ) );			\
+	DBG ( "Modular invert:\n" );					\
+	DBG_HDA ( 0, &invertend_temp, sizeof ( invertend_temp ) );	\
+	bigint_mod_invert ( &invertend_temp, &inverse_temp, tmp );	\
+	DBG_HDA ( 0, &inverse_temp, sizeof ( inverse_temp ) );		\
+	bigint_done ( &inverse_temp, inverse_raw,			\
+		      sizeof ( inverse_raw ) );				\
+									\
+	ok ( memcmp ( inverse_raw, expected_raw,			\
+		      sizeof ( inverse_raw ) ) == 0 );			\
+	} while ( 0 )
+
 /**
  * Report result of big integer modular multiplication test
  *
@@ -1760,6 +1804,23 @@ static void bigint_test_exec ( void ) {
 				    0xfb, 0x5d, 0x55 ),
 			   BIGINT ( 0x27, 0x31, 0x49, 0xc3, 0xf5, 0x06, 0x1f,
 				    0x3c, 0x7c, 0xd5 ) );
+	bigint_mod_invert_ok ( BIGINT ( 0x01 ), BIGINT ( 0x01 ) );
+	bigint_mod_invert_ok ( BIGINT ( 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+					0xff, 0xff ),
+			       BIGINT ( 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+					0xff, 0xff ) );
+	bigint_mod_invert_ok ( BIGINT ( 0x95, 0x6a, 0xc5, 0xe7, 0x2e, 0x5b,
+					0x44, 0xed, 0xbf, 0x7e, 0xfe, 0x8d,
+					0xf4, 0x5a, 0x48, 0xc1 ),
+			       BIGINT ( 0xad, 0xb8, 0x3d, 0x85, 0x10, 0xdf,
+					0xea, 0x70, 0x71, 0x2c, 0x80, 0xf4,
+					0x6e, 0x66, 0x47, 0x41 ) );
+	bigint_mod_invert_ok ( BIGINT ( 0x35, 0xe4, 0x80, 0x48, 0xdd, 0xa1,
+					0x46, 0xc0, 0x84, 0x63, 0xc1, 0xe4,
+					0xf7, 0xbf, 0xb3, 0x05 ),
+			       BIGINT ( 0xf2, 0x9c, 0x63, 0x29, 0xfa, 0xe4,
+					0xbf, 0x90, 0xa6, 0x9a, 0xec, 0xcf,
+					0x5f, 0xe2, 0x21, 0xcd ) );
 	bigint_mod_multiply_ok ( BIGINT ( 0x37 ),
 				 BIGINT ( 0x67 ),
 				 BIGINT ( 0x3f ),

From b23204b383cdea4ad64276991b55d8019fee3265 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 22 Oct 2024 12:51:48 +0100
Subject: [PATCH 214/237] [riscv] Add support for the SBI debug console

Add the ability to issue Supervisor Binary Interface (SBI) calls via
the ECALL instruction, and use the SBI DBCN extension to implement a
debug console.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/riscv/Makefile                    |   1 +
 src/arch/riscv/include/ipxe/sbi.h          | 157 +++++++++++++++++++++
 src/arch/riscv/interface/sbi/sbi_console.c | 101 +++++++++++++
 src/config/config.c                        |   3 +
 src/config/console.h                       |   1 +
 5 files changed, 263 insertions(+)
 create mode 100644 src/arch/riscv/include/ipxe/sbi.h
 create mode 100644 src/arch/riscv/interface/sbi/sbi_console.c

diff --git a/src/arch/riscv/Makefile b/src/arch/riscv/Makefile
index 668d7db39..d3ae4e84c 100644
--- a/src/arch/riscv/Makefile
+++ b/src/arch/riscv/Makefile
@@ -10,6 +10,7 @@ INCDIRS		:= arch/$(ARCH)/include arch/riscv/include $(INCDIRS)
 # RISCV-specific directories containing source files
 #
 SRCDIRS		+= arch/riscv/core
+SRCDIRS		+= arch/riscv/interface/sbi
 
 # RISCV-specific flags
 #
diff --git a/src/arch/riscv/include/ipxe/sbi.h b/src/arch/riscv/include/ipxe/sbi.h
new file mode 100644
index 000000000..529f1d0c9
--- /dev/null
+++ b/src/arch/riscv/include/ipxe/sbi.h
@@ -0,0 +1,157 @@
+#ifndef _IPXE_SBI_H
+#define _IPXE_SBI_H
+
+/** @file
+ *
+ * Supervisor Binary Interface (SBI)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/** An SBI function return value */
+struct sbi_return {
+	/** Error status (returned in a0) */
+	long error;
+	/** Data value (returned in a1) */
+	long value;
+};
+
+/**
+ * @defgroup sbierrors SBI errors
+ *
+ * *{
+ */
+#define SBI_SUCCESS 0			/**< Completed successfully */
+#define SBI_ERR_FAILED -1		/**< Failed */
+#define SBI_ERR_NOT_SUPPORTED -2	/**< Not supported */
+#define SBI_ERR_INVALID_PARAM -3	/**< Invalid parameter(s) */
+#define SBI_ERR_DENIED -4		/**< Denied or not allowed */
+#define SBI_ERR_INVALID_ADDRESS -5	/**< Invalid address(es) */
+#define SBI_ERR_ALREADY_AVAILABLE -6	/**< Already available */
+#define SBI_ERR_ALREADY_STARTED -7	/**< Already started */
+#define SBI_ERR_ALREADY_STOPPED -8	/**< Already stopped */
+#define SBI_ERR_NO_SHMEM -9		/**< Shared memory not available */
+#define SBI_ERR_INVALID_STATE -10	/**< Invalid state */
+#define SBI_ERR_BAD_RANGE -11		/**< Bad (or invalid) range */
+#define SBI_ERR_TIMEOUT -12		/**< Failed due to timeout */
+#define SBI_ERR_IO -13			/**< Input/output error */
+/** @} */
+
+/** Construct SBI extension ID */
+#define SBI_EID( c1, c2, c3, c4 ) \
+	( (int) ( ( (c1) << 24 ) | ( (c2) << 16 ) | ( (c3) << 8 ) | (c4) ) )
+
+/**
+ * Call supervisor with no parameters
+ *
+ * @v eid		Extension ID
+ * @v fid		Function ID
+ * @ret ret		Return value
+ */
+static inline __attribute__ (( always_inline )) struct sbi_return
+sbi_ecall_0 ( int eid, int fid ) {
+	register unsigned long a7 asm ( "a7" ) = ( ( long ) eid );
+	register unsigned long a6 asm ( "a6" ) = ( ( long ) fid );
+	register unsigned long a0 asm ( "a0" );
+	register unsigned long a1 asm ( "a1" );
+	struct sbi_return ret;
+
+	__asm__ __volatile__ ( "ecall"
+			       : "=r" ( a0 ), "=r" ( a1 )
+			       : "r" ( a6 ), "r" ( a7 )
+			       : "memory" );
+	ret.error = a0;
+	ret.value = a1;
+	return ret;
+}
+
+/**
+ * Call supervisor with one parameter
+ *
+ * @v eid		Extension ID
+ * @v fid		Function ID
+ * @v param0		Parameter 0
+ * @ret ret		Return value
+ */
+static inline __attribute__ (( always_inline )) struct sbi_return
+sbi_ecall_1 ( int eid, int fid, unsigned long p0 ) {
+	register unsigned long a7 asm ( "a7" ) = ( ( long ) eid );
+	register unsigned long a6 asm ( "a6" ) = ( ( long ) fid );
+	register unsigned long a0 asm ( "a0" ) = p0;
+	register unsigned long a1 asm ( "a1" );
+	struct sbi_return ret;
+
+	__asm__ __volatile__ ( "ecall"
+			       : "+r" ( a0 ), "=r" ( a1 )
+			       : "r" ( a6 ), "r" ( a7 )
+			       : "memory" );
+	ret.error = a0;
+	ret.value = a1;
+	return ret;
+}
+
+/**
+ * Call supervisor with two parameters
+ *
+ * @v eid		Extension ID
+ * @v fid		Function ID
+ * @v param0		Parameter 0
+ * @v param1		Parameter 1
+ * @ret ret		Return value
+ */
+static inline __attribute__ (( always_inline )) struct sbi_return
+sbi_ecall_2 ( int eid, int fid, unsigned long p0, unsigned long p1 ) {
+	register unsigned long a7 asm ( "a7" ) = ( ( long ) eid );
+	register unsigned long a6 asm ( "a6" ) = ( ( long ) fid );
+	register unsigned long a0 asm ( "a0" ) = p0;
+	register unsigned long a1 asm ( "a1" ) = p1;
+	struct sbi_return ret;
+
+	__asm__ __volatile__ ( "ecall"
+			       : "+r" ( a0 ), "+r" ( a1 )
+			       : "r" ( a6 ), "r" ( a7 )
+			       : "memory" );
+	ret.error = a0;
+	ret.value = a1;
+	return ret;
+}
+
+/**
+ * Call supervisor with three parameters
+ *
+ * @v eid		Extension ID
+ * @v fid		Function ID
+ * @v param0		Parameter 0
+ * @v param1		Parameter 1
+ * @v param2		Parameter 2
+ * @ret ret		Return value
+ */
+static inline __attribute__ (( always_inline )) struct sbi_return
+sbi_ecall_3 ( int eid, int fid, unsigned long p0, unsigned long p1,
+	      unsigned long p2 ) {
+	register unsigned long a7 asm ( "a7" ) = ( ( long ) eid );
+	register unsigned long a6 asm ( "a6" ) = ( ( long ) fid );
+	register unsigned long a0 asm ( "a0" ) = p0;
+	register unsigned long a1 asm ( "a1" ) = p1;
+	register unsigned long a2 asm ( "a2" ) = p2;
+	struct sbi_return ret;
+
+	__asm__ __volatile__ ( "ecall"
+			       : "+r" ( a0 ), "+r" ( a1 )
+			       : "r" ( a2 ), "r" ( a6 ), "r" ( a7 )
+			       : "memory" );
+	ret.error = a0;
+	ret.value = a1;
+	return ret;
+}
+
+/** Debug console extension */
+#define SBI_DBCN SBI_EID ( 'D', 'B', 'C', 'N' )
+#define SBI_DBCN_WRITE 0x00		/**< Console Write */
+#define SBI_DBCN_READ 0x01		/**< Console Read */
+#define SBI_DBCN_WRITE_BYTE 0x02	/**< Console Write Byte */
+
+#endif /* _IPXE_SBI_H */
diff --git a/src/arch/riscv/interface/sbi/sbi_console.c b/src/arch/riscv/interface/sbi/sbi_console.c
new file mode 100644
index 000000000..835255559
--- /dev/null
+++ b/src/arch/riscv/interface/sbi/sbi_console.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+/** @file
+ *
+ * SBI debug console
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/sbi.h>
+#include <ipxe/io.h>
+#include <ipxe/console.h>
+#include <config/console.h>
+
+/* Set default console usage if applicable */
+#if ! ( defined ( CONSOLE_SBI ) && CONSOLE_EXPLICIT ( CONSOLE_SBI ) )
+#undef CONSOLE_SBI
+#define CONSOLE_SBI ( CONSOLE_USAGE_ALL & ~CONSOLE_USAGE_LOG )
+#endif
+
+/** Buffered input character (if any) */
+static unsigned char sbi_console_input;
+
+/**
+ * Print a character to SBI console
+ *
+ * @v character		Character to be printed
+ */
+static void sbi_putchar ( int character ) {
+
+	/* Write byte to console */
+	sbi_ecall_1 ( SBI_DBCN, SBI_DBCN_WRITE_BYTE, character );
+}
+
+/**
+ * Get character from SBI console
+ *
+ * @ret character	Character read from console, if any
+ */
+static int sbi_getchar ( void ) {
+	int character;
+
+	/* Consume and return buffered character, if any */
+	character = sbi_console_input;
+	sbi_console_input = 0;
+	return character;
+}
+
+/**
+ * Check for character ready to read from SBI console
+ *
+ * @ret True		Character available to read
+ * @ret False		No character available to read
+ */
+static int sbi_iskey ( void ) {
+	struct sbi_return ret;
+
+	/* Do nothing if we already have a buffered character */
+	if ( sbi_console_input )
+		return sbi_console_input;
+
+	/* Read and buffer byte from console, if any */
+	ret = sbi_ecall_3 ( SBI_DBCN, SBI_DBCN_READ,
+			    sizeof ( sbi_console_input ),
+			    virt_to_phys ( &sbi_console_input ), 0 );
+	if ( ret.error )
+		return 0;
+
+	/* Return number of characters read and buffered */
+	return ret.value;
+}
+
+/** SBI console */
+struct console_driver sbi_console_driver __console_driver = {
+	.putchar = sbi_putchar,
+	.getchar = sbi_getchar,
+	.iskey = sbi_iskey,
+	.usage = CONSOLE_SBI,
+};
diff --git a/src/config/config.c b/src/config/config.c
index 0f950eb9d..4cfa5dd41 100644
--- a/src/config/config.c
+++ b/src/config/config.c
@@ -78,6 +78,9 @@ REQUIRE_OBJECT ( vmconsole );
 #ifdef CONSOLE_DEBUGCON
 REQUIRE_OBJECT ( debugcon );
 #endif
+#ifdef CONSOLE_SBI
+REQUIRE_OBJECT ( sbi_console );
+#endif
 
 /*
  * Drag in all requested network protocols
diff --git a/src/config/console.h b/src/config/console.h
index 9f770d094..0ff328b7c 100644
--- a/src/config/console.h
+++ b/src/config/console.h
@@ -41,6 +41,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 //#define	CONSOLE_VMWARE		/* VMware logfile console */
 //#define	CONSOLE_DEBUGCON	/* Bochs/QEMU/KVM debug port console */
 //#define	CONSOLE_INT13		/* INT13 disk log console */
+//#define	CONSOLE_SBI		/* RISC-V SBI debug console */
 
 /*
  * Very obscure console types

From 21940425c4b50e2b8f0a2ccf1db2c86873c28462 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 22 Oct 2024 15:13:59 +0100
Subject: [PATCH 215/237] [riscv] Add support for reboot and power off via SBI

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/riscv/include/bits/errfile.h     |  2 +
 src/arch/riscv/include/bits/reboot.h      | 14 +++++
 src/arch/riscv/include/ipxe/sbi.h         | 10 +++
 src/arch/riscv/include/ipxe/sbi_reboot.h  | 18 ++++++
 src/arch/riscv/interface/sbi/sbi_reboot.c | 76 +++++++++++++++++++++++
 5 files changed, 120 insertions(+)
 create mode 100644 src/arch/riscv/include/bits/reboot.h
 create mode 100644 src/arch/riscv/include/ipxe/sbi_reboot.h
 create mode 100644 src/arch/riscv/interface/sbi/sbi_reboot.c

diff --git a/src/arch/riscv/include/bits/errfile.h b/src/arch/riscv/include/bits/errfile.h
index 2b1202147..e97c71a37 100644
--- a/src/arch/riscv/include/bits/errfile.h
+++ b/src/arch/riscv/include/bits/errfile.h
@@ -14,6 +14,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @{
  */
 
+#define ERRFILE_sbi_reboot	( ERRFILE_ARCH | ERRFILE_CORE | 0x00000000 )
+
 /** @} */
 
 #endif /* _BITS_ERRFILE_H */
diff --git a/src/arch/riscv/include/bits/reboot.h b/src/arch/riscv/include/bits/reboot.h
new file mode 100644
index 000000000..01272483b
--- /dev/null
+++ b/src/arch/riscv/include/bits/reboot.h
@@ -0,0 +1,14 @@
+#ifndef _BITS_REBOOT_H
+#define _BITS_REBOOT_H
+
+/** @file
+ *
+ * RISCV-specific reboot API implementations
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/sbi_reboot.h>
+
+#endif /* _BITS_REBOOT_H */
diff --git a/src/arch/riscv/include/ipxe/sbi.h b/src/arch/riscv/include/ipxe/sbi.h
index 529f1d0c9..44685cc87 100644
--- a/src/arch/riscv/include/ipxe/sbi.h
+++ b/src/arch/riscv/include/ipxe/sbi.h
@@ -148,6 +148,16 @@ sbi_ecall_3 ( int eid, int fid, unsigned long p0, unsigned long p1,
 	return ret;
 }
 
+/** Convert an SBI error code to an iPXE status code */
+#define ESBI( error ) EPLATFORM ( EINFO_EPLATFORM, error )
+
+/** System reset extension */
+#define SBI_SRST SBI_EID ( 'S', 'R', 'S', 'T' )
+#define SBI_SRST_SYSTEM_RESET 0x00	/**< Reset system */
+#define SBI_RESET_SHUTDOWN 0x00000000	/**< Shutdown */
+#define SBI_RESET_COLD 0x00000001	/**< Cold reboot */
+#define SBI_RESET_WARM 0x00000002	/**< Warm reboot */
+
 /** Debug console extension */
 #define SBI_DBCN SBI_EID ( 'D', 'B', 'C', 'N' )
 #define SBI_DBCN_WRITE 0x00		/**< Console Write */
diff --git a/src/arch/riscv/include/ipxe/sbi_reboot.h b/src/arch/riscv/include/ipxe/sbi_reboot.h
new file mode 100644
index 000000000..e8d6e82bf
--- /dev/null
+++ b/src/arch/riscv/include/ipxe/sbi_reboot.h
@@ -0,0 +1,18 @@
+#ifndef _IPXE_BIOS_REBOOT_H
+#define _IPXE_BIOS_REBOOT_H
+
+/** @file
+ *
+ * Supervisor Binary Interface (SBI) reboot mechanism
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#ifdef REBOOT_SBI
+#define REBOOT_PREFIX_sbi
+#else
+#define REBOOT_PREFIX_sbi __sbi_
+#endif
+
+#endif /* _IPXE_BIOS_REBOOT_H */
diff --git a/src/arch/riscv/interface/sbi/sbi_reboot.c b/src/arch/riscv/interface/sbi/sbi_reboot.c
new file mode 100644
index 000000000..64365b0d3
--- /dev/null
+++ b/src/arch/riscv/interface/sbi/sbi_reboot.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Supervisor Binary Interface (SBI) reboot mechanism
+ *
+ */
+
+#include <errno.h>
+#include <string.h>
+#include <ipxe/sbi.h>
+#include <ipxe/reboot.h>
+
+/**
+ * Reboot system
+ *
+ * @v warm		Perform a warm reboot
+ */
+static void sbi_reboot ( int warm ) {
+	struct sbi_return ret;
+	int rc;
+
+	/* Reboot system */
+	ret = sbi_ecall_2 ( SBI_SRST, SBI_SRST_SYSTEM_RESET,
+			    ( warm ? SBI_RESET_WARM : SBI_RESET_COLD ), 0 );
+
+	/* Any return is an error */
+	rc = -ESBI ( ret.error );
+	DBGC ( SBI_SRST, "SBI %s reset failed: %s\n",
+	       ( warm ? "warm" : "cold" ), strerror ( rc ) );
+}
+
+/**
+ * Power off system
+ *
+ * @ret rc		Return status code
+ */
+static int sbi_poweroff ( void ) {
+	struct sbi_return ret;
+	int rc;
+
+	/* Shut down system */
+	ret = sbi_ecall_2 ( SBI_SRST, SBI_SRST_SYSTEM_RESET,
+			    SBI_RESET_SHUTDOWN, 0 );
+
+	/* Any return is an error */
+	rc = -ESBI ( ret.error );
+	DBGC ( SBI_SRST, "SBI shutdown failed: %s\n", strerror ( rc ) );
+	return rc;
+}
+
+PROVIDE_REBOOT ( sbi, reboot, sbi_reboot );
+PROVIDE_REBOOT ( sbi, poweroff, sbi_poweroff );

From 33d80b1cd8dd73580c7843d4a1fb8b6b3f458e22 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 25 Oct 2024 14:05:00 +0100
Subject: [PATCH 216/237] [smbios] Provide a null SMBIOS API for platforms with
 no concept of SMBIOS

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/null_smbios.c         | 47 ++++++++++++++++++++++++++++++++++
 src/include/ipxe/errfile.h     |  1 +
 src/include/ipxe/null_smbios.h | 18 +++++++++++++
 src/include/ipxe/smbios.h      |  1 +
 4 files changed, 67 insertions(+)
 create mode 100644 src/core/null_smbios.c
 create mode 100644 src/include/ipxe/null_smbios.h

diff --git a/src/core/null_smbios.c b/src/core/null_smbios.c
new file mode 100644
index 000000000..d46aab3b5
--- /dev/null
+++ b/src/core/null_smbios.c
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Null SMBIOS API
+ *
+ */
+
+#include <errno.h>
+#include <ipxe/smbios.h>
+
+/**
+ * Find SMBIOS
+ *
+ * @v smbios		SMBIOS entry point descriptor structure to fill in
+ * @ret rc		Return status code
+ */
+static int null_find_smbios ( struct smbios *smbios __unused ) {
+
+	return -ENOTSUP;
+}
+
+PROVIDE_SMBIOS ( null, find_smbios, null_find_smbios );
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 6183e5b7e..48b35c537 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -83,6 +83,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_efi_path	       ( ERRFILE_CORE | 0x002b0000 )
 #define ERRFILE_efi_mp		       ( ERRFILE_CORE | 0x002c0000 )
 #define ERRFILE_efi_service	       ( ERRFILE_CORE | 0x002d0000 )
+#define ERRFILE_null_smbios	       ( ERRFILE_CORE | 0x002e0000 )
 
 #define ERRFILE_eisa		     ( ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_isa		     ( ERRFILE_DRIVER | 0x00010000 )
diff --git a/src/include/ipxe/null_smbios.h b/src/include/ipxe/null_smbios.h
new file mode 100644
index 000000000..c430dd089
--- /dev/null
+++ b/src/include/ipxe/null_smbios.h
@@ -0,0 +1,18 @@
+#ifndef _IPXE_NULL_SMBIOS_H
+#define _IPXE_NULL_SMBIOS_H
+
+/** @file
+ *
+ * Null SMBIOS API
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#ifdef SMBIOS_NULL
+#define SMBIOS_PREFIX_null
+#else
+#define SMBIOS_PREFIX_null __null_
+#endif
+
+#endif /* _IPXE_NULL_SMBIOS_H */
diff --git a/src/include/ipxe/smbios.h b/src/include/ipxe/smbios.h
index 077a67a85..f36a5ad40 100644
--- a/src/include/ipxe/smbios.h
+++ b/src/include/ipxe/smbios.h
@@ -25,6 +25,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	PROVIDE_SINGLE_API ( SMBIOS_PREFIX_ ## _subsys, _api_func, _func )
 
 /* Include all architecture-independent SMBIOS API headers */
+#include <ipxe/null_smbios.h>
 #include <ipxe/efi/efi_smbios.h>
 #include <ipxe/linux/linux_smbios.h>
 

From abfa7c3ab1323967c38b11d4732b2d27ecf2f094 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 25 Oct 2024 14:21:27 +0100
Subject: [PATCH 217/237] [uaccess] Rename UACCESS_EFI to UACCESS_FLAT

Running with flat physical addressing is a fairly common early boot
environment.  Rename UACCESS_EFI to UACCESS_FLAT so that this code may
be reused in non-UEFI boot environments that also use flat physical
addressing.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/defaults/efi.h                     |   2 +-
 .../efi/efi_uaccess.c => core/uaccess.c}      |  26 ++---
 src/include/ipxe/efi/efi_uaccess.h            | 103 ------------------
 src/include/ipxe/uaccess.h                    |  76 ++++++++++++-
 4 files changed, 89 insertions(+), 118 deletions(-)
 rename src/{interface/efi/efi_uaccess.c => core/uaccess.c} (63%)
 delete mode 100644 src/include/ipxe/efi/efi_uaccess.h

diff --git a/src/config/defaults/efi.h b/src/config/defaults/efi.h
index b32381de6..607f94c14 100644
--- a/src/config/defaults/efi.h
+++ b/src/config/defaults/efi.h
@@ -9,7 +9,7 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#define UACCESS_EFI
+#define UACCESS_FLAT
 #define IOMAP_VIRT
 #define PCIAPI_EFI
 #define DMAAPI_OP
diff --git a/src/interface/efi/efi_uaccess.c b/src/core/uaccess.c
similarity index 63%
rename from src/interface/efi/efi_uaccess.c
rename to src/core/uaccess.c
index e058be66b..d3a9ca17d 100644
--- a/src/interface/efi/efi_uaccess.c
+++ b/src/core/uaccess.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -24,21 +24,21 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
-#include <ipxe/efi/efi.h>
 
 /** @file
  *
- * iPXE user access API for EFI
+ * iPXE user access API
  *
  */
 
-PROVIDE_UACCESS_INLINE ( efi, phys_to_user );
-PROVIDE_UACCESS_INLINE ( efi, user_to_phys );
-PROVIDE_UACCESS_INLINE ( efi, virt_to_user );
-PROVIDE_UACCESS_INLINE ( efi, user_to_virt );
-PROVIDE_UACCESS_INLINE ( efi, userptr_add );
-PROVIDE_UACCESS_INLINE ( efi, memcpy_user );
-PROVIDE_UACCESS_INLINE ( efi, memmove_user );
-PROVIDE_UACCESS_INLINE ( efi, memset_user );
-PROVIDE_UACCESS_INLINE ( efi, strlen_user );
-PROVIDE_UACCESS_INLINE ( efi, memchr_user );
+/* Flat address space user access API */
+PROVIDE_UACCESS_INLINE ( flat, phys_to_user );
+PROVIDE_UACCESS_INLINE ( flat, user_to_phys );
+PROVIDE_UACCESS_INLINE ( flat, virt_to_user );
+PROVIDE_UACCESS_INLINE ( flat, user_to_virt );
+PROVIDE_UACCESS_INLINE ( flat, userptr_add );
+PROVIDE_UACCESS_INLINE ( flat, memcpy_user );
+PROVIDE_UACCESS_INLINE ( flat, memmove_user );
+PROVIDE_UACCESS_INLINE ( flat, memset_user );
+PROVIDE_UACCESS_INLINE ( flat, strlen_user );
+PROVIDE_UACCESS_INLINE ( flat, memchr_user );
diff --git a/src/include/ipxe/efi/efi_uaccess.h b/src/include/ipxe/efi/efi_uaccess.h
deleted file mode 100644
index 3cc750405..000000000
--- a/src/include/ipxe/efi/efi_uaccess.h
+++ /dev/null
@@ -1,103 +0,0 @@
-#ifndef _IPXE_EFI_UACCESS_H
-#define _IPXE_EFI_UACCESS_H
-
-/** @file
- *
- * iPXE user access API for EFI
- *
- * EFI runs with flat physical addressing, so the various mappings
- * between virtual addresses, I/O addresses and bus addresses are all
- * no-ops.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#ifdef UACCESS_EFI
-#define UACCESS_PREFIX_efi
-#else
-#define UACCESS_PREFIX_efi __efi_
-#endif
-
-/**
- * Convert physical address to user pointer
- *
- * @v phys_addr		Physical address
- * @ret userptr		User pointer
- */
-static inline __always_inline userptr_t
-UACCESS_INLINE ( efi, phys_to_user ) ( unsigned long phys_addr ) {
-	return phys_addr;
-}
-
-/**
- * Convert user buffer to physical address
- *
- * @v userptr		User pointer
- * @v offset		Offset from user pointer
- * @ret phys_addr	Physical address
- */
-static inline __always_inline unsigned long
-UACCESS_INLINE ( efi, user_to_phys ) ( userptr_t userptr, off_t offset ) {
-	return ( userptr + offset );
-}
-
-static inline __always_inline userptr_t
-UACCESS_INLINE ( efi, virt_to_user ) ( volatile const void *addr ) {
-	return trivial_virt_to_user ( addr );
-}
-
-static inline __always_inline void *
-UACCESS_INLINE ( efi, user_to_virt ) ( userptr_t userptr, off_t offset ) {
-	return trivial_user_to_virt ( userptr, offset );
-}
-
-static inline __always_inline userptr_t
-UACCESS_INLINE ( efi, userptr_add ) ( userptr_t userptr, off_t offset ) {
-	return trivial_userptr_add ( userptr, offset );
-}
-
-static inline __always_inline off_t
-UACCESS_INLINE ( efi, userptr_sub ) ( userptr_t userptr,
-				      userptr_t subtrahend ) {
-	return trivial_userptr_sub ( userptr, subtrahend );
-}
-
-static inline __always_inline void
-UACCESS_INLINE ( efi, memcpy_user ) ( userptr_t dest, off_t dest_off,
-					userptr_t src, off_t src_off,
-					size_t len ) {
-	trivial_memcpy_user ( dest, dest_off, src, src_off, len );
-}
-
-static inline __always_inline void
-UACCESS_INLINE ( efi, memmove_user ) ( userptr_t dest, off_t dest_off,
-					 userptr_t src, off_t src_off,
-					 size_t len ) {
-	trivial_memmove_user ( dest, dest_off, src, src_off, len );
-}
-
-static inline __always_inline int
-UACCESS_INLINE ( efi, memcmp_user ) ( userptr_t first, off_t first_off,
-				      userptr_t second, off_t second_off,
-				      size_t len ) {
-	return trivial_memcmp_user ( first, first_off, second, second_off, len);
-}
-
-static inline __always_inline void
-UACCESS_INLINE ( efi, memset_user ) ( userptr_t buffer, off_t offset,
-					int c, size_t len ) {
-	trivial_memset_user ( buffer, offset, c, len );
-}
-
-static inline __always_inline size_t
-UACCESS_INLINE ( efi, strlen_user ) ( userptr_t buffer, off_t offset ) {
-	return trivial_strlen_user ( buffer, offset );
-}
-
-static inline __always_inline off_t
-UACCESS_INLINE ( efi, memchr_user ) ( userptr_t buffer, off_t offset,
-					int c, size_t len ) {
-	return trivial_memchr_user ( buffer, offset, c, len );
-}
-
-#endif /* _IPXE_EFI_UACCESS_H */
diff --git a/src/include/ipxe/uaccess.h b/src/include/ipxe/uaccess.h
index a3f78566a..b71361cab 100644
--- a/src/include/ipxe/uaccess.h
+++ b/src/include/ipxe/uaccess.h
@@ -26,6 +26,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/api.h>
 #include <config/ioapi.h>
 
+#ifdef UACCESS_FLAT
+#define UACCESS_PREFIX_flat
+#else
+#define UACCESS_PREFIX_flat __flat_
+#endif
+
 /**
  * A pointer to a user buffer
  *
@@ -216,8 +222,76 @@ trivial_memchr_user ( userptr_t buffer, off_t offset, int c, size_t len ) {
 #define PROVIDE_UACCESS_INLINE( _subsys, _api_func ) \
 	PROVIDE_SINGLE_API_INLINE ( UACCESS_PREFIX_ ## _subsys, _api_func )
 
+static inline __always_inline userptr_t
+UACCESS_INLINE ( flat, phys_to_user ) ( unsigned long phys_addr ) {
+	return phys_addr;
+}
+
+static inline __always_inline unsigned long
+UACCESS_INLINE ( flat, user_to_phys ) ( userptr_t userptr, off_t offset ) {
+	return ( userptr + offset );
+}
+
+static inline __always_inline userptr_t
+UACCESS_INLINE ( flat, virt_to_user ) ( volatile const void *addr ) {
+	return trivial_virt_to_user ( addr );
+}
+
+static inline __always_inline void *
+UACCESS_INLINE ( flat, user_to_virt ) ( userptr_t userptr, off_t offset ) {
+	return trivial_user_to_virt ( userptr, offset );
+}
+
+static inline __always_inline userptr_t
+UACCESS_INLINE ( flat, userptr_add ) ( userptr_t userptr, off_t offset ) {
+	return trivial_userptr_add ( userptr, offset );
+}
+
+static inline __always_inline off_t
+UACCESS_INLINE ( flat, userptr_sub ) ( userptr_t userptr,
+				       userptr_t subtrahend ) {
+	return trivial_userptr_sub ( userptr, subtrahend );
+}
+
+static inline __always_inline void
+UACCESS_INLINE ( flat, memcpy_user ) ( userptr_t dest, off_t dest_off,
+				       userptr_t src, off_t src_off,
+				       size_t len ) {
+	trivial_memcpy_user ( dest, dest_off, src, src_off, len );
+}
+
+static inline __always_inline void
+UACCESS_INLINE ( flat, memmove_user ) ( userptr_t dest, off_t dest_off,
+					userptr_t src, off_t src_off,
+					size_t len ) {
+	trivial_memmove_user ( dest, dest_off, src, src_off, len );
+}
+
+static inline __always_inline int
+UACCESS_INLINE ( flat, memcmp_user ) ( userptr_t first, off_t first_off,
+				       userptr_t second, off_t second_off,
+				       size_t len ) {
+	return trivial_memcmp_user ( first, first_off, second, second_off, len);
+}
+
+static inline __always_inline void
+UACCESS_INLINE ( flat, memset_user ) ( userptr_t buffer, off_t offset,
+				       int c, size_t len ) {
+	trivial_memset_user ( buffer, offset, c, len );
+}
+
+static inline __always_inline size_t
+UACCESS_INLINE ( flat, strlen_user ) ( userptr_t buffer, off_t offset ) {
+	return trivial_strlen_user ( buffer, offset );
+}
+
+static inline __always_inline off_t
+UACCESS_INLINE ( flat, memchr_user ) ( userptr_t buffer, off_t offset,
+				       int c, size_t len ) {
+	return trivial_memchr_user ( buffer, offset, c, len );
+}
+
 /* Include all architecture-independent user access API headers */
-#include <ipxe/efi/efi_uaccess.h>
 #include <ipxe/linux/linux_uaccess.h>
 
 /* Include all architecture-dependent user access API headers */

From cc45ca372c9e297c961ee40911753be89f38170a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Fri, 25 Oct 2024 14:40:28 +0100
Subject: [PATCH 218/237] [pci] Drag in PCI settings mechanism only when PCI
 support is present

Allow for the existence of platforms with no PCI bus by including the
PCI settings mechanism only if PCI bus support is included.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/config/config.c     |  3 ---
 src/config/config_pci.c | 36 ++++++++++++++++++++++++++++++++++++
 src/drivers/bus/pci.c   |  6 ++++++
 3 files changed, 42 insertions(+), 3 deletions(-)
 create mode 100644 src/config/config_pci.c

diff --git a/src/config/config.c b/src/config/config.c
index 4cfa5dd41..254bff995 100644
--- a/src/config/config.c
+++ b/src/config/config.c
@@ -352,9 +352,6 @@ REQUIRE_OBJECT ( efi_bofm );
 /*
  * Drag in relevant settings sources
  */
-#ifdef PCI_SETTINGS
-REQUIRE_OBJECT ( pci_settings );
-#endif
 #ifdef VMWARE_SETTINGS
 REQUIRE_OBJECT ( guestinfo );
 #endif
diff --git a/src/config/config_pci.c b/src/config/config_pci.c
new file mode 100644
index 000000000..cb8df712a
--- /dev/null
+++ b/src/config/config_pci.c
@@ -0,0 +1,36 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/settings.h>
+
+/** @file
+ *
+ * PCI configuration options
+ *
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+#ifdef PCI_SETTINGS
+REQUIRE_OBJECT ( pci_settings );
+#endif
diff --git a/src/drivers/bus/pci.c b/src/drivers/bus/pci.c
index 05c9a5c26..60740ac18 100644
--- a/src/drivers/bus/pci.c
+++ b/src/drivers/bus/pci.c
@@ -438,3 +438,9 @@ struct root_device pci_root_device __root_device = {
 	.dev = { .name = "PCI" },
 	.driver = &pci_root_driver,
 };
+
+/* Drag in objects via pcibus_probe() */
+REQUIRING_SYMBOL ( pcibus_probe );
+
+/* Drag in PCI configuration */
+REQUIRE_OBJECT ( config_pci );

From 74710b8316b40c9e3081c1014813867d4764796b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 13:45:51 +0000
Subject: [PATCH 219/237] [fdt] Add ability to parse unsigned integer
 properties

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/fdt.c         | 38 ++++++++++++++++++++++++++++++++++++++
 src/include/ipxe/fdt.h |  1 +
 2 files changed, 39 insertions(+)

diff --git a/src/core/fdt.c b/src/core/fdt.c
index f439422cf..b7289fd36 100644
--- a/src/core/fdt.c
+++ b/src/core/fdt.c
@@ -367,6 +367,44 @@ const char * fdt_string ( unsigned int offset, const char *name ) {
 	return desc.data;
 }
 
+/**
+ * Find integer property
+ *
+ * @v offset		Starting node offset
+ * @v name		Property name
+ * @v value		Integer value to fill in
+ * @ret rc		Return status code
+ */
+int fdt_u64 ( unsigned int offset, const char *name, uint64_t *value ) {
+	struct fdt_descriptor desc;
+	const uint8_t *data;
+	size_t remaining;
+	int rc;
+
+	/* Clear value */
+	*value = 0;
+
+	/* Find property */
+	if ( ( rc = fdt_property ( offset, name, &desc ) ) != 0 )
+		return rc;
+
+	/* Check range */
+	if ( desc.len > sizeof ( *value ) ) {
+		DBGC ( &fdt, "FDT oversized integer property \"%s\"\n", name );
+		return -ERANGE;
+	}
+
+	/* Parse value */
+	data = desc.data;
+	remaining = desc.len;
+	while ( remaining-- ) {
+		*value <<= 8;
+		*value |= *(data++);
+	}
+
+	return 0;
+}
+
 /**
  * Get MAC address from property
  *
diff --git a/src/include/ipxe/fdt.h b/src/include/ipxe/fdt.h
index 97efa100c..9b9b01b6c 100644
--- a/src/include/ipxe/fdt.h
+++ b/src/include/ipxe/fdt.h
@@ -96,6 +96,7 @@ struct fdt {
 extern int fdt_path ( const char *path, unsigned int *offset );
 extern int fdt_alias ( const char *name, unsigned int *offset );
 extern const char * fdt_string ( unsigned int offset, const char *name );
+extern int fdt_u64 ( unsigned int offset, const char *name, uint64_t *value );
 extern int fdt_mac ( unsigned int offset, struct net_device *netdev );
 extern int register_fdt ( const struct fdt_header *hdr );
 

From b0a8eabbf4d32d789686c32809750f95986dc7cb Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 11:44:41 +0000
Subject: [PATCH 220/237] [riscv] Add support for checking CPU extensions
 reported via device tree

RISC-V seems to allow for direct discovery of CPU features only from
M-mode (e.g. by setting up a trap handler and then attempting to
access a CSR), with S-mode code expected to read the resulting
constructed ISA description from the device tree.

Add the ability to check for the presence of named extensions listed
in the "riscv,isa" property of the device tree node corresponding to
the boot hart.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/riscv/core/hart.c            | 100 ++++++++++++++++++++++++++
 src/arch/riscv/include/bits/errfile.h |   1 +
 src/arch/riscv/include/ipxe/hart.h    |  16 +++++
 3 files changed, 117 insertions(+)
 create mode 100644 src/arch/riscv/core/hart.c
 create mode 100644 src/arch/riscv/include/ipxe/hart.h

diff --git a/src/arch/riscv/core/hart.c b/src/arch/riscv/core/hart.c
new file mode 100644
index 000000000..c4f2bd0a5
--- /dev/null
+++ b/src/arch/riscv/core/hart.c
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Hardware threads (harts)
+ *
+ */
+
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <ipxe/fdt.h>
+#include <ipxe/hart.h>
+
+/** Boot hart ID */
+unsigned long boot_hart;
+
+/** Colour for debug messages */
+#define colour &boot_hart
+
+/**
+ * Find boot hart node
+ *
+ * @v offset		Boot hart node offset
+ * @ret rc		Return status code
+ */
+static int hart_node ( unsigned int *offset ) {
+	char path[27 /* "/cpus/cpu@XXXXXXXXXXXXXXXX" + NUL */ ];
+	int rc;
+
+	/* Construct node path */
+	snprintf ( path, sizeof ( path ), "/cpus/cpu@%lx", boot_hart );
+
+	/* Find node */
+	if ( ( rc = fdt_path ( path, offset ) ) != 0 ) {
+		DBGC ( colour, "HART could not find %s: %s\n",
+		       path, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Check for supported extension
+ *
+ * @v ext		Extension name (including leading underscore)
+ * @ret rc		Return status code
+ */
+int hart_supported ( const char *ext ) {
+	unsigned int offset;
+	const char *isa;
+	const char *tmp;
+	int rc;
+
+	/* Find boot hart node */
+	if ( ( rc = hart_node ( &offset ) ) != 0 )
+		return rc;
+
+	/* Get ISA description */
+	isa = fdt_string ( offset, "riscv,isa" );
+	if ( ! isa ) {
+		DBGC ( colour, "HART could not identify ISA\n" );
+		return -ENOENT;
+	}
+	DBGC ( colour, "HART supports %s\n", isa );
+
+	/* Check for presence of extension */
+	tmp = isa;
+	while ( ( tmp = strstr ( tmp, ext ) ) != NULL ) {
+		tmp += strlen ( ext );
+		if ( ( *tmp == '\0' ) || ( *tmp == '_' ) )
+			return 0;
+	}
+
+	return -ENOTSUP;
+}
diff --git a/src/arch/riscv/include/bits/errfile.h b/src/arch/riscv/include/bits/errfile.h
index e97c71a37..03f98c206 100644
--- a/src/arch/riscv/include/bits/errfile.h
+++ b/src/arch/riscv/include/bits/errfile.h
@@ -15,6 +15,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 
 #define ERRFILE_sbi_reboot	( ERRFILE_ARCH | ERRFILE_CORE | 0x00000000 )
+#define ERRFILE_hart		( ERRFILE_ARCH | ERRFILE_CORE | 0x00010000 )
 
 /** @} */
 
diff --git a/src/arch/riscv/include/ipxe/hart.h b/src/arch/riscv/include/ipxe/hart.h
new file mode 100644
index 000000000..c201b6c77
--- /dev/null
+++ b/src/arch/riscv/include/ipxe/hart.h
@@ -0,0 +1,16 @@
+#ifndef _IPXE_HART_H
+#define _IPXE_HART_H
+
+/** @file
+ *
+ * Hardware threads (harts)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+extern unsigned long boot_hart;
+
+extern int hart_supported ( const char *ext );
+
+#endif /* _IPXE_HART_H */

From cd54e7c844e2148b46c960d72ed853f8f75bf400 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 13:48:11 +0000
Subject: [PATCH 221/237] [riscv] Add support for RDTIME as a timer source

The Zicntr extension defines an unprivileged wall-clock time CSR that
roughly matches the behaviour of an invariant TSC on x86.  The nominal
frequency of this timer may be read from the "timebase-frequency"
property of the CPU node in the device tree.

Add a timer source using RDTIME to provide implementations of udelay()
and currticks(), modelled on the existing RDTSC-based timer for x86.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/riscv/core/zicntr.c          | 193 ++++++++++++++++++++++++++
 src/arch/riscv/include/bits/errfile.h |   1 +
 src/config/config_timer.c             |   3 +
 3 files changed, 197 insertions(+)
 create mode 100644 src/arch/riscv/core/zicntr.c

diff --git a/src/arch/riscv/core/zicntr.c b/src/arch/riscv/core/zicntr.c
new file mode 100644
index 000000000..0ba453c75
--- /dev/null
+++ b/src/arch/riscv/core/zicntr.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Base counters and timers extension (Zicntr)
+ *
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/fdt.h>
+#include <ipxe/hart.h>
+#include <ipxe/timer.h>
+
+/** Timer increment per microsecond */
+static unsigned long zicntr_mhz;
+
+/** Minimum resolution for scaled timer */
+#define ZICNTR_SCALED_HZ 32
+
+/**
+ * Timer scale (expressed as a bit shift)
+ *
+ * We use this to avoid the need for 64-bit divsion on 32-bit systems.
+ */
+static unsigned int zicntr_scale;
+
+/** Number of timer ticks per scaled timer increment */
+static unsigned long zicntr_ticks;
+
+/** Colour for debug messages */
+#define colour &zicntr_mhz
+
+/**
+ * Get low XLEN bits of current time
+ *
+ * @ret time		Current time
+ */
+static inline __attribute__ (( always_inline )) unsigned long
+rdtime_low ( void ) {
+	unsigned long time;
+
+	/* Read low XLEN bits of current time */
+	__asm__ ( "rdtime %0" : "=r" ( time ) );
+	return time;
+}
+
+/**
+ * Get current time, scaled to avoid rollover within a realistic timescale
+ *
+ * @ret time		Scaled current time
+ */
+static inline __attribute__ (( always_inline )) unsigned long
+rdtime_scaled ( void ) {
+	union {
+		uint64_t time;
+		struct {
+			uint32_t low;
+			uint32_t high;
+		};
+	} u;
+	unsigned long tmp __attribute__ (( unused ));
+
+	/* Read full current time */
+#if __riscv_xlen >= 64
+	__asm__ ( "rdtime %0" : "=r" ( u.time ) );
+#else
+	__asm__ ( "1:\n\t"
+		  "rdtimeh %1\n\t"
+		  "rdtime %0\n\t"
+		  "rdtimeh %2\n\t"
+		  "bne %1, %2, 1b\n\t"
+		  : "=r" ( u.low ), "=r" ( u.high ), "=r" ( tmp ) );
+#endif
+
+	/* Scale time to avoid XLEN-bit rollover */
+	return ( u.time >> zicntr_scale );
+}
+
+/**
+ * Get current system time in ticks
+ *
+ * @ret ticks		Current time, in ticks
+ */
+static unsigned long zicntr_currticks ( void ) {
+	unsigned long scaled;
+
+	/* Get scaled time and convert to ticks */
+	scaled = rdtime_scaled();
+	return ( scaled * zicntr_ticks );
+}
+
+/**
+ * Delay for a fixed number of microseconds
+ *
+ * @v usecs		Number of microseconds for which to delay
+ */
+static void zicntr_udelay ( unsigned long usecs ) {
+	unsigned long start;
+	unsigned long elapsed;
+	unsigned long threshold;
+
+	/* Delay until sufficient time has elapsed */
+	start = rdtime_low();
+	threshold = ( usecs * zicntr_mhz );
+	do {
+		elapsed = ( rdtime_low() - start );
+	} while ( elapsed < threshold );
+}
+
+/**
+ * Probe timer
+ *
+ * @ret rc		Return status code
+ */
+static int zicntr_probe ( void ) {
+	unsigned int offset;
+	union {
+		uint64_t freq;
+		int64_t sfreq;
+	} u;
+	int rc;
+
+	/* Check if Zicntr extension is supported */
+	if ( ( rc = hart_supported ( "_zicntr" ) ) != 0 ) {
+		DBGC ( colour, "ZICNTR not supported: %s\n", strerror ( rc ) );
+		return rc;
+	}
+
+	/* Get timer frequency */
+	if ( ( ( rc = fdt_path ( "/cpus", &offset ) ) != 0 ) ||
+	     ( ( rc = fdt_u64 ( offset, "timebase-frequency",
+				&u.freq ) ) != 0 ) ) {
+		DBGC ( colour, "ZICNTR could not determine frequency: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Convert to MHz (without 64-bit division) */
+	do {
+		zicntr_mhz++;
+		u.sfreq -= 1000000;
+	} while ( u.sfreq > 0 );
+
+	/* Calibrate currticks() scaling factor */
+	zicntr_scale = 31;
+	zicntr_ticks = ( ( 1UL << zicntr_scale ) /
+			 ( zicntr_mhz * ( 1000000 / TICKS_PER_SEC ) ) );
+	while ( zicntr_ticks > ( TICKS_PER_SEC / ZICNTR_SCALED_HZ ) ) {
+		zicntr_scale--;
+		zicntr_ticks >>= 1;
+	}
+	DBGC ( colour, "ZICNTR at %ld MHz, %ld ticks per 2^%d increments\n",
+	       zicntr_mhz, zicntr_ticks, zicntr_scale );
+	if ( ! zicntr_ticks ) {
+		DBGC ( colour, "ZICNTR has zero ticks per 2^%d increments\n",
+		       zicntr_scale );
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/** Zicntr timer */
+struct timer zicntr_timer __timer ( TIMER_PREFERRED ) = {
+	.name = "zicntr",
+	.probe = zicntr_probe,
+	.currticks = zicntr_currticks,
+	.udelay = zicntr_udelay,
+};
diff --git a/src/arch/riscv/include/bits/errfile.h b/src/arch/riscv/include/bits/errfile.h
index 03f98c206..62288ad9c 100644
--- a/src/arch/riscv/include/bits/errfile.h
+++ b/src/arch/riscv/include/bits/errfile.h
@@ -16,6 +16,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define ERRFILE_sbi_reboot	( ERRFILE_ARCH | ERRFILE_CORE | 0x00000000 )
 #define ERRFILE_hart		( ERRFILE_ARCH | ERRFILE_CORE | 0x00010000 )
+#define ERRFILE_zicntr		( ERRFILE_ARCH | ERRFILE_CORE | 0x00020000 )
 
 /** @} */
 
diff --git a/src/config/config_timer.c b/src/config/config_timer.c
index d53c39939..a4fe69b00 100644
--- a/src/config/config_timer.c
+++ b/src/config/config_timer.c
@@ -49,3 +49,6 @@ REQUIRE_OBJECT ( linux_timer );
 #ifdef TIMER_ACPI
 REQUIRE_OBJECT ( acpi_timer );
 #endif
+#ifdef TIMER_ZICNTR
+REQUIRE_OBJECT ( zicntr );
+#endif

From be0c9788a13423b851af4d38be149af17f7b8494 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 00:10:18 +0000
Subject: [PATCH 222/237] [riscv] Add support for the seed CSR as an entropy
 source

The Zkr entropy source extension defines a potentially unprivileged
seed CSR that can be read to obtain 16 bits of entropy input, with a
mandated requirement that 256 entropy input bits read from the seed
CSR will contain at least 128 bits of min-entropy.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/riscv/core/zkr.c             | 110 ++++++++++++++++++++++++++
 src/arch/riscv/include/bits/errfile.h |   1 +
 src/config/config_entropy.c           |   3 +
 3 files changed, 114 insertions(+)
 create mode 100644 src/arch/riscv/core/zkr.c

diff --git a/src/arch/riscv/core/zkr.c b/src/arch/riscv/core/zkr.c
new file mode 100644
index 000000000..5cb44c2d0
--- /dev/null
+++ b/src/arch/riscv/core/zkr.c
@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Entropy source extension (Zkr)
+ *
+ */
+
+#include <errno.h>
+#include <ipxe/hart.h>
+#include <ipxe/entropy.h>
+#include <ipxe/drbg.h>
+
+struct entropy_source zkr_entropy __entropy_source ( ENTROPY_PREFERRED );
+
+/** Seed CSR operational state */
+#define ZKR_SEED_OPST_MASK 0xc0000000UL
+#define ZKR_SEED_OPST_ES16 0x80000000UL /**< 16 bits of entropy available */
+
+/** Number of times to retry reading from seed CSR */
+#define ZKR_SEED_MAX_RETRY 1024
+
+/** Colour for debug messages */
+#define colour &zkr_entropy
+
+/**
+ * Enable entropy gathering
+ *
+ * @ret rc		Return status code
+ */
+static int zkr_entropy_enable ( void ) {
+	int rc;
+
+	/* Check if Zkr extension is supported */
+	if ( ( rc = hart_supported ( "_zkr" ) ) != 0 ) {
+		DBGC ( colour, "ZKR not supported: %s\n", strerror ( rc ) );
+		return rc;
+	}
+
+	/* RISC-V ISA mandates that 128 bits of full entropy shall be
+	 * obtained from 256 entropy bits read from the seed CSR.
+	 *
+	 * Each 16-bit sample therefore contains 8 bits of
+	 * min-entropy.
+	 */
+	entropy_init ( &zkr_entropy, MIN_ENTROPY ( 8.0 ) );
+
+	return 0;
+}
+
+/**
+ * Get noise sample
+ *
+ * @ret noise		Noise sample
+ * @ret rc		Return status code
+ */
+static int zkr_get_noise ( noise_sample_t *noise ) {
+	unsigned long seed;
+	unsigned int i;
+
+	/* Read entropy from seed CSR */
+	for ( i = 0 ; i < ZKR_SEED_MAX_RETRY ; i++ ) {
+
+		/* Read seed CSR */
+		__asm__ ( "csrrw %0, seed, zero" : "=r" ( seed ) );
+
+		/* Check operationsl state */
+		if ( ( seed & ZKR_SEED_OPST_MASK ) == ZKR_SEED_OPST_ES16 ) {
+
+			/* Return entropy from both halves of the
+			 * 16-bit entropy source value.
+			 */
+			*noise = ( seed ^ ( seed >> 8 ) );
+			return 0;
+		}
+	}
+
+	DBGC ( colour, "ZKR could not source entropy (seed %#08lx)\n", seed );
+	return -EBUSY;
+}
+
+/** Hardware entropy source */
+struct entropy_source zkr_entropy __entropy_source ( ENTROPY_PREFERRED ) = {
+	.name = "zkr",
+	.enable = zkr_entropy_enable,
+	.get_noise = zkr_get_noise,
+};
diff --git a/src/arch/riscv/include/bits/errfile.h b/src/arch/riscv/include/bits/errfile.h
index 62288ad9c..bdd2927a4 100644
--- a/src/arch/riscv/include/bits/errfile.h
+++ b/src/arch/riscv/include/bits/errfile.h
@@ -17,6 +17,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_sbi_reboot	( ERRFILE_ARCH | ERRFILE_CORE | 0x00000000 )
 #define ERRFILE_hart		( ERRFILE_ARCH | ERRFILE_CORE | 0x00010000 )
 #define ERRFILE_zicntr		( ERRFILE_ARCH | ERRFILE_CORE | 0x00020000 )
+#define ERRFILE_zkr		( ERRFILE_ARCH | ERRFILE_CORE | 0x00030000 )
 
 /** @} */
 
diff --git a/src/config/config_entropy.c b/src/config/config_entropy.c
index 9f12f1fa3..92aa97884 100644
--- a/src/config/config_entropy.c
+++ b/src/config/config_entropy.c
@@ -49,3 +49,6 @@ REQUIRE_OBJECT ( linux_entropy );
 #ifdef ENTROPY_RDRAND
 REQUIRE_OBJECT ( rdrand );
 #endif
+#ifdef ENTROPY_ZKR
+REQUIRE_OBJECT ( zkr );
+#endif

From 06a9330004b6d83a7e1ba3a74f1fec4f49b3226a Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 16:40:44 +0000
Subject: [PATCH 223/237] [riscv] Add missing volatile qualifiers on timer and
 seed CSR accesses

The timer and entropy seed CSRs will, by design, return different
values each time they are read.

Add the missing volatile qualifiers on the inline assembly to prevent
gcc from assuming that repeated invocations may be elided.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/riscv/core/zicntr.c | 17 +++++++++--------
 src/arch/riscv/core/zkr.c    |  3 ++-
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/arch/riscv/core/zicntr.c b/src/arch/riscv/core/zicntr.c
index 0ba453c75..826f31a68 100644
--- a/src/arch/riscv/core/zicntr.c
+++ b/src/arch/riscv/core/zicntr.c
@@ -64,7 +64,7 @@ rdtime_low ( void ) {
 	unsigned long time;
 
 	/* Read low XLEN bits of current time */
-	__asm__ ( "rdtime %0" : "=r" ( time ) );
+	__asm__ __volatile__ ( "rdtime %0" : "=r" ( time ) );
 	return time;
 }
 
@@ -86,14 +86,15 @@ rdtime_scaled ( void ) {
 
 	/* Read full current time */
 #if __riscv_xlen >= 64
-	__asm__ ( "rdtime %0" : "=r" ( u.time ) );
+	__asm__ __volatile__ ( "rdtime %0" : "=r" ( u.time ) );
 #else
-	__asm__ ( "1:\n\t"
-		  "rdtimeh %1\n\t"
-		  "rdtime %0\n\t"
-		  "rdtimeh %2\n\t"
-		  "bne %1, %2, 1b\n\t"
-		  : "=r" ( u.low ), "=r" ( u.high ), "=r" ( tmp ) );
+	__asm__ __volatile__ ( "1:\n\t"
+			       "rdtimeh %1\n\t"
+			       "rdtime %0\n\t"
+			       "rdtimeh %2\n\t"
+			       "bne %1, %2, 1b\n\t"
+			       : "=r" ( u.low ), "=r" ( u.high ),
+				 "=r" ( tmp ) );
 #endif
 
 	/* Scale time to avoid XLEN-bit rollover */
diff --git a/src/arch/riscv/core/zkr.c b/src/arch/riscv/core/zkr.c
index 5cb44c2d0..bf1609671 100644
--- a/src/arch/riscv/core/zkr.c
+++ b/src/arch/riscv/core/zkr.c
@@ -85,7 +85,8 @@ static int zkr_get_noise ( noise_sample_t *noise ) {
 	for ( i = 0 ; i < ZKR_SEED_MAX_RETRY ; i++ ) {
 
 		/* Read seed CSR */
-		__asm__ ( "csrrw %0, seed, zero" : "=r" ( seed ) );
+		__asm__ __volatile__ ( "csrrw %0, seed, zero" :
+				       "=r" ( seed ) );
 
 		/* Check operationsl state */
 		if ( ( seed & ZKR_SEED_OPST_MASK ) == ZKR_SEED_OPST_ES16 ) {

From d9c0d26e1775cb0e45b30ad6b36bae90abb1fc78 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 16:39:23 +0000
Subject: [PATCH 224/237] [pci] Provide a null PCI API for platforms with no
 PCI bus

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/core/null_pci.c         |  43 ++++++++++
 src/include/ipxe/null_pci.h | 154 ++++++++++++++++++++++++++++++++++++
 src/include/ipxe/pci_io.h   |   1 +
 3 files changed, 198 insertions(+)
 create mode 100644 src/core/null_pci.c
 create mode 100644 src/include/ipxe/null_pci.h

diff --git a/src/core/null_pci.c b/src/core/null_pci.c
new file mode 100644
index 000000000..196eabb6f
--- /dev/null
+++ b/src/core/null_pci.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Null PCI API
+ *
+ */
+
+#include <ipxe/pci.h>
+
+PROVIDE_PCIAPI_INLINE ( null, pci_can_probe );
+PROVIDE_PCIAPI_INLINE ( null, pci_discover );
+PROVIDE_PCIAPI_INLINE ( null, pci_read_config_byte );
+PROVIDE_PCIAPI_INLINE ( null, pci_read_config_word );
+PROVIDE_PCIAPI_INLINE ( null, pci_read_config_dword );
+PROVIDE_PCIAPI_INLINE ( null, pci_write_config_byte );
+PROVIDE_PCIAPI_INLINE ( null, pci_write_config_word );
+PROVIDE_PCIAPI_INLINE ( null, pci_write_config_dword );
+PROVIDE_PCIAPI_INLINE ( null, pci_ioremap );
diff --git a/src/include/ipxe/null_pci.h b/src/include/ipxe/null_pci.h
new file mode 100644
index 000000000..476ede202
--- /dev/null
+++ b/src/include/ipxe/null_pci.h
@@ -0,0 +1,154 @@
+#ifndef _IPXE_NULL_PCI_H
+#define _IPXE_NULL_PCI_H
+
+#include <stdint.h>
+
+/** @file
+ *
+ * Null PCI API
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#ifdef PCIAPI_NULL
+#define PCIAPI_PREFIX_null
+#else
+#define PCIAPI_PREFIX_null __null_
+#endif
+
+struct pci_device;
+
+/**
+ * Check if PCI bus probing is allowed
+ *
+ * @ret ok		Bus probing is allowed
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( null, pci_can_probe ) ( void ) {
+	return 0;
+}
+
+/**
+ * Find next PCI bus:dev.fn address range in system
+ *
+ * @v busdevfn		Starting PCI bus:dev.fn address
+ * @v range		PCI bus:dev.fn address range to fill in
+ */
+static inline __always_inline void
+PCIAPI_INLINE ( null, pci_discover ) ( uint32_t busdevfn __unused,
+				       struct pci_range *range ) {
+
+	range->start = 0;
+	range->count = 0;
+}
+
+/**
+ * Read byte from PCI configuration space via PCI BIOS
+ *
+ * @v pci	PCI device
+ * @v where	Location within PCI configuration space
+ * @v value	Value read
+ * @ret rc	Return status code
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( null, pci_read_config_byte ) ( struct pci_device *pci __unused,
+					       unsigned int where __unused,
+					       uint8_t *value ) {
+	*value = 0xff;
+	return 0;
+}
+
+/**
+ * Read word from PCI configuration space via PCI BIOS
+ *
+ * @v pci	PCI device
+ * @v where	Location within PCI configuration space
+ * @v value	Value read
+ * @ret rc	Return status code
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( null, pci_read_config_word ) ( struct pci_device *pci __unused,
+					       unsigned int where __unused,
+					       uint16_t *value ) {
+	*value = 0xffff;
+	return 0;
+}
+
+/**
+ * Read dword from PCI configuration space via PCI BIOS
+ *
+ * @v pci	PCI device
+ * @v where	Location within PCI configuration space
+ * @v value	Value read
+ * @ret rc	Return status code
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( null, pci_read_config_dword ) ( struct pci_device *pci __unused,
+						unsigned int where __unused,
+						uint32_t *value ) {
+	*value = 0xffffffff;
+	return 0;
+}
+
+/**
+ * Write byte to PCI configuration space via PCI BIOS
+ *
+ * @v pci	PCI device
+ * @v where	Location within PCI configuration space
+ * @v value	Value to be written
+ * @ret rc	Return status code
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( null, pci_write_config_byte ) ( struct pci_device *pci __unused,
+						unsigned int where __unused,
+						uint8_t value __unused ) {
+	return 0;
+}
+
+/**
+ * Write word to PCI configuration space via PCI BIOS
+ *
+ * @v pci	PCI device
+ * @v where	Location within PCI configuration space
+ * @v value	Value to be written
+ * @ret rc	Return status code
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( null, pci_write_config_word ) ( struct pci_device *pci __unused,
+						unsigned int where __unused,
+						uint16_t value __unused ) {
+	return 0;
+}
+
+/**
+ * Write dword to PCI configuration space via PCI BIOS
+ *
+ * @v pci	PCI device
+ * @v where	Location within PCI configuration space
+ * @v value	Value to be written
+ * @ret rc	Return status code
+ */
+static inline __always_inline int
+PCIAPI_INLINE ( null, pci_write_config_dword ) ( struct pci_device *pci
+						 __unused,
+						 unsigned int where __unused,
+						 uint32_t value __unused ) {
+	return 0;
+}
+
+/**
+ * Map PCI bus address as an I/O address
+ *
+ * @v bus_addr		PCI bus address
+ * @v len		Length of region
+ * @ret io_addr		I/O address, or NULL on error
+ */
+static inline __always_inline void *
+PCIAPI_INLINE ( null, pci_ioremap ) ( struct pci_device *pci __unused,
+				      unsigned long bus_addr __unused,
+				      size_t len __unused ) {
+	return NULL;
+}
+
+#endif /* _IPXE_NULL_PCI_H */
diff --git a/src/include/ipxe/pci_io.h b/src/include/ipxe/pci_io.h
index 322fdbb24..9b4cbcab4 100644
--- a/src/include/ipxe/pci_io.h
+++ b/src/include/ipxe/pci_io.h
@@ -59,6 +59,7 @@ struct pci_range {
 	PROVIDE_SINGLE_API_INLINE ( PCIAPI_PREFIX_ ## _subsys, _api_func )
 
 /* Include all architecture-independent I/O API headers */
+#include <ipxe/null_pci.h>
 #include <ipxe/ecam_io.h>
 #include <ipxe/efi/efi_pci_api.h>
 #include <ipxe/linux/linux_pci.h>

From 7ccd08dbf4e257cadb3a9e61fec1f37afb19949e Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 18:22:14 +0000
Subject: [PATCH 225/237] [build] Allow default platform to vary by
 architecture

Restructure the parsing of the build directory name from

  bin[[-<arch>]-<platform>]

to

  bin[-<arch>[-<platform>]]

and allow for a per-architecture default build platform.

For the sake of backwards compatibility, handle "bin-efi" as a special
case equivalent to "bin-i386-efi".

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/Makefile.housekeeping | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/Makefile.housekeeping b/src/Makefile.housekeeping
index 5d9c03930..ba61e2166 100644
--- a/src/Makefile.housekeeping
+++ b/src/Makefile.housekeeping
@@ -284,7 +284,7 @@ ifeq ($(wildcard $(BIN)),)
 $(shell $(MKDIR) -p $(BIN))
 endif
 
-# Target to allow e.g. "make bin-efi arch"
+# Target to allow e.g. "make bin-x86_64-efi arch"
 #
 $(BIN) :
 	@# Do nothing, silently
@@ -333,8 +333,13 @@ else
 BIN_AP		:= $(BIN_APS)
 BIN_SECUREBOOT	:= 0
 endif
-BIN_PLATFORM	:= $(lastword $(BIN_AP))
-BIN_ARCH	:= $(wordlist 2,$(words $(BIN_AP)),discard $(BIN_AP))
+ifeq ($(BIN_AP),efi)
+BIN_ARCH	:= i386
+BIN_PLATFORM	:= efi
+else
+BIN_ARCH	:= $(word 1,$(BIN_AP))
+BIN_PLATFORM	:= $(word 2,$(BIN_AP))
+endif
 
 # Determine build architecture
 DEFAULT_ARCH	:= i386
@@ -345,8 +350,10 @@ arch :
 .PHONY : arch
 
 # Determine build platform
-DEFAULT_PLATFORM := pcbios
-PLATFORM	:= $(firstword $(BIN_PLATFORM) $(DEFAULT_PLATFORM))
+DEFAULT_PLATFORM_i386 := pcbios
+DEFAULT_PLATFORM_x86_64 := pcbios
+DEFAULT_PLATFORM := $(DEFAULT_PLATFORM_$(ARCH))
+PLATFORM	:= $(firstword $(BIN_PLATFORM) $(DEFAULT_PLATFORM) none)
 CFLAGS		+= -DPLATFORM=$(PLATFORM) -DPLATFORM_$(PLATFORM)
 platform :
 	@$(ECHO) $(PLATFORM)

From e0e102ee24bdab68b0318b505402f60cfb88a5f0 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 14:40:35 +0000
Subject: [PATCH 226/237] [sbi] Add support for running as a RISC-V SBI payload

Add basic support for running directly on top of SBI, with no UEFI
firmware present.  Build as e.g.:

  make CROSS=riscv64-linux-gnu- bin-riscv64/ipxe.sbi

The resulting binary can be tested in QEMU using e.g.:

  qemu-system-riscv64 -M virt -cpu max -serial stdio \
                      -kernel bin-riscv64/ipxe.sbi

No drivers or executable binary formats are supported yet, but the
unit test suite may be run successfully.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/Makefile.housekeeping                    |   2 +
 src/arch/riscv/Makefile                      |   1 +
 src/arch/riscv/Makefile.sbi                  |  16 ++
 src/arch/riscv/core/stack.S                  |  45 ++++++
 src/arch/riscv/include/bits/umalloc.h        |  14 ++
 src/arch/riscv/include/ipxe/errno/sbi.h      |  19 +++
 src/arch/riscv/include/ipxe/sbi_umalloc.h    |  18 +++
 src/arch/riscv/interface/sbi/sbi_umalloc.c   |  54 +++++++
 src/arch/riscv/prefix/sbiprefix.S            | 156 +++++++++++++++++++
 src/arch/riscv/scripts/sbi.lds               | 119 ++++++++++++++
 src/arch/riscv32/Makefile.sbi                |   6 +
 src/arch/riscv32/include/ipxe/sbi/dhcparch.h |  20 +++
 src/arch/riscv64/Makefile.sbi                |   6 +
 src/arch/riscv64/include/ipxe/sbi/dhcparch.h |  20 +++
 src/config/config_sbi.c                      |   0
 src/config/defaults/sbi.h                    |  36 +++++
 16 files changed, 532 insertions(+)
 create mode 100644 src/arch/riscv/Makefile.sbi
 create mode 100644 src/arch/riscv/core/stack.S
 create mode 100644 src/arch/riscv/include/bits/umalloc.h
 create mode 100644 src/arch/riscv/include/ipxe/errno/sbi.h
 create mode 100644 src/arch/riscv/include/ipxe/sbi_umalloc.h
 create mode 100644 src/arch/riscv/interface/sbi/sbi_umalloc.c
 create mode 100644 src/arch/riscv/prefix/sbiprefix.S
 create mode 100644 src/arch/riscv/scripts/sbi.lds
 create mode 100644 src/arch/riscv32/Makefile.sbi
 create mode 100644 src/arch/riscv32/include/ipxe/sbi/dhcparch.h
 create mode 100644 src/arch/riscv64/Makefile.sbi
 create mode 100644 src/arch/riscv64/include/ipxe/sbi/dhcparch.h
 create mode 100644 src/config/config_sbi.c
 create mode 100644 src/config/defaults/sbi.h

diff --git a/src/Makefile.housekeeping b/src/Makefile.housekeeping
index ba61e2166..2b4356de1 100644
--- a/src/Makefile.housekeeping
+++ b/src/Makefile.housekeeping
@@ -352,6 +352,8 @@ arch :
 # Determine build platform
 DEFAULT_PLATFORM_i386 := pcbios
 DEFAULT_PLATFORM_x86_64 := pcbios
+DEFAULT_PLATFORM_riscv32 := sbi
+DEFAULT_PLATFORM_riscv64 := sbi
 DEFAULT_PLATFORM := $(DEFAULT_PLATFORM_$(ARCH))
 PLATFORM	:= $(firstword $(BIN_PLATFORM) $(DEFAULT_PLATFORM) none)
 CFLAGS		+= -DPLATFORM=$(PLATFORM) -DPLATFORM_$(PLATFORM)
diff --git a/src/arch/riscv/Makefile b/src/arch/riscv/Makefile
index d3ae4e84c..324e1403e 100644
--- a/src/arch/riscv/Makefile
+++ b/src/arch/riscv/Makefile
@@ -11,6 +11,7 @@ INCDIRS		:= arch/$(ARCH)/include arch/riscv/include $(INCDIRS)
 #
 SRCDIRS		+= arch/riscv/core
 SRCDIRS		+= arch/riscv/interface/sbi
+SRCDIRS		+= arch/riscv/prefix
 
 # RISCV-specific flags
 #
diff --git a/src/arch/riscv/Makefile.sbi b/src/arch/riscv/Makefile.sbi
new file mode 100644
index 000000000..dee1b6e5d
--- /dev/null
+++ b/src/arch/riscv/Makefile.sbi
@@ -0,0 +1,16 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Build a position-independent executable, with relocations required
+# only for data values.  Runtime relocations are applied by the
+# prefix code.
+#
+CFLAGS		+= -mcmodel=medany -fpie
+LDFLAGS		+= -pie --no-dynamic-linker
+
+# Linker script
+#
+LDSCRIPT	= arch/riscv/scripts/sbi.lds
+
+# Media types
+#
+MEDIA		+= sbi
diff --git a/src/arch/riscv/core/stack.S b/src/arch/riscv/core/stack.S
new file mode 100644
index 000000000..1cd1da7c5
--- /dev/null
+++ b/src/arch/riscv/core/stack.S
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * Internal stack
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+#define STACK_ALIGN 16
+
+#define STACK_SIZE 8192
+
+	.section ".stack", "aw", @nobits
+	.balign STACK_ALIGN
+	.globl _stack
+_stack:
+	.space STACK_SIZE
+	.globl _estack
+_estack:
diff --git a/src/arch/riscv/include/bits/umalloc.h b/src/arch/riscv/include/bits/umalloc.h
new file mode 100644
index 000000000..a7171ca2c
--- /dev/null
+++ b/src/arch/riscv/include/bits/umalloc.h
@@ -0,0 +1,14 @@
+#ifndef _BITS_UMALLOC_H
+#define _BITS_UMALLOC_H
+
+/** @file
+ *
+ * RISCV-specific user memory allocation API implementations
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/sbi_umalloc.h>
+
+#endif /* _BITS_UMALLOC_H */
diff --git a/src/arch/riscv/include/ipxe/errno/sbi.h b/src/arch/riscv/include/ipxe/errno/sbi.h
new file mode 100644
index 000000000..2428183d4
--- /dev/null
+++ b/src/arch/riscv/include/ipxe/errno/sbi.h
@@ -0,0 +1,19 @@
+#ifndef _IPXE_ERRNO_SBI_H
+#define _IPXE_ERRNO_SBI_H
+
+/**
+ * @file
+ *
+ * RISC-V SBI platform error codes
+ *
+ * We never need to return SBI error codes ourselves, so we
+ * arbitrarily choose to use the Linux error codes as platform error
+ * codes.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/errno/linux.h>
+
+#endif /* _IPXE_ERRNO_SBI_H */
diff --git a/src/arch/riscv/include/ipxe/sbi_umalloc.h b/src/arch/riscv/include/ipxe/sbi_umalloc.h
new file mode 100644
index 000000000..5763239fb
--- /dev/null
+++ b/src/arch/riscv/include/ipxe/sbi_umalloc.h
@@ -0,0 +1,18 @@
+#ifndef _IPXE_SBI_UMALLOC_H
+#define _IPXE_SBI_UMALLOC_H
+
+/** @file
+ *
+ * External memory allocation
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#ifdef UMALLOC_SBI
+#define UMALLOC_PREFIX_sbi
+#else
+#define UMALLOC_PREFIX_sbi __sbi_
+#endif
+
+#endif /* _IPXE_SBI_UMALLOC_H */
diff --git a/src/arch/riscv/interface/sbi/sbi_umalloc.c b/src/arch/riscv/interface/sbi/sbi_umalloc.c
new file mode 100644
index 000000000..2f9935adb
--- /dev/null
+++ b/src/arch/riscv/interface/sbi/sbi_umalloc.c
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <ipxe/umalloc.h>
+
+/** @file
+ *
+ * iPXE user memory allocation API for SBI
+ *
+ */
+
+/** Equivalent of NOWHERE for user pointers */
+#define UNOWHERE ( ~UNULL )
+
+/**
+ * Reallocate external memory
+ *
+ * @v old_ptr		Memory previously allocated by umalloc(), or UNULL
+ * @v new_size		Requested size
+ * @ret new_ptr		Allocated memory, or UNULL
+ *
+ * Calling realloc() with a new size of zero is a valid way to free a
+ * memory block.
+ */
+static userptr_t sbi_urealloc ( userptr_t old_ptr, size_t new_size ) {
+
+	/* External allocation not yet implemented: allocate from heap */
+	return ( ( userptr_t ) realloc ( ( ( void * ) old_ptr ), new_size ) );
+}
+
+PROVIDE_UMALLOC ( sbi, urealloc, sbi_urealloc );
diff --git a/src/arch/riscv/prefix/sbiprefix.S b/src/arch/riscv/prefix/sbiprefix.S
new file mode 100644
index 000000000..0de001951
--- /dev/null
+++ b/src/arch/riscv/prefix/sbiprefix.S
@@ -0,0 +1,156 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+/** @file
+ *
+ * SBI position-independent executable prefix
+ *
+ */
+
+	.section ".note.GNU-stack", "", @progbits
+	.text
+
+/* SBI debug console extension */
+#define SBI_DBCN ( ( 'D' << 24 ) | ( 'B' << 16 ) | ( 'C' << 8 ) | 'N' )
+#define SBI_DBCN_WRITE 0x00
+
+/* SBI system reset extension */
+#define SBI_SRST ( ( 'S' << 24 ) | ( 'R' << 16 ) | ( 'S' << 8 ) | 'T' )
+#define SBI_SRST_SYSTEM_RESET 0x00
+#define SBI_RESET_COLD 0x00000001
+
+/* Relative relocation type */
+#define R_RISCV_RELATIVE 3
+
+	/* Layout of a relocation record */
+	.struct 0
+rela_offset:	.space ( __riscv_xlen / 8 )
+rela_type:	.space ( __riscv_xlen / 8 )
+rela_addend:	.space ( __riscv_xlen / 8 )
+rela_len:
+	.previous
+
+	/*
+	 * Display progress message via debug console
+	 */
+	.macro	progress message
+#ifndef NDEBUG
+	.section ".prefix.data", "aw", @progbits
+progress_\@:
+	.ascii	"\message"
+	.equ	progress_\@_len, . - progress_\@
+	.size	progress_\@, . - progress_\@
+	.previous
+	li	a7, SBI_DBCN
+	li	a6, SBI_DBCN_WRITE
+	li	a0, progress_\@_len
+	la	a1, progress_\@
+	mv	a2, zero
+	ecall
+#endif
+	.endm
+
+	/*
+	 * SBI entry point
+	 */
+	.section ".prefix", "ax", @progbits
+	.org 0
+	.globl	_sbi_start
+_sbi_start:
+	/* Preserve arguments */
+	mv	s0, a0
+	mv	s1, a1
+	progress "\nSBI->iPXE"
+
+	/* Apply dynamic relocations */
+	la	t0, _reloc
+	la	t1, _ereloc
+	la	t2, _sbi_start
+1:	/* Read relocation record */
+	LOADN	t3, rela_offset(t0)
+	LOADN	t4, rela_type(t0)
+	LOADN	t5, rela_addend(t0)
+	/* Check relocation type */
+	addi	t4, t4, -R_RISCV_RELATIVE
+	bnez	t4, 2f
+	/* Apply relocation */
+	add	t3, t3, t2
+	add	t5, t5, t2
+	STOREN	t5, (t3)
+2:	/* Loop */
+	addi	t0, t0, rela_len
+	blt	t0, t1, 1b
+	progress " .reloc"
+
+	/* Zero the bss */
+	la	t0, _bss
+	la	t1, _ebss
+1:	STOREN	zero, (t0)
+	addi	t0, t0, ( __riscv_xlen / 8 )
+	blt	t0, t1, 1b
+	progress " .bss"
+
+	/* Set up stack */
+	la	sp, _estack
+	progress " .stack"
+
+	/* Store boot hart */
+	la	t0, boot_hart
+	STOREN	s0, (t0)
+
+	/* Register device tree */
+	mv	a0, s1
+	call	register_fdt
+
+	/* Call main program */
+	progress "\n\n"
+	call	main
+
+	/* We have no return path, since the M-mode SBI implementation
+	 * will have jumped to us by setting our start address in MEPC
+	 * and issuing an MRET instruction.
+	 *
+	 * Attempt a system reset, since there is nothing else we can
+	 * viably do at this point.
+	 */
+	progress "\niPXE->SBI reset\n"
+	li	a7, SBI_SRST
+	li	a6, SBI_SRST_SYSTEM_RESET
+	li	a0, SBI_RESET_COLD
+	mv	a1, zero
+	ecall
+
+	/* If reset failed, lock the system */
+	progress "(reset failed)\n"
+1:	wfi
+	j	1b
+	.size	_sbi_start, . - _sbi_start
+
+	/* File split information for the compressor */
+	.section ".zinfo", "a", @progbits
+	.ascii	"COPY"
+	.word	0
+	.word	_sbi_filesz
+	.word	1
diff --git a/src/arch/riscv/scripts/sbi.lds b/src/arch/riscv/scripts/sbi.lds
new file mode 100644
index 000000000..a65b27181
--- /dev/null
+++ b/src/arch/riscv/scripts/sbi.lds
@@ -0,0 +1,119 @@
+/*
+ * Linker script for RISC-V SBI images
+ *
+ */
+
+SECTIONS {
+
+    /* Start at virtual address zero */
+    . = 0;
+
+    /* Weak symbols that need zero values if not otherwise defined */
+    .weak 0x0 : {
+	_weak = .;
+	*(.weak)
+	*(.weak.*)
+	_eweak = .;
+    }
+    _assert = ASSERT ( ( _weak == _eweak ), ".weak is non-zero length" );
+
+    /* Prefix code */
+    .prefix : {
+        _prefix = .;
+	*(.prefix)
+	*(.prefix.*)
+	_eprefix = .;
+    }
+
+    /* Program code */
+    .text : {
+	_text = .;
+	*(.text)
+	*(.text.*)
+	_etext = .;
+    }
+
+    /* Align to page size to allow linker to generate W^X segments */
+    . = ALIGN ( 4096 );
+
+    /* Read-only data */
+    .rodata : {
+	_rodata = .;
+	*(.rodata)
+	*(.rodata.*)
+	_erodata = .;
+    }
+
+    /* Writable data */
+    .data : {
+	_data = .;
+	*(.data)
+	*(.data.*)
+	KEEP(*(SORT(.tbl.*)))	/* Various tables.  See include/tables.h */
+	KEEP(*(.provided))
+	KEEP(*(.provided.*))
+	_edata = .;
+    }
+
+    /* Uninitialised and discardable data */
+    OVERLAY : {
+
+	/* Runtime relocations (discarded after use) */
+	.rela.dyn {
+	    _reloc = .;
+	    *(.rela)
+	    *(.rela.dyn)
+	}
+
+	/* Compressor information block */
+	.zinfo {
+	    _zinfo = .;
+	    KEEP(*(.zinfo))
+	    KEEP(*(.zinfo.*))
+	    _ezinfo = .;
+	}
+
+	/* Uninitialised data */
+	.bss {
+	    _bss = .;
+	    *(.bss)
+	    *(.bss.*)
+	    *(COMMON)
+	    *(.stack)
+	    *(.stack.*)
+	    /* Align to allow for easy zeroing by prefix code */
+	    . = ALIGN ( 16 );
+	    _ebss = .;
+	}
+    }
+
+    /* Calculate end of relocations
+     *
+     * This cannot be done by placing "_ereloc = .;" inside the
+     * .rela.dyn section, since the dynamic relocations are not
+     * present in the input sections but are instead generated during
+     * linking.
+     */
+    _ereloc = ( _reloc + __load_stop_reladyn - __load_start_reladyn );
+
+    /* Length of initialised data */
+    _sbi_filesz = ABSOLUTE ( _ereloc );
+
+    /* Unwanted sections */
+    /DISCARD/ : {
+	*(.comment)
+	*(.comment.*)
+	*(.note)
+	*(.note.*)
+	*(.eh_frame)
+	*(.eh_frame.*)
+	*(.dynamic)
+	*(.dynsym)
+	*(.dynstr)
+	*(.einfo)
+	*(.einfo.*)
+	*(.discard)
+	*(.discard.*)
+	*(.pci_devlist.*)
+    }
+}
diff --git a/src/arch/riscv32/Makefile.sbi b/src/arch/riscv32/Makefile.sbi
new file mode 100644
index 000000000..d62287770
--- /dev/null
+++ b/src/arch/riscv32/Makefile.sbi
@@ -0,0 +1,6 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Include generic SBI Makefile
+#
+MAKEDEPS	+= arch/riscv/Makefile.sbi
+include arch/riscv/Makefile.sbi
diff --git a/src/arch/riscv32/include/ipxe/sbi/dhcparch.h b/src/arch/riscv32/include/ipxe/sbi/dhcparch.h
new file mode 100644
index 000000000..713d4cf5d
--- /dev/null
+++ b/src/arch/riscv32/include/ipxe/sbi/dhcparch.h
@@ -0,0 +1,20 @@
+#ifndef _IPXE_SBI_DHCPARCH_H
+#define _IPXE_SBI_DHCPARCH_H
+
+/** @file
+ *
+ * DHCP client architecture definitions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/dhcp.h>
+
+/** DHCP client architecture */
+#define DHCP_ARCH_CLIENT_ARCHITECTURE DHCP_CLIENT_ARCHITECTURE_RISCV32
+
+/** DHCP client network device interface */
+#define DHCP_ARCH_CLIENT_NDI 1 /* UNDI */ , 3, 10 /* v3.10 */
+
+#endif /* _IPXE_SBI_DHCPARCH_H */
diff --git a/src/arch/riscv64/Makefile.sbi b/src/arch/riscv64/Makefile.sbi
new file mode 100644
index 000000000..d62287770
--- /dev/null
+++ b/src/arch/riscv64/Makefile.sbi
@@ -0,0 +1,6 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Include generic SBI Makefile
+#
+MAKEDEPS	+= arch/riscv/Makefile.sbi
+include arch/riscv/Makefile.sbi
diff --git a/src/arch/riscv64/include/ipxe/sbi/dhcparch.h b/src/arch/riscv64/include/ipxe/sbi/dhcparch.h
new file mode 100644
index 000000000..e172f064f
--- /dev/null
+++ b/src/arch/riscv64/include/ipxe/sbi/dhcparch.h
@@ -0,0 +1,20 @@
+#ifndef _IPXE_SBI_DHCPARCH_H
+#define _IPXE_SBI_DHCPARCH_H
+
+/** @file
+ *
+ * DHCP client architecture definitions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/dhcp.h>
+
+/** DHCP client architecture */
+#define DHCP_ARCH_CLIENT_ARCHITECTURE DHCP_CLIENT_ARCHITECTURE_RISCV64
+
+/** DHCP client network device interface */
+#define DHCP_ARCH_CLIENT_NDI 1 /* UNDI */ , 3, 10 /* v3.10 */
+
+#endif /* _IPXE_SBI_DHCPARCH_H */
diff --git a/src/config/config_sbi.c b/src/config/config_sbi.c
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/config/defaults/sbi.h b/src/config/defaults/sbi.h
new file mode 100644
index 000000000..42fb515f0
--- /dev/null
+++ b/src/config/defaults/sbi.h
@@ -0,0 +1,36 @@
+#ifndef CONFIG_DEFAULTS_SBI_H
+#define CONFIG_DEFAULTS_SBI_H
+
+/** @file
+ *
+ * Configuration defaults for RISC-V SBI
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#define IOAPI_RISCV
+#define IOMAP_VIRT
+#define DMAAPI_FLAT
+#define UACCESS_FLAT
+#define TIMER_ZICNTR
+#define ENTROPY_ZKR
+
+#define CONSOLE_SBI
+#define REBOOT_SBI
+#define UMALLOC_SBI
+
+#define ACPI_NULL
+#define MPAPI_NULL
+#define NAP_NULL
+#define PCIAPI_NULL
+#define SANBOOT_NULL
+#define SMBIOS_NULL
+#define TIME_NULL
+
+#define IMAGE_SCRIPT
+
+#define REBOOT_CMD
+#define POWEROFF_CMD
+
+#endif /* CONFIG_DEFAULTS_SBI_H */

From 19f44d2998f6197b70fa2726c551bc8fe6956fa2 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 28 Oct 2024 22:58:56 +0000
Subject: [PATCH 227/237] [riscv] Check if seed CSR is accessible from S-mode

The seed CSR defined by the Zkr extension is accessible only in M-mode
by default.  Older versions of OpenSBI (prior to version 1.4) do not
set mseccfg.sseed, with the result that attempts to access the seed
CSR from S-mode will raise an illegal instruction exception.

Add a facility for testing the accessibility of arbitrary CSRs, and
use it to check that the seed CSR is accessible before reporting the
seed CSR entropy source as being functional.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/riscv/core/zkr.c         |  7 +++
 src/arch/riscv/include/ipxe/csr.h | 75 +++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 src/arch/riscv/include/ipxe/csr.h

diff --git a/src/arch/riscv/core/zkr.c b/src/arch/riscv/core/zkr.c
index bf1609671..4b6316ea0 100644
--- a/src/arch/riscv/core/zkr.c
+++ b/src/arch/riscv/core/zkr.c
@@ -31,6 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/hart.h>
+#include <ipxe/csr.h>
 #include <ipxe/entropy.h>
 #include <ipxe/drbg.h>
 
@@ -60,6 +61,12 @@ static int zkr_entropy_enable ( void ) {
 		return rc;
 	}
 
+	/* Check if seed CSR is accessible in S-mode */
+	if ( ! csr_can_write ( "seed", 0 ) ) {
+		DBGC ( colour, "ZKR cannot access seed CSR\n" );
+		return -ENOTSUP;
+	}
+
 	/* RISC-V ISA mandates that 128 bits of full entropy shall be
 	 * obtained from 256 entropy bits read from the seed CSR.
 	 *
diff --git a/src/arch/riscv/include/ipxe/csr.h b/src/arch/riscv/include/ipxe/csr.h
new file mode 100644
index 000000000..c14974472
--- /dev/null
+++ b/src/arch/riscv/include/ipxe/csr.h
@@ -0,0 +1,75 @@
+#ifndef _IPXE_CSR_H
+#define _IPXE_CSR_H
+
+/** @file
+ *
+ * Control and status registers (CSRs)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Check if CSR can be read
+ *
+ * @v name		CSR name
+ * @v allowed		CSR can be read
+ */
+#define csr_can_read( name ) ( {					\
+	unsigned long stvec_orig;					\
+	unsigned long stvec_temp;					\
+	unsigned long csr;						\
+	int allowed = 0;						\
+									\
+	__asm__ __volatile__ ( /* Set temporary trap vector */		\
+			       "la %3, 1f\n\t"				\
+			       "csrrw %2, stvec, %3\n\t"		\
+			       /* Try reading CSR */			\
+			       "csrr %1, " name "\n\t"			\
+			       /* Mark as allowed if not trapped */	\
+			       "addi %0, %0, 1\n\t"			\
+			       /* Temporary trap vector */		\
+			       ".balign 4\n\t"				\
+			       "\n1:\n\t"				\
+			       /* Restore original trap vector */	\
+			       "csrw stvec, %2\n\t"			\
+			       : "+r" ( allowed ),			\
+				 "=r" ( csr ),				\
+				 "=r" ( stvec_orig ),			\
+				 "=r" ( stvec_temp ) );			\
+	allowed;							\
+	} )
+
+/**
+ * Check if CSR can be written
+ *
+ * @v name		CSR name
+ * @v value		Value to write
+ * @v allowed		CSR can be written
+ */
+#define csr_can_write( name, value ) ( {				\
+	unsigned long stvec_orig;					\
+	unsigned long stvec_temp;					\
+	unsigned long csr = (value);					\
+	int allowed = 0;						\
+									\
+	__asm__ __volatile__ ( /* Set temporary trap vector */		\
+			       "la %3, 1f\n\t"				\
+			       "csrrw %2, stvec, %3\n\t"		\
+			       /* Try writing CSR */			\
+			       "csrrw %1, " name ", %1\n\t"		\
+			       /* Mark as allowed if not trapped */	\
+			       "addi %0, %0, 1\n\t"			\
+			       /* Temporary trap vector */		\
+			       ".balign 4\n\t"				\
+			       "\n1:\n\t"				\
+			       /* Restore original trap vector */	\
+			       "csrw stvec, %2\n\t"			\
+			       : "+r" ( allowed ),			\
+				 "+r" ( csr ),				\
+				 "=r" ( stvec_orig ),			\
+				 "=r" ( stvec_temp ) );			\
+	allowed;							\
+	} )
+
+#endif /* _IPXE_CSR_H */

From 8fc11d8a4ad41f15af3d081250865f971312d871 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 29 Oct 2024 12:50:37 +0000
Subject: [PATCH 228/237] [build] Allow for per-architecture cross-compilation
 prefixes

We currently require the variable CROSS (or CROSS_COMPILE) to be set
to specify the global cross-compilation prefix.  This becomes
cumbersome when developing across multiple CPU architectures,
requiring frequent editing of build command lines and preventing
incompatible architectures from being built with a single command.

Allow a default cross-compilation prefix for each architecture to be
specified via the CROSS_COMPILE_<arch> variables.  These may then be
provided as environment variables, e.g. using

  export CROSS_COMPILE_arm32=arm-linux-gnu-
  export CROSS_COMPILE_arm64=aarch64-linux-gnu-
  export CROSS_COMPILE_loong64=loongarch64-linux-gnu-
  export CROSS_COMPILE_riscv32=riscv64-linux-gnu-
  export CROSS_COMPILE_riscv64=riscv64-linux-gnu-

This change requires some portions of the Makefile to be rearranged,
to allow for the fact that $(CROSS_COMPILE) may not have been set
until the build directory has been parsed to determine the CPU
architecture.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/Makefile              |  20 +--
 src/Makefile.housekeeping | 247 ++++++++++++++++++++------------------
 2 files changed, 139 insertions(+), 128 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 95719628f..548a4e3f1 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -26,16 +26,16 @@ PRINTF		:= printf
 PERL		:= perl
 PYTHON		:= python
 TRUE		:= true
-CC		:= $(CROSS_COMPILE)gcc
-CPP		:= $(CC) -E
-AS		:= $(CROSS_COMPILE)as
-LD		:= $(CROSS_COMPILE)ld
-SIZE		:= $(CROSS_COMPILE)size
-AR		:= $(CROSS_COMPILE)ar
-RANLIB		:= $(CROSS_COMPILE)ranlib
-OBJCOPY		:= $(CROSS_COMPILE)objcopy
-NM		:= $(CROSS_COMPILE)nm
-OBJDUMP		:= $(CROSS_COMPILE)objdump
+CC		= $(CROSS_COMPILE)gcc
+CPP		= $(CC) -E
+AS		= $(CROSS_COMPILE)as
+LD		= $(CROSS_COMPILE)ld
+SIZE		= $(CROSS_COMPILE)size
+AR		= $(CROSS_COMPILE)ar
+RANLIB		= $(CROSS_COMPILE)ranlib
+OBJCOPY		= $(CROSS_COMPILE)objcopy
+NM		= $(CROSS_COMPILE)nm
+OBJDUMP		= $(CROSS_COMPILE)objdump
 OPENSSL		:= openssl
 CSPLIT		:= csplit
 PARSEROM	:= ./util/parserom.pl
diff --git a/src/Makefile.housekeeping b/src/Makefile.housekeeping
index 2b4356de1..1926920fc 100644
--- a/src/Makefile.housekeeping
+++ b/src/Makefile.housekeeping
@@ -3,6 +3,20 @@
 # This file contains various boring housekeeping functions that would
 # otherwise seriously clutter up the main Makefile.
 
+###############################################################################
+#
+# Make syntax does not allow use of comma or space in certain places.
+# This ugly workaround is suggested in the manual.
+#
+COMMA	:= ,
+EMPTY	:=
+SPACE	:= $(EMPTY) $(EMPTY)
+HASH	:= \#
+define NEWLINE
+
+
+endef
+
 ###############################################################################
 #
 # Find a usable "echo -e" substitute.
@@ -68,56 +82,6 @@ HOST_OS		:= $(shell uname -s)
 hostos :
 	@$(ECHO) $(HOST_OS)
 
-###############################################################################
-#
-# Determine compiler
-
-CCDEFS		:= $(shell $(CC) -E -x c -c /dev/null -dM | cut -d" " -f2)
-ccdefs:
-	@$(ECHO) $(CCDEFS)
-
-ifeq ($(filter __GNUC__,$(CCDEFS)),__GNUC__)
-CCTYPE		:= gcc
-endif
-cctype:
-	@$(ECHO) $(CCTYPE)
-
-###############################################################################
-#
-# Check for tools that can cause failed builds
-#
-
-ifeq ($(CCTYPE),gcc)
-GCC_2_96_BANNER := $(shell $(CC) -v 2>&1 | grep -is 'gcc version 2\.96')
-ifneq ($(GCC_2_96_BANNER),)
-$(warning gcc 2.96 is unsuitable for compiling iPXE)
-$(warning Use gcc 2.95 or a newer version instead)
-$(error Unsuitable build environment found)
-endif
-endif
-
-PERL_UNICODE_CHECK := $(shell $(PERL) -e 'use bytes; print chr(255)' | wc -c)
-ifeq ($(PERL_UNICODE_CHECK),2)
-$(warning Your Perl version has a Unicode handling bug)
-$(warning Execute this command before building iPXE:)
-$(warning export LANG=$${LANG%.UTF-8})
-$(error Unsuitable build environment found)
-endif
-
-LD_GOLD_BANNER := $(shell $(LD) -v 2>&1 | grep 'GNU gold')
-ifneq ($(LD_GOLD_BANNER),)
-$(warning GNU gold is unsuitable for building iPXE)
-$(warning Use GNU ld instead)
-$(error Unsuitable build environment found)
-endif
-
-OBJCOPY_ETC_BANNER := $(shell $(OBJCOPY) --version | grep 'elftoolchain')
-ifneq ($(OBJCOPY_ETC_BANNER),)
-$(warning The elftoolchain objcopy is unsuitable for building iPXE)
-$(warning Use binutils objcopy instead)
-$(error Unsuitable build environment found)
-endif
-
 ###############################################################################
 #
 # Check if $(eval ...) is available to use
@@ -130,74 +94,6 @@ endif
 eval :
 	@$(ECHO) $(HAVE_EVAL)
 
-###############################################################################
-#
-# Check for various tool workarounds
-#
-
-WORKAROUND_CFLAGS :=
-WORKAROUND_ASFLAGS :=
-WORKAROUND_LDFLAGS :=
-
-# Make syntax does not allow use of comma or space in certain places.
-# This ugly workaround is suggested in the manual.
-#
-COMMA	:= ,
-EMPTY	:=
-SPACE	:= $(EMPTY) $(EMPTY)
-HASH	:= \#
-define NEWLINE
-
-
-endef
-
-# gcc 4.4 generates .eh_frame sections by default, which distort the
-# output of "size".  Inhibit this.
-#
-ifeq ($(CCTYPE),gcc)
-CFI_TEST = $(CC) -fno-dwarf2-cfi-asm -fno-exceptions -fno-unwind-tables \
-		 -fno-asynchronous-unwind-tables -x c -c /dev/null \
-		 -o /dev/null >/dev/null 2>&1
-CFI_FLAGS := $(shell $(CFI_TEST) && \
-	       $(ECHO) '-fno-dwarf2-cfi-asm -fno-exceptions ' \
-		       '-fno-unwind-tables -fno-asynchronous-unwind-tables')
-WORKAROUND_CFLAGS += $(CFI_FLAGS)
-endif
-
-# gcc 4.6 generates spurious warnings if -Waddress is in force.
-# Inhibit this.
-#
-ifeq ($(CCTYPE),gcc)
-WNA_TEST = $(CC) -Waddress -x c -c /dev/null -o /dev/null >/dev/null 2>&1
-WNA_FLAGS := $(shell $(WNA_TEST) && $(ECHO) '-Wno-address')
-WORKAROUND_CFLAGS += $(WNA_FLAGS)
-
-# gcc 8.0 generates warnings for certain suspect string operations. Our
-# sources have been vetted for correct usage.  Turn off these warnings.
-#
-WNST_TEST = $(CC) -Wstringop-truncation -x c -c /dev/null -o /dev/null \
-		  >/dev/null 2>&1
-WNST_FLAGS := $(shell $(WNST_TEST) && $(ECHO) '-Wno-stringop-truncation')
-WORKAROUND_CFLAGS += $(WNST_FLAGS)
-
-# gcc 9.1 generates warnings for taking address of packed member which
-# may result in an unaligned pointer value.  Inhibit the warnings.
-#
-WNAPM_TEST = $(CC) -Wno-address-of-packed-member -x c -c /dev/null \
-		   -o /dev/null >/dev/null 2>&1
-WNAPM_FLAGS := $(shell $(WNAPM_TEST) && \
-		 $(ECHO) '-Wno-address-of-packed-member')
-WORKAROUND_CFLAGS += $(WNAPM_FLAGS)
-endif
-
-# Some versions of gas choke on division operators, treating them as
-# comment markers.  Specifying --divide will work around this problem,
-# but isn't available on older gas versions.
-#
-DIVIDE_TEST = $(AS) --divide /dev/null -o /dev/null 2>/dev/null
-DIVIDE_FLAGS := $(shell $(DIVIDE_TEST) && $(ECHO) '--divide')
-WORKAROUND_ASFLAGS += $(DIVIDE_FLAGS)
-
 ###############################################################################
 #
 # Build verbosity
@@ -367,9 +263,124 @@ CFLAGS		+= -DSECUREBOOT=$(SECUREBOOT)
 secureboot :
 	@$(ECHO) $(SECUREBOOT)
 
+# Set cross-compilation prefix automatically if not specified
+ifeq ($(CROSS_COMPILE),)
+CROSS_COMPILE	:= $(CROSS_COMPILE_$(ARCH))
+endif
+
 endif # defined(BIN)
 
+###############################################################################
+#
+# Determine compiler
+
+CCDEFS		:= $(shell $(CC) -E -x c -c /dev/null -dM | cut -d" " -f2)
+ccdefs:
+	@$(ECHO) $(CCDEFS)
+
+ifeq ($(filter __GNUC__,$(CCDEFS)),__GNUC__)
+CCTYPE		:= gcc
+endif
+cctype:
+	@$(ECHO) $(CCTYPE)
+
+###############################################################################
+#
+# Check for tools that can cause failed builds
+#
+
+ifeq ($(CCTYPE),gcc)
+GCC_2_96_BANNER := $(shell $(CC) -v 2>&1 | grep -is 'gcc version 2\.96')
+ifneq ($(GCC_2_96_BANNER),)
+$(warning gcc 2.96 is unsuitable for compiling iPXE)
+$(warning Use gcc 2.95 or a newer version instead)
+$(error Unsuitable build environment found)
+endif
+endif
+
+PERL_UNICODE_CHECK := $(shell $(PERL) -e 'use bytes; print chr(255)' | wc -c)
+ifeq ($(PERL_UNICODE_CHECK),2)
+$(warning Your Perl version has a Unicode handling bug)
+$(warning Execute this command before building iPXE:)
+$(warning export LANG=$${LANG%.UTF-8})
+$(error Unsuitable build environment found)
+endif
+
+LD_GOLD_BANNER := $(shell $(LD) -v 2>&1 | grep 'GNU gold')
+ifneq ($(LD_GOLD_BANNER),)
+$(warning GNU gold is unsuitable for building iPXE)
+$(warning Use GNU ld instead)
+$(error Unsuitable build environment found)
+endif
+
+OBJCOPY_ETC_BANNER := $(shell $(OBJCOPY) --version | grep 'elftoolchain')
+ifneq ($(OBJCOPY_ETC_BANNER),)
+$(warning The elftoolchain objcopy is unsuitable for building iPXE)
+$(warning Use binutils objcopy instead)
+$(error Unsuitable build environment found)
+endif
+
+###############################################################################
+#
+# Check for various tool workarounds
+#
+
+WORKAROUND_CFLAGS :=
+WORKAROUND_ASFLAGS :=
+WORKAROUND_LDFLAGS :=
+
+# gcc 4.4 generates .eh_frame sections by default, which distort the
+# output of "size".  Inhibit this.
+#
+ifeq ($(CCTYPE),gcc)
+CFI_TEST = $(CC) -fno-dwarf2-cfi-asm -fno-exceptions -fno-unwind-tables \
+		 -fno-asynchronous-unwind-tables -x c -c /dev/null \
+		 -o /dev/null >/dev/null 2>&1
+CFI_FLAGS := $(shell $(CFI_TEST) && \
+	       $(ECHO) '-fno-dwarf2-cfi-asm -fno-exceptions ' \
+		       '-fno-unwind-tables -fno-asynchronous-unwind-tables')
+WORKAROUND_CFLAGS += $(CFI_FLAGS)
+endif
+
+# gcc 4.6 generates spurious warnings if -Waddress is in force.
+# Inhibit this.
+#
+ifeq ($(CCTYPE),gcc)
+WNA_TEST = $(CC) -Waddress -x c -c /dev/null -o /dev/null >/dev/null 2>&1
+WNA_FLAGS := $(shell $(WNA_TEST) && $(ECHO) '-Wno-address')
+WORKAROUND_CFLAGS += $(WNA_FLAGS)
+
+# gcc 8.0 generates warnings for certain suspect string operations. Our
+# sources have been vetted for correct usage.  Turn off these warnings.
+#
+WNST_TEST = $(CC) -Wstringop-truncation -x c -c /dev/null -o /dev/null \
+		  >/dev/null 2>&1
+WNST_FLAGS := $(shell $(WNST_TEST) && $(ECHO) '-Wno-stringop-truncation')
+WORKAROUND_CFLAGS += $(WNST_FLAGS)
+
+# gcc 9.1 generates warnings for taking address of packed member which
+# may result in an unaligned pointer value.  Inhibit the warnings.
+#
+WNAPM_TEST = $(CC) -Wno-address-of-packed-member -x c -c /dev/null \
+		   -o /dev/null >/dev/null 2>&1
+WNAPM_FLAGS := $(shell $(WNAPM_TEST) && \
+		 $(ECHO) '-Wno-address-of-packed-member')
+WORKAROUND_CFLAGS += $(WNAPM_FLAGS)
+endif
+
+# Some versions of gas choke on division operators, treating them as
+# comment markers.  Specifying --divide will work around this problem,
+# but isn't available on older gas versions.
+#
+DIVIDE_TEST = $(AS) --divide /dev/null -o /dev/null 2>/dev/null
+DIVIDE_FLAGS := $(shell $(DIVIDE_TEST) && $(ECHO) '--divide')
+WORKAROUND_ASFLAGS += $(DIVIDE_FLAGS)
+
+###############################################################################
+#
 # Include architecture-specific Makefile
+#
+
 ifdef ARCH
 MAKEDEPS	+= arch/$(ARCH)/Makefile
 include arch/$(ARCH)/Makefile

From e9a23a5b394f40c1525c40416105eaaa1787f749 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 20 Nov 2024 14:21:16 +0000
Subject: [PATCH 229/237] [efi] Ensure local drives are connected when
 attempting a SAN boot

UEFI systems may choose not to connect drivers for local disk drives
when the boot policy is set to attempt a network boot.  This may cause
the "sanboot" command to be unable to boot from a local drive, since
the relevant block device and filesystem drivers may not have been
connected.

Fix by ensuring that all available drivers are connected before
attempting to boot from an EFI block device.

Reported-by: Andrew Cottrell <andrew.cottrell@xtxmarkets.com>
Tested-by: Andrew Cottrell <andrew.cottrell@xtxmarkets.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/interface/efi/efi_block.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index 2f0187a0a..6296953c5 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -990,6 +990,9 @@ static int efi_block_boot ( unsigned int drive,
 	EFI_STATUS efirc;
 	int rc;
 
+	/* Ensure that any local drives are connected */
+	efi_driver_reconnect_all();
+
 	/* Release SNP devices */
 	efi_snp_release();
 

From da6da6eb3b83fe92002e9c8e245933498ba19a48 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 7 Nov 2024 14:43:56 +0000
Subject: [PATCH 230/237] [crypto] Add bigint_msb_is_set() to clarify code

Add a dedicated bigint_msb_is_set() to reduce the amount of open
coding required in the common case of testing the sign of a two's
complement big integer.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       |  5 ++---
 src/crypto/x25519.c       |  3 +--
 src/include/ipxe/bigint.h | 27 +++++++++++++++++++++++++++
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index 3f5db8517..35701a1d0 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -162,14 +162,13 @@ void bigint_reduce_raw ( const bigint_element_t *minuend0,
 		bigint_t ( minuend_size ) modulus;
 	} *temp = tmp;
 	const unsigned int width = ( 8 * sizeof ( bigint_element_t ) );
-	const bigint_element_t msb_mask = ( 1UL << ( width - 1 ) );
 	bigint_element_t *element;
 	unsigned int minuend_max;
 	unsigned int modulus_max;
 	unsigned int subshift;
-	bigint_element_t msb;
 	int offset;
 	int shift;
+	int msb;
 	int i;
 
 	/* Start profiling */
@@ -289,7 +288,7 @@ void bigint_reduce_raw ( const bigint_element_t *minuend0,
 		} else {
 			bigint_subtract ( &temp->modulus, &temp->minuend );
 		}
-		msb = ( temp->minuend.element[ minuend_size - 1 ] & msb_mask );
+		msb = bigint_msb_is_set ( &temp->minuend );
 		if ( shift > 0 )
 			bigint_shr ( &temp->modulus );
 	}
diff --git a/src/crypto/x25519.c b/src/crypto/x25519.c
index d58f7168c..19f9a2c02 100644
--- a/src/crypto/x25519.c
+++ b/src/crypto/x25519.c
@@ -563,7 +563,6 @@ void x25519_invert ( const union x25519_oct258 *invertend,
  * @v value		Big integer to be subtracted from, if possible
  */
 static void x25519_reduce_by ( const x25519_t *subtrahend, x25519_t *value ) {
-	unsigned int max_bit = ( ( 8 * sizeof ( *value ) ) - 1 );
 	x25519_t tmp;
 
 	/* Conditionally subtract subtrahend
@@ -573,7 +572,7 @@ static void x25519_reduce_by ( const x25519_t *subtrahend, x25519_t *value ) {
 	 */
 	bigint_copy ( value, &tmp );
 	bigint_subtract ( subtrahend, value );
-	bigint_swap ( value, &tmp, bigint_bit_is_set ( value, max_bit ) );
+	bigint_swap ( value, &tmp, bigint_msb_is_set ( value ) );
 }
 
 /**
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index f7900e0e6..a85761815 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -142,6 +142,16 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	unsigned int size = bigint_size (value);			\
 	bigint_bit_is_set_raw ( (value)->element, size, bit ); } )
 
+/**
+ * Test if most significant bit is set in big integer
+ *
+ * @v value		Big integer
+ * @ret is_set		Most significant bit is set
+ */
+#define bigint_msb_is_set( value ) ( {					\
+	unsigned int size = bigint_size (value);			\
+	bigint_msb_is_set_raw ( (value)->element, size ); } )
+
 /**
  * Find highest bit set in big integer
  *
@@ -358,6 +368,23 @@ bigint_bit_is_set_raw ( const bigint_element_t *value0, unsigned int size,
 	return ( !! ( value->element[index] & ( 1UL << subindex ) ) );
 }
 
+/**
+ * Test if most significant bit is set in big integer
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ * @ret is_set		Most significant bit is set
+ */
+static inline __attribute__ (( always_inline )) int
+bigint_msb_is_set_raw ( const bigint_element_t *value0, unsigned int size ) {
+	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( const void * ) value0 );
+	unsigned int index = ( size - 1 );
+	unsigned int subindex = ( ( 8 * sizeof ( value->element[0] ) ) - 1 );
+
+	return ( !! ( value->element[index] & ( 1UL << subindex ) ) );
+}
+
 void bigint_init_raw ( bigint_element_t *value0, unsigned int size,
 		       const void *data, size_t len );
 void bigint_done_raw ( const bigint_element_t *value0, unsigned int size,

From 167a08f08928c7e469f50d5d364287abb784e99c Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 26 Nov 2024 12:53:01 +0000
Subject: [PATCH 231/237] [crypto] Expose carry flag from big integer addition
 and subtraction

Expose the effective carry (or borrow) out flag from big integer
addition and subtraction, and use this to elide an explicit bit test
when performing x25519 reduction.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/arch/arm32/include/bits/bigint.h   | 23 ++++++---
 src/arch/arm64/include/bits/bigint.h   | 19 ++++---
 src/arch/loong64/include/bits/bigint.h | 36 ++++++++------
 src/arch/riscv/include/bits/bigint.h   | 36 ++++++++------
 src/arch/x86/include/bits/bigint.h     | 20 +++++---
 src/crypto/x25519.c                    |  5 +-
 src/include/ipxe/bigint.h              | 18 ++++---
 src/tests/bigint_test.c                | 68 +++++++++++++++++---------
 8 files changed, 140 insertions(+), 85 deletions(-)

diff --git a/src/arch/arm32/include/bits/bigint.h b/src/arch/arm32/include/bits/bigint.h
index 39d3dc347..95de32d83 100644
--- a/src/arch/arm32/include/bits/bigint.h
+++ b/src/arch/arm32/include/bits/bigint.h
@@ -43,8 +43,9 @@ bigint_init_raw ( uint32_t *value0, unsigned int size,
  * @v addend0		Element 0 of big integer to add
  * @v value0		Element 0 of big integer to be added to
  * @v size		Number of elements
+ * @ret carry		Carry out
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_add_raw ( const uint32_t *addend0, uint32_t *value0,
 		 unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -54,8 +55,9 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0,
 	uint32_t *discard_end;
 	uint32_t discard_addend_i;
 	uint32_t discard_value_i;
+	int carry;
 
-	__asm__ __volatile__ ( "adds %2, %0, %8, lsl #2\n\t" /* clear CF */
+	__asm__ __volatile__ ( "adds %2, %0, %9, lsl #2\n\t" /* clear CF */
 			       "\n1:\n\t"
 			       "ldmia %0!, {%3}\n\t"
 			       "ldr %4, [%1]\n\t"
@@ -68,9 +70,11 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0,
 				 "=l" ( discard_end ),
 				 "=l" ( discard_addend_i ),
 				 "=l" ( discard_value_i ),
+				 "=@cccs" ( carry ),
 				 "+m" ( *value )
-			       : "0" ( addend0 ), "1" ( value0 ), "l" ( size )
-			       : "cc" );
+			       : "0" ( addend0 ), "1" ( value0 ),
+				 "l" ( size ) );
+	return carry;
 }
 
 /**
@@ -79,8 +83,9 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0,
  * @v subtrahend0	Element 0 of big integer to subtract
  * @v value0		Element 0 of big integer to be subtracted from
  * @v size		Number of elements
+ * @ret borrow		Borrow out
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0,
 		      unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -90,8 +95,9 @@ bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0,
 	uint32_t *discard_end;
 	uint32_t discard_subtrahend_i;
 	uint32_t discard_value_i;
+	int borrow;
 
-	__asm__ __volatile__ ( "add %2, %0, %8, lsl #2\n\t"
+	__asm__ __volatile__ ( "add %2, %0, %9, lsl #2\n\t"
 			       "cmp %2, %0\n\t" /* set CF */
 			       "\n1:\n\t"
 			       "ldmia %0!, {%3}\n\t"
@@ -105,10 +111,11 @@ bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0,
 				 "=l" ( discard_end ),
 				 "=l" ( discard_subtrahend_i ),
 				 "=l" ( discard_value_i ),
+				 "=@cccc" ( borrow ),
 				 "+m" ( *value )
 			       : "0" ( subtrahend0 ), "1" ( value0 ),
-				 "l" ( size )
-			       : "cc" );
+				 "l" ( size ) );
+	return borrow;
 }
 
 /**
diff --git a/src/arch/arm64/include/bits/bigint.h b/src/arch/arm64/include/bits/bigint.h
index 50493499e..5f79dc0c3 100644
--- a/src/arch/arm64/include/bits/bigint.h
+++ b/src/arch/arm64/include/bits/bigint.h
@@ -43,8 +43,9 @@ bigint_init_raw ( uint64_t *value0, unsigned int size,
  * @v addend0		Element 0 of big integer to add
  * @v value0		Element 0 of big integer to be added to
  * @v size		Number of elements
+ * @ret carry		Carry out
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 		 unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -54,6 +55,7 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 	uint64_t discard_addend_i;
 	uint64_t discard_value_i;
 	unsigned int discard_size;
+	int carry;
 
 	__asm__ __volatile__ ( "cmn xzr, xzr\n\t" /* clear CF */
 			       "\n1:\n\t"
@@ -68,9 +70,11 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 				 "=r" ( discard_size ),
 				 "=r" ( discard_addend_i ),
 				 "=r" ( discard_value_i ),
+				 "=@cccs" ( carry ),
 				 "+m" ( *value )
-			       : "0" ( addend0 ), "1" ( value0 ), "2" ( size )
-			       : "cc" );
+			       : "0" ( addend0 ), "1" ( value0 ),
+				 "2" ( size ) );
+	return carry;
 }
 
 /**
@@ -79,8 +83,9 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
  * @v subtrahend0	Element 0 of big integer to subtract
  * @v value0		Element 0 of big integer to be subtracted from
  * @v size		Number of elements
+ * @ret borrow		Borrow out
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 		      unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -90,6 +95,7 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 	uint64_t discard_subtrahend_i;
 	uint64_t discard_value_i;
 	unsigned int discard_size;
+	int borrow;
 
 	__asm__ __volatile__ ( "cmp xzr, xzr\n\t" /* set CF */
 			       "\n1:\n\t"
@@ -104,10 +110,11 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 				 "=r" ( discard_size ),
 				 "=r" ( discard_subtrahend_i ),
 				 "=r" ( discard_value_i ),
+				 "=@cccc" ( borrow ),
 				 "+m" ( *value )
 			       : "0" ( subtrahend0 ), "1" ( value0 ),
-				 "2" ( size )
-			       : "cc" );
+				 "2" ( size ) );
+	return borrow;
 }
 
 /**
diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h
index 234d8dfa7..0222354df 100644
--- a/src/arch/loong64/include/bits/bigint.h
+++ b/src/arch/loong64/include/bits/bigint.h
@@ -43,8 +43,9 @@ bigint_init_raw ( uint64_t *value0, unsigned int size,
  * @v addend0		Element 0 of big integer to add
  * @v value0		Element 0 of big integer to be added to
  * @v size		Number of elements
+ * @ret carry		Carry out
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 		 unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -53,20 +54,20 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 	uint64_t *discard_value;
 	uint64_t discard_addend_i;
 	uint64_t discard_value_i;
-	uint64_t discard_carry;
 	uint64_t discard_temp;
 	unsigned int discard_size;
+	uint64_t carry;
 
 	__asm__ __volatile__ ( "\n1:\n\t"
 			       /* Load addend[i] and value[i] */
 			       "ld.d %3, %0, 0\n\t"
 			       "ld.d %4, %1, 0\n\t"
 			       /* Add carry flag and addend */
-			       "add.d %4, %4, %5\n\t"
-			       "sltu %6, %4, %5\n\t"
+			       "add.d %4, %4, %6\n\t"
+			       "sltu %5, %4, %6\n\t"
 			       "add.d %4, %4, %3\n\t"
-			       "sltu %5, %4, %3\n\t"
-			       "or %5, %5, %6\n\t"
+			       "sltu %6, %4, %3\n\t"
+			       "or %6, %5, %6\n\t"
 			       /* Store value[i] */
 			       "st.d %4, %1, 0\n\t"
 			       /* Loop */
@@ -79,11 +80,12 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 				 "=r" ( discard_size ),
 				 "=r" ( discard_addend_i ),
 				 "=r" ( discard_value_i ),
-				 "=r" ( discard_carry ),
 				 "=r" ( discard_temp ),
+				 "=r" ( carry ),
 				 "+m" ( *value )
 			       : "0" ( addend0 ), "1" ( value0 ),
-				 "2" ( size ), "5" ( 0 ) );
+				 "2" ( size ), "6" ( 0 ) );
+	return carry;
 }
 
 /**
@@ -92,8 +94,9 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
  * @v subtrahend0	Element 0 of big integer to subtract
  * @v value0		Element 0 of big integer to be subtracted from
  * @v size		Number of elements
+ * @ret borrow		Borrow out
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 		      unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -102,20 +105,20 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 	uint64_t *discard_value;
 	uint64_t discard_subtrahend_i;
 	uint64_t discard_value_i;
-	uint64_t discard_carry;
 	uint64_t discard_temp;
 	unsigned int discard_size;
+	uint64_t borrow;
 
 	__asm__ __volatile__ ( "\n1:\n\t"
 			       /* Load subtrahend[i] and value[i] */
 			       "ld.d %3, %0, 0\n\t"
 			       "ld.d %4, %1, 0\n\t"
 			       /* Subtract carry flag and subtrahend */
-			       "sltu %6, %4, %5\n\t"
-			       "sub.d %4, %4, %5\n\t"
-			       "sltu %5, %4, %3\n\t"
+			       "sltu %5, %4, %6\n\t"
+			       "sub.d %4, %4, %6\n\t"
+			       "sltu %6, %4, %3\n\t"
 			       "sub.d %4, %4, %3\n\t"
-			       "or %5, %5, %6\n\t"
+			       "or %6, %5, %6\n\t"
 			       /* Store value[i] */
 			       "st.d %4, %1, 0\n\t"
 			       /* Loop */
@@ -128,11 +131,12 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 				 "=r" ( discard_size ),
 				 "=r" ( discard_subtrahend_i ),
 				 "=r" ( discard_value_i ),
-				 "=r" ( discard_carry ),
 				 "=r" ( discard_temp ),
+				 "=r" ( borrow ),
 				 "+m" ( *value )
 			       : "0" ( subtrahend0 ), "1" ( value0 ),
-				 "2" ( size ), "5" ( 0 ) );
+				 "2" ( size ), "6" ( 0 ) );
+	return borrow;
 }
 
 /**
diff --git a/src/arch/riscv/include/bits/bigint.h b/src/arch/riscv/include/bits/bigint.h
index 5b22b9ac7..ab1070d88 100644
--- a/src/arch/riscv/include/bits/bigint.h
+++ b/src/arch/riscv/include/bits/bigint.h
@@ -43,8 +43,9 @@ bigint_init_raw ( unsigned long *value0, unsigned int size,
  * @v addend0		Element 0 of big integer to add
  * @v value0		Element 0 of big integer to be added to
  * @v size		Number of elements
+ * @ret carry		Carry out
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_add_raw ( const unsigned long *addend0, unsigned long *value0,
 		 unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -54,19 +55,19 @@ bigint_add_raw ( const unsigned long *addend0, unsigned long *value0,
 	unsigned long *discard_value;
 	unsigned long discard_addend_i;
 	unsigned long discard_value_i;
-	unsigned long discard_carry;
 	unsigned long discard_temp;
+	unsigned long carry;
 
 	__asm__ __volatile__ ( "\n1:\n\t"
 			       /* Load addend[i] and value[i] */
 			       LOADN " %2, (%0)\n\t"
 			       LOADN " %3, (%1)\n\t"
 			       /* Add carry flag and addend */
-			       "add %3, %3, %4\n\t"
-			       "sltu %5, %3, %4\n\t"
+			       "add %3, %3, %5\n\t"
+			       "sltu %4, %3, %5\n\t"
 			       "add %3, %3, %2\n\t"
-			       "sltu %4, %3, %2\n\t"
-			       "or %4, %4, %5\n\t"
+			       "sltu %5, %3, %2\n\t"
+			       "or %5, %4, %5\n\t"
 			       /* Store value[i] */
 			       STOREN " %3, (%1)\n\t"
 			       /* Loop */
@@ -77,12 +78,13 @@ bigint_add_raw ( const unsigned long *addend0, unsigned long *value0,
 				 "=&r" ( discard_value ),
 				 "=&r" ( discard_addend_i ),
 				 "=&r" ( discard_value_i ),
-				 "=&r" ( discard_carry ),
 				 "=&r" ( discard_temp ),
+				 "=&r" ( carry ),
 				 "+m" ( *value )
 			       : "r" ( valueN ),
 				 "i" ( sizeof ( unsigned long ) ),
-				 "0" ( addend0 ), "1" ( value0 ), "4" ( 0 ) );
+				 "0" ( addend0 ), "1" ( value0 ), "5" ( 0 ) );
+	return carry;
 }
 
 /**
@@ -91,8 +93,9 @@ bigint_add_raw ( const unsigned long *addend0, unsigned long *value0,
  * @v subtrahend0	Element 0 of big integer to subtract
  * @v value0		Element 0 of big integer to be subtracted from
  * @v size		Number of elements
+ * @ret borrow		Borrow out
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_subtract_raw ( const unsigned long *subtrahend0, unsigned long *value0,
 		      unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -102,19 +105,19 @@ bigint_subtract_raw ( const unsigned long *subtrahend0, unsigned long *value0,
 	unsigned long *discard_value;
 	unsigned long discard_subtrahend_i;
 	unsigned long discard_value_i;
-	unsigned long discard_carry;
 	unsigned long discard_temp;
+	unsigned long borrow;
 
 	__asm__ __volatile__ ( "\n1:\n\t"
 			       /* Load subtrahend[i] and value[i] */
 			       LOADN " %2, (%0)\n\t"
 			       LOADN " %3, (%1)\n\t"
 			       /* Subtract carry flag and subtrahend */
-			       "sltu %5, %3, %4\n\t"
-			       "sub %3, %3, %4\n\t"
-			       "sltu %4, %3, %2\n\t"
+			       "sltu %4, %3, %5\n\t"
+			       "sub %3, %3, %5\n\t"
+			       "sltu %5, %3, %2\n\t"
 			       "sub %3, %3, %2\n\t"
-			       "or %4, %4, %5\n\t"
+			       "or %5, %5, %4\n\t"
 			       /* Store value[i] */
 			       STOREN " %3, (%1)\n\t"
 			       /* Loop */
@@ -125,13 +128,14 @@ bigint_subtract_raw ( const unsigned long *subtrahend0, unsigned long *value0,
 				 "=&r" ( discard_value ),
 				 "=&r" ( discard_subtrahend_i ),
 				 "=&r" ( discard_value_i ),
-				 "=&r" ( discard_carry ),
 				 "=&r" ( discard_temp ),
+				 "=&r" ( borrow ),
 				 "+m" ( *value )
 			       : "r" ( valueN ),
 				 "i" ( sizeof ( unsigned long ) ),
 				 "0" ( subtrahend0 ), "1" ( value0 ),
-				 "4" ( 0 ) );
+				 "5" ( 0 ) );
+	return borrow;
 }
 
 /**
diff --git a/src/arch/x86/include/bits/bigint.h b/src/arch/x86/include/bits/bigint.h
index 5580030ee..4026ca481 100644
--- a/src/arch/x86/include/bits/bigint.h
+++ b/src/arch/x86/include/bits/bigint.h
@@ -52,8 +52,9 @@ bigint_init_raw ( uint32_t *value0, unsigned int size,
  * @v addend0		Element 0 of big integer to add
  * @v value0		Element 0 of big integer to be added to
  * @v size		Number of elements
+ * @ret carry		Carry flag
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_add_raw ( const uint32_t *addend0, uint32_t *value0,
 		 unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -61,17 +62,20 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0,
 	long index;
 	void *discard_S;
 	long discard_c;
+	int carry;
 
 	__asm__ __volatile__ ( "xor %0, %0\n\t" /* Zero %0 and clear CF */
 			       "\n1:\n\t"
 			       "lodsl\n\t"
-			       "adcl %%eax, (%4,%0,4)\n\t"
+			       "adcl %%eax, (%5,%0,4)\n\t"
 			       "inc %0\n\t" /* Does not affect CF */
 			       "loop 1b\n\t"
 			       : "=&r" ( index ), "=&S" ( discard_S ),
-				 "=&c" ( discard_c ), "+m" ( *value )
+				 "=&c" ( discard_c ), "=@ccc" ( carry ),
+				 "+m" ( *value )
 			       : "r" ( value0 ), "1" ( addend0 ), "2" ( size )
 			       : "eax" );
+	return carry;
 }
 
 /**
@@ -80,8 +84,9 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0,
  * @v subtrahend0	Element 0 of big integer to subtract
  * @v value0		Element 0 of big integer to be subtracted from
  * @v size		Number of elements
+ * @ret borrow		Borrow flag
  */
-static inline __attribute__ (( always_inline )) void
+static inline __attribute__ (( always_inline )) int
 bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0,
 		      unsigned int size ) {
 	bigint_t ( size ) __attribute__ (( may_alias )) *value =
@@ -89,18 +94,21 @@ bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0,
 	long index;
 	void *discard_S;
 	long discard_c;
+	int borrow;
 
 	__asm__ __volatile__ ( "xor %0, %0\n\t" /* Zero %0 and clear CF */
 			       "\n1:\n\t"
 			       "lodsl\n\t"
-			       "sbbl %%eax, (%4,%0,4)\n\t"
+			       "sbbl %%eax, (%5,%0,4)\n\t"
 			       "inc %0\n\t" /* Does not affect CF */
 			       "loop 1b\n\t"
 			       : "=&r" ( index ), "=&S" ( discard_S ),
-				 "=&c" ( discard_c ), "+m" ( *value )
+				 "=&c" ( discard_c ), "=@ccc" ( borrow ),
+				 "+m" ( *value )
 			       : "r" ( value0 ), "1" ( subtrahend0 ),
 				 "2" ( size )
 			       : "eax" );
+	return borrow;
 }
 
 /**
diff --git a/src/crypto/x25519.c b/src/crypto/x25519.c
index 19f9a2c02..ab5d2e8b0 100644
--- a/src/crypto/x25519.c
+++ b/src/crypto/x25519.c
@@ -564,6 +564,7 @@ void x25519_invert ( const union x25519_oct258 *invertend,
  */
 static void x25519_reduce_by ( const x25519_t *subtrahend, x25519_t *value ) {
 	x25519_t tmp;
+	int underflow;
 
 	/* Conditionally subtract subtrahend
 	 *
@@ -571,8 +572,8 @@ static void x25519_reduce_by ( const x25519_t *subtrahend, x25519_t *value ) {
 	 * time) if the subtraction underflows.
 	 */
 	bigint_copy ( value, &tmp );
-	bigint_subtract ( subtrahend, value );
-	bigint_swap ( value, &tmp, bigint_msb_is_set ( value ) );
+	underflow = bigint_subtract ( subtrahend, value );
+	bigint_swap ( value, &tmp, underflow );
 }
 
 /**
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index a85761815..2a0a200c5 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -70,23 +70,25 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  * @v addend		Big integer to add
  * @v value		Big integer to be added to
+ * @ret carry		Carry out
  */
-#define bigint_add( addend, value ) do {				\
+#define bigint_add( addend, value ) ( {					\
 	unsigned int size = bigint_size (addend);			\
 	bigint_add_raw ( (addend)->element, (value)->element, size );	\
-	} while ( 0 )
+	} )
 
 /**
  * Subtract big integers
  *
  * @v subtrahend	Big integer to subtract
  * @v value		Big integer to be subtracted from
+ * @ret borrow		Borrow out
  */
-#define bigint_subtract( subtrahend, value ) do {			\
+#define bigint_subtract( subtrahend, value ) ( {			\
 	unsigned int size = bigint_size (subtrahend);			\
 	bigint_subtract_raw ( (subtrahend)->element, (value)->element,	\
 			      size );					\
-	} while ( 0 )
+	} )
 
 /**
  * Shift big integer left
@@ -389,10 +391,10 @@ void bigint_init_raw ( bigint_element_t *value0, unsigned int size,
 		       const void *data, size_t len );
 void bigint_done_raw ( const bigint_element_t *value0, unsigned int size,
 		       void *out, size_t len );
-void bigint_add_raw ( const bigint_element_t *addend0,
-		      bigint_element_t *value0, unsigned int size );
-void bigint_subtract_raw ( const bigint_element_t *subtrahend0,
-			   bigint_element_t *value0, unsigned int size );
+int bigint_add_raw ( const bigint_element_t *addend0,
+		     bigint_element_t *value0, unsigned int size );
+int bigint_subtract_raw ( const bigint_element_t *subtrahend0,
+			  bigint_element_t *value0, unsigned int size );
 void bigint_shl_raw ( bigint_element_t *value0, unsigned int size );
 void bigint_shr_raw ( bigint_element_t *value0, unsigned int size );
 int bigint_is_zero_raw ( const bigint_element_t *value0, unsigned int size );
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 166c771b9..271d76723 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -58,24 +58,24 @@ void bigint_done_sample ( const bigint_element_t *value0, unsigned int size,
 	bigint_done ( value, out, len );
 }
 
-void bigint_add_sample ( const bigint_element_t *addend0,
-			 bigint_element_t *value0, unsigned int size ) {
+int bigint_add_sample ( const bigint_element_t *addend0,
+			bigint_element_t *value0, unsigned int size ) {
 	const bigint_t ( size ) *addend __attribute__ (( may_alias ))
 		= ( ( const void * ) addend0 );
 	bigint_t ( size ) *value __attribute__ (( may_alias ))
 		= ( ( void * ) value0 );
 
-	bigint_add ( addend, value );
+	return bigint_add ( addend, value );
 }
 
-void bigint_subtract_sample ( const bigint_element_t *subtrahend0,
-			      bigint_element_t *value0, unsigned int size ) {
+int bigint_subtract_sample ( const bigint_element_t *subtrahend0,
+			     bigint_element_t *value0, unsigned int size ) {
 	const bigint_t ( size ) *subtrahend __attribute__ (( may_alias ))
 		= ( ( const void * ) subtrahend0 );
 	bigint_t ( size ) *value __attribute__ (( may_alias ))
 		= ( ( void * ) value0 );
 
-	bigint_subtract ( subtrahend, value );
+	return bigint_subtract ( subtrahend, value );
 }
 
 void bigint_shl_sample ( bigint_element_t *value0, unsigned int size ) {
@@ -253,16 +253,19 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
  * @v addend		Big integer to add
  * @v value		Big integer to be added to
  * @v expected		Big integer expected result
+ * @v overflow		Expected result overflows range
  */
-#define bigint_add_ok( addend, value, expected ) do {			\
+#define bigint_add_ok( addend, value, expected, overflow ) do {		\
 	static const uint8_t addend_raw[] = addend;			\
 	static const uint8_t value_raw[] = value;			\
 	static const uint8_t expected_raw[] = expected;			\
 	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
 	unsigned int size =						\
 		bigint_required_size ( sizeof ( value_raw ) );		\
+	unsigned int msb = ( 8 * sizeof ( value_raw ) );		\
 	bigint_t ( size ) addend_temp;					\
 	bigint_t ( size ) value_temp;					\
+	int carry;							\
 	{} /* Fix emacs alignment */					\
 									\
 	assert ( bigint_size ( &addend_temp ) ==			\
@@ -273,12 +276,15 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	DBG ( "Add:\n" );						\
 	DBG_HDA ( 0, &addend_temp, sizeof ( addend_temp ) );		\
 	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
-	bigint_add ( &addend_temp, &value_temp );			\
+	carry = bigint_add ( &addend_temp, &value_temp );		\
 	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
 	bigint_done ( &value_temp, result_raw, sizeof ( result_raw ) );	\
 									\
 	ok ( memcmp ( result_raw, expected_raw,				\
 		      sizeof ( result_raw ) ) == 0 );			\
+	if ( sizeof ( result_raw ) < sizeof ( value_temp ) )		\
+		carry += bigint_bit_is_set ( &value_temp, msb );	\
+	ok ( carry == overflow );					\
 	} while ( 0 )
 
 /**
@@ -287,8 +293,10 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
  * @v subtrahend	Big integer to subtract
  * @v value		Big integer to be subtracted from
  * @v expected		Big integer expected result
+ * @v underflow		Expected result underflows range
  */
-#define bigint_subtract_ok( subtrahend, value, expected ) do {		\
+#define bigint_subtract_ok( subtrahend, value, expected,		\
+			    underflow ) do {				\
 	static const uint8_t subtrahend_raw[] = subtrahend;		\
 	static const uint8_t value_raw[] = value;			\
 	static const uint8_t expected_raw[] = expected;			\
@@ -297,6 +305,7 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 		bigint_required_size ( sizeof ( value_raw ) );		\
 	bigint_t ( size ) subtrahend_temp;				\
 	bigint_t ( size ) value_temp;					\
+	int borrow;							\
 	{} /* Fix emacs alignment */					\
 									\
 	assert ( bigint_size ( &subtrahend_temp ) ==			\
@@ -307,12 +316,13 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	DBG ( "Subtract:\n" );						\
 	DBG_HDA ( 0, &subtrahend_temp, sizeof ( subtrahend_temp ) );	\
 	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
-	bigint_subtract ( &subtrahend_temp, &value_temp );		\
+	borrow = bigint_subtract ( &subtrahend_temp, &value_temp );	\
 	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
 	bigint_done ( &value_temp, result_raw, sizeof ( result_raw ) );	\
 									\
 	ok ( memcmp ( result_raw, expected_raw,				\
 		      sizeof ( result_raw ) ) == 0 );			\
+	ok ( borrow == underflow );					\
 	} while ( 0 )
 
 /**
@@ -724,16 +734,28 @@ static void bigint_test_exec ( void ) {
 
 	bigint_add_ok ( BIGINT ( 0x8a ),
 			BIGINT ( 0x43 ),
-			BIGINT ( 0xcd ) );
+			BIGINT ( 0xcd ), 0 );
 	bigint_add_ok ( BIGINT ( 0xc5, 0x7b ),
 			BIGINT ( 0xd6, 0xb1 ),
-			BIGINT ( 0x9c, 0x2c ) );
+			BIGINT ( 0x9c, 0x2c ), 1 );
 	bigint_add_ok ( BIGINT ( 0xf9, 0xd9, 0xdc ),
 			BIGINT ( 0x6d, 0x4b, 0xca ),
-			BIGINT ( 0x67, 0x25, 0xa6 ) );
+			BIGINT ( 0x67, 0x25, 0xa6 ), 1 );
 	bigint_add_ok ( BIGINT ( 0xdd, 0xc2, 0x20, 0x5f ),
 			BIGINT ( 0x80, 0x32, 0xc4, 0xb0 ),
-			BIGINT ( 0x5d, 0xf4, 0xe5, 0x0f ) );
+			BIGINT ( 0x5d, 0xf4, 0xe5, 0x0f ), 1 );
+	bigint_add_ok ( BIGINT ( 0x5e, 0x46, 0x4d, 0xc6, 0xa2, 0x7d, 0x45,
+				 0xc3 ),
+			BIGINT ( 0xd6, 0xc0, 0xd7, 0xd4, 0xf6, 0x04, 0x47,
+				 0xed ),
+			BIGINT ( 0x35, 0x07, 0x25, 0x9b, 0x98, 0x81, 0x8d,
+				 0xb0 ), 1 );
+	bigint_add_ok ( BIGINT ( 0x0e, 0x46, 0x4d, 0xc6, 0xa2, 0x7d, 0x45,
+				 0xc3 ),
+			BIGINT ( 0xd6, 0xc0, 0xd7, 0xd4, 0xf6, 0x04, 0x47,
+				 0xed ),
+			BIGINT ( 0xe5, 0x07, 0x25, 0x9b, 0x98, 0x81, 0x8d,
+				 0xb0 ), 0 );
 	bigint_add_ok ( BIGINT ( 0x01, 0xed, 0x45, 0x4b, 0x41, 0xeb, 0x4c,
 				 0x2e, 0x53, 0x07, 0x15, 0x51, 0x56, 0x47,
 				 0x29, 0xfc, 0x9c, 0xbd, 0xbd, 0xfb, 0x1b,
@@ -745,7 +767,7 @@ static void bigint_test_exec ( void ) {
 			BIGINT ( 0x75, 0xdb, 0x41, 0x80, 0x73, 0x0e, 0x23,
 				 0xe0, 0x3d, 0x98, 0x70, 0x36, 0x11, 0x03,
 				 0xcb, 0x35, 0x0f, 0x6c, 0x09, 0x17, 0xdc,
-				 0xd6, 0xd0 ) );
+				 0xd6, 0xd0 ), 0 );
 	bigint_add_ok ( BIGINT ( 0x06, 0x8e, 0xd6, 0x18, 0xbb, 0x4b, 0x0c,
 				 0xc5, 0x85, 0xde, 0xee, 0x9b, 0x3f, 0x65,
 				 0x63, 0x86, 0xf5, 0x5a, 0x9f, 0xa2, 0xd7,
@@ -802,19 +824,19 @@ static void bigint_test_exec ( void ) {
 				 0x68, 0x76, 0xf5, 0x20, 0xa1, 0xa8, 0x1a,
 				 0x9f, 0x60, 0x58, 0xff, 0xb6, 0x76, 0x45,
 				 0xe6, 0xed, 0x61, 0x8d, 0xe6, 0xc0, 0x72,
-				 0x1c, 0x07 ) );
+				 0x1c, 0x07 ), 0 );
 	bigint_subtract_ok ( BIGINT ( 0x83 ),
 			     BIGINT ( 0x50 ),
-			     BIGINT ( 0xcd ) );
+			     BIGINT ( 0xcd ), 1 );
 	bigint_subtract_ok ( BIGINT ( 0x2c, 0x7c ),
 			     BIGINT ( 0x49, 0x0e ),
-			     BIGINT ( 0x1c, 0x92 ) );
+			     BIGINT ( 0x1c, 0x92 ), 0 );
 	bigint_subtract_ok ( BIGINT ( 0x9c, 0x30, 0xbf ),
 			     BIGINT ( 0xde, 0x4e, 0x07 ),
-			     BIGINT ( 0x42, 0x1d, 0x48 ) );
+			     BIGINT ( 0x42, 0x1d, 0x48 ), 0 );
 	bigint_subtract_ok ( BIGINT ( 0xbb, 0x77, 0x32, 0x5a ),
 			     BIGINT ( 0x5a, 0xd5, 0xfe, 0x28 ),
-			     BIGINT ( 0x9f, 0x5e, 0xcb, 0xce ) );
+			     BIGINT ( 0x9f, 0x5e, 0xcb, 0xce ), 1 );
 	bigint_subtract_ok ( BIGINT ( 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 				      0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 				      0xff, 0xff, 0xff, 0xff, 0xff, 0xff ),
@@ -823,7 +845,7 @@ static void bigint_test_exec ( void ) {
 				      0x00, 0x00, 0x00, 0x00, 0x00, 0x2a ),
 			     BIGINT ( 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-				      0x00, 0x00, 0x00, 0x00, 0x00, 0x2b ) );
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x2b ), 1 );
 	bigint_subtract_ok ( BIGINT ( 0x7b, 0xaa, 0x16, 0xcf, 0x15, 0x87,
 				      0xe0, 0x4f, 0x2c, 0xa3, 0xec, 0x2f,
 				      0x46, 0xfb, 0x83, 0xc6, 0xe0, 0xee,
@@ -835,7 +857,7 @@ static void bigint_test_exec ( void ) {
 			     BIGINT ( 0xca, 0xab, 0x9f, 0x54, 0x4e, 0x48,
 				      0x75, 0x8c, 0x63, 0x28, 0x69, 0x78,
 				      0xe8, 0x8a, 0x3d, 0xd8, 0x4b, 0x24,
-				      0xb8, 0xa4, 0x71, 0x6d, 0x6b ) );
+				      0xb8, 0xa4, 0x71, 0x6d, 0x6b ), 1 );
 	bigint_subtract_ok ( BIGINT ( 0x5b, 0x06, 0x77, 0x7b, 0xfd, 0x34,
 				      0x5f, 0x0f, 0xd9, 0xbd, 0x8e, 0x5d,
 				      0xc8, 0x4a, 0x70, 0x95, 0x1b, 0xb6,
@@ -901,7 +923,7 @@ static void bigint_test_exec ( void ) {
 				      0x29, 0x8c, 0x43, 0x9f, 0xf0, 0x9d,
 				      0xda, 0xc8, 0x8c, 0x71, 0x86, 0x97,
 				      0x7f, 0xcb, 0x94, 0x31, 0x1d, 0xbc,
-				      0x44, 0x1a ) );
+				      0x44, 0x1a ), 0 );
 	bigint_shl_ok ( BIGINT ( 0xe0 ),
 			BIGINT ( 0xc0 ) );
 	bigint_shl_ok ( BIGINT ( 0x43, 0x1d ),

From 9cbf5c4f86b45773badec2498fac22e8bc6d7dd1 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Tue, 26 Nov 2024 14:45:51 +0000
Subject: [PATCH 232/237] [crypto] Eliminate temporary working space for
 bigint_reduce()

Direct modular reduction is expected to be used in situations where
there is no requirement to retain the original (unreduced) value.

Modify the API for bigint_reduce() to reduce the value in place,
(removing the separate result buffer), impose a constraint that the
modulus and value have the same size, and require the modulus to be
passed in writable memory (to allow for scaling in place).  This
removes the requirement for additional temporary working space.

Reverse the order of arguments so that the constant input is first,
to match the usage pattern for bigint_add() et al.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       | 71 ++++++++++++---------------------
 src/include/ipxe/bigint.h | 34 ++++------------
 src/tests/bigint_test.c   | 84 +++++++++++++++++++--------------------
 3 files changed, 72 insertions(+), 117 deletions(-)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index 35701a1d0..4b37c062b 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -139,32 +139,20 @@ void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 /**
  * Reduce big integer
  *
- * @v minuend0		Element 0 of big integer to be reduced
- * @v minuend_size	Number of elements in minuend
  * @v modulus0		Element 0 of big integer modulus
- * @v modulus_size	Number of elements in modulus and result
- * @v result0		Element 0 of big integer to hold result
- * @v tmp		Temporary working space
+ * @v value0		Element 0 of big integer to be reduced
+ * @v size		Number of elements in modulus and value
  */
-void bigint_reduce_raw ( const bigint_element_t *minuend0,
-			 unsigned int minuend_size,
-			 const bigint_element_t *modulus0,
-			 unsigned int modulus_size,
-			 bigint_element_t *result0, void *tmp ) {
-	const bigint_t ( minuend_size ) __attribute__ (( may_alias ))
-		*minuend = ( ( const void * ) minuend0 );
-	const bigint_t ( modulus_size ) __attribute__ (( may_alias ))
-		*modulus = ( ( const void * ) modulus0 );
-	bigint_t ( modulus_size ) __attribute__ (( may_alias ))
-		*result = ( ( void * ) result0 );
-	struct {
-		bigint_t ( minuend_size ) minuend;
-		bigint_t ( minuend_size ) modulus;
-	} *temp = tmp;
+void bigint_reduce_raw ( bigint_element_t *modulus0, bigint_element_t *value0,
+			 unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias ))
+		*modulus = ( ( void * ) modulus0 );
+	bigint_t ( size ) __attribute__ (( may_alias ))
+		*value = ( ( void * ) value0 );
 	const unsigned int width = ( 8 * sizeof ( bigint_element_t ) );
 	bigint_element_t *element;
-	unsigned int minuend_max;
 	unsigned int modulus_max;
+	unsigned int value_max;
 	unsigned int subshift;
 	int offset;
 	int shift;
@@ -174,31 +162,23 @@ void bigint_reduce_raw ( const bigint_element_t *minuend0,
 	/* Start profiling */
 	profile_start ( &bigint_mod_profiler );
 
-	/* Sanity check */
-	assert ( minuend_size >= modulus_size );
-	assert ( sizeof ( *temp ) == bigint_reduce_tmp_len ( minuend ) );
-
-	/* Copy minuend and modulus to temporary working space */
-	bigint_shrink ( minuend, &temp->minuend );
-	bigint_grow ( modulus, &temp->modulus );
-
 	/* Normalise the modulus
 	 *
 	 * Scale the modulus by shifting left such that both modulus
-	 * "m" and minuend "x" have the same most significant set bit.
-	 * (If this is not possible, then the minuend is already less
+	 * "m" and value "x" have the same most significant set bit.
+	 * (If this is not possible, then the value is already less
 	 * than the modulus, and we may therefore skip reduction
 	 * completely.)
 	 */
-	minuend_max = bigint_max_set_bit ( minuend );
+	value_max = bigint_max_set_bit ( value );
 	modulus_max = bigint_max_set_bit ( modulus );
-	shift = ( minuend_max - modulus_max );
+	shift = ( value_max - modulus_max );
 	if ( shift < 0 )
 		goto skip;
 	subshift = ( shift & ( width - 1 ) );
 	offset = ( shift / width );
-	element = temp->modulus.element;
-	for ( i = ( ( minuend_max - 1 ) / width ) ; ; i-- ) {
+	element = modulus->element;
+	for ( i = ( ( value_max - 1 ) / width ) ; ; i-- ) {
 		element[i] = ( element[ i - offset ] << subshift );
 		if ( i <= offset )
 			break;
@@ -210,14 +190,14 @@ void bigint_reduce_raw ( const bigint_element_t *minuend0,
 	for ( i-- ; i >= 0 ; i-- )
 		element[i] = 0;
 
-	/* Reduce the minuend "x" by iteratively adding or subtracting
+	/* Reduce the value "x" by iteratively adding or subtracting
 	 * the scaled modulus "m".
 	 *
 	 * On each loop iteration, we maintain the invariant:
 	 *
 	 *    -2m <= x < 2m
 	 *
-	 * If x is positive, we obtain the new minuend x' by
+	 * If x is positive, we obtain the new value x' by
 	 * subtracting m, otherwise we add m:
 	 *
 	 *      0 <= x < 2m   =>   x' := x - m   =>   -m <= x' < m
@@ -284,21 +264,18 @@ void bigint_reduce_raw ( const bigint_element_t *minuend0,
 	 */
 	for ( msb = 0 ; ( msb || ( shift >= 0 ) ) ; shift-- ) {
 		if ( msb ) {
-			bigint_add ( &temp->modulus, &temp->minuend );
+			bigint_add ( modulus, value );
 		} else {
-			bigint_subtract ( &temp->modulus, &temp->minuend );
+			bigint_subtract ( modulus, value );
 		}
-		msb = bigint_msb_is_set ( &temp->minuend );
+		msb = bigint_msb_is_set ( value );
 		if ( shift > 0 )
-			bigint_shr ( &temp->modulus );
+			bigint_shr ( modulus );
 	}
 
  skip:
 	/* Sanity check */
-	assert ( ! bigint_is_geq ( &temp->minuend, &temp->modulus ) );
-
-	/* Copy result */
-	bigint_shrink ( &temp->minuend, result );
+	assert ( ! bigint_is_geq ( value, modulus ) );
 
 	/* Stop profiling */
 	profile_stop ( &bigint_mod_profiler );
@@ -396,7 +373,9 @@ void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 	bigint_multiply ( multiplicand, multiplier, &temp->result );
 
 	/* Reduce result */
-	bigint_reduce ( &temp->result, modulus, result, temp );
+	bigint_grow ( modulus, &temp->modulus );
+	bigint_reduce ( &temp->modulus, &temp->result );
+	bigint_shrink ( &temp->result, result );
 
 	/* Sanity check */
 	assert ( ! bigint_is_geq ( result, modulus ) );
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 2a0a200c5..330d7deec 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -232,32 +232,15 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /**
  * Reduce big integer
  *
- * @v minuend		Big integer to be reduced
  * @v modulus		Big integer modulus
- * @v result		Big integer to hold result
- * @v tmp		Temporary working space
+ * @v value		Big integer to be reduced
  */
-#define bigint_reduce( minuend, modulus, result, tmp ) do {		\
-	unsigned int minuend_size = bigint_size (minuend);		\
-	unsigned int modulus_size = bigint_size (modulus);		\
-	bigint_reduce_raw ( (minuend)->element, minuend_size,		\
-			    (modulus)->element, modulus_size,		\
-			    (result)->element, tmp );			\
+#define bigint_reduce( modulus, value ) do {	\
+		unsigned int size = bigint_size (modulus);		\
+		bigint_reduce_raw ( (modulus)->element,			\
+				    (value)->element, size );		\
 	} while ( 0 )
 
-/**
- * Calculate temporary working space required for reduction
- *
- * @v minuend		Big integer to be reduced
- * @ret len		Length of temporary working space
- */
-#define bigint_reduce_tmp_len( minuend ) ( {				\
-	unsigned int size = bigint_size (minuend);			\
-	sizeof ( struct {						\
-		bigint_t ( size ) temp_minuend;				\
-		bigint_t ( size ) temp_modulus;				\
-	} ); } )
-
 /**
  * Compute inverse of odd big integer modulo its own size
  *
@@ -422,11 +405,8 @@ void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 			   const bigint_element_t *multiplier0,
 			   unsigned int multiplier_size,
 			   bigint_element_t *result0 );
-void bigint_reduce_raw ( const bigint_element_t *minuend0,
-			 unsigned int minuend_size,
-			 const bigint_element_t *modulus0,
-			 unsigned int modulus_size,
-			 bigint_element_t *result0, void *tmp );
+void bigint_reduce_raw ( bigint_element_t *modulus0, bigint_element_t *value0,
+			 unsigned int size );
 void bigint_mod_invert_raw ( const bigint_element_t *invertend0,
 			     bigint_element_t *inverse0,
 			     unsigned int size, void *tmp );
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 271d76723..61f78fff9 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -185,19 +185,14 @@ void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
 	bigint_multiply ( multiplicand, multiplier, result );
 }
 
-void bigint_reduce_sample ( const bigint_element_t *minuend0,
-			    unsigned int minuend_size,
-			    const bigint_element_t *modulus0,
-			    unsigned int modulus_size,
-			    bigint_element_t *result0, void *tmp ) {
-	const bigint_t ( minuend_size ) __attribute__ (( may_alias ))
-		*minuend = ( ( const void * ) minuend0 );
-	const bigint_t ( modulus_size ) __attribute__ (( may_alias ))
-		*modulus = ( ( const void * ) modulus0 );
-	bigint_t ( modulus_size ) __attribute__ (( may_alias ))
-		*result = ( ( void * ) result0 );
+void bigint_reduce_sample ( bigint_element_t *modulus0,
+			    bigint_element_t *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias ))
+		*modulus = ( ( void * ) modulus0 );
+	bigint_t ( size ) __attribute__ (( may_alias ))
+		*value = ( ( void * ) value0 );
 
-	bigint_reduce ( minuend, modulus, result, tmp );
+	bigint_reduce ( modulus, value );
 }
 
 void bigint_mod_invert_sample ( const bigint_element_t *invertend0,
@@ -555,43 +550,40 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 /**
  * Report result of big integer modular direct reduction test
  *
- * @v minuend		Big integer to be reduced
  * @v modulus		Big integer modulus
+ * @v value		Big integer to be reduced
  * @v expected		Big integer expected result
  */
-#define bigint_reduce_ok( minuend, modulus, expected ) do {		\
-	static const uint8_t minuend_raw[] = minuend;			\
+#define bigint_reduce_ok( modulus, value, expected ) do {		\
 	static const uint8_t modulus_raw[] = modulus;			\
+	static const uint8_t value_raw[] = value;			\
 	static const uint8_t expected_raw[] = expected;			\
 	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
-	unsigned int minuend_size =					\
-		bigint_required_size ( sizeof ( minuend_raw ) );	\
-	unsigned int modulus_size =					\
+	unsigned int size =						\
 		bigint_required_size ( sizeof ( modulus_raw ) );	\
-	bigint_t ( minuend_size ) minuend_temp;				\
-	bigint_t ( modulus_size ) modulus_temp;				\
-	bigint_t ( modulus_size ) result_temp;				\
-	size_t tmp_len = bigint_reduce_tmp_len ( &minuend_temp );	\
-	uint8_t tmp[tmp_len];						\
+	bigint_t ( size ) modulus_temp;					\
+	bigint_t ( size ) value_temp;					\
 	{} /* Fix emacs alignment */					\
 									\
-	assert ( bigint_size ( &result_temp ) ==			\
-		 bigint_size ( &modulus_temp ) );			\
-	bigint_init ( &minuend_temp, minuend_raw,			\
-		      sizeof ( minuend_raw ) );				\
+	assert ( bigint_size ( &modulus_temp ) ==			\
+		 bigint_size ( &value_temp ) );				\
 	bigint_init ( &modulus_temp, modulus_raw,			\
 		      sizeof ( modulus_raw ) );				\
+	bigint_init ( &value_temp, value_raw, sizeof ( value_raw ) );	\
 	DBG ( "Modular reduce:\n" );					\
-	DBG_HDA ( 0, &minuend_temp, sizeof ( minuend_temp ) );		\
 	DBG_HDA ( 0, &modulus_temp, sizeof ( modulus_temp ) );		\
-	bigint_reduce ( &minuend_temp, &modulus_temp, &result_temp,	\
-			tmp );						\
-	DBG_HDA ( 0, &result_temp, sizeof ( result_temp ) );		\
-	bigint_done ( &result_temp, result_raw,				\
-		      sizeof ( result_raw ) );				\
+	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
+	bigint_reduce ( &modulus_temp, &value_temp );			\
+	DBG_HDA ( 0, &value_temp, sizeof ( value_temp ) );		\
+	bigint_done ( &value_temp, result_raw, sizeof ( result_raw ) );	\
 									\
 	ok ( memcmp ( result_raw, expected_raw,				\
 		      sizeof ( result_raw ) ) == 0 );			\
+									\
+	bigint_init ( &value_temp, modulus_raw,				\
+		      sizeof ( modulus_raw ) );				\
+	ok ( memcmp ( &modulus_temp, &value_temp,			\
+		      sizeof ( modulus_temp ) ) == 0 );			\
 	} while ( 0 )
 
 /**
@@ -1797,16 +1789,16 @@ static void bigint_test_exec ( void ) {
 				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 				      0x00, 0x00, 0x00, 0x01 ) );
-	bigint_reduce_ok ( BIGINT ( 0x00 ),
-			   BIGINT ( 0xaf ),
+	bigint_reduce_ok ( BIGINT ( 0xaf ),
+			   BIGINT ( 0x00 ),
 			   BIGINT ( 0x00 ) );
 	bigint_reduce_ok ( BIGINT ( 0xab ),
 			   BIGINT ( 0xab ),
 			   BIGINT ( 0x00 ) );
-	bigint_reduce_ok ( BIGINT ( 0x1d, 0x97, 0x63, 0xc9, 0x97, 0xcd, 0x43,
-				    0xcb, 0x8e, 0x71, 0xac, 0x41, 0xdd ),
-			   BIGINT ( 0xcc, 0x9d, 0xa0, 0x79, 0x96, 0x6a, 0x46,
+	bigint_reduce_ok ( BIGINT ( 0xcc, 0x9d, 0xa0, 0x79, 0x96, 0x6a, 0x46,
 				    0xd5, 0xb4, 0x30, 0xd2, 0x2b, 0xbf ),
+			   BIGINT ( 0x1d, 0x97, 0x63, 0xc9, 0x97, 0xcd, 0x43,
+				    0xcb, 0x8e, 0x71, 0xac, 0x41, 0xdd ),
 			   BIGINT ( 0x1d, 0x97, 0x63, 0xc9, 0x97, 0xcd, 0x43,
 				    0xcb, 0x8e, 0x71, 0xac, 0x41, 0xdd ) );
 	bigint_reduce_ok ( BIGINT ( 0x21, 0xfa, 0x4f, 0xce, 0x0f, 0x0f, 0x4d,
@@ -1815,15 +1807,19 @@ static void bigint_test_exec ( void ) {
 				    0x43, 0xaa, 0xad, 0x21, 0x30, 0xe5 ),
 			   BIGINT ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 				    0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ) );
-	bigint_reduce_ok ( BIGINT ( 0xf9, 0x78, 0x96, 0x39, 0xee, 0x98, 0x42,
+	bigint_reduce_ok ( BIGINT ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				    0x00, 0x00, 0x00, 0xf3, 0x65, 0x35, 0x41,
+				    0x66, 0x65 ),
+			   BIGINT ( 0xf9, 0x78, 0x96, 0x39, 0xee, 0x98, 0x42,
 				    0x6a, 0xb8, 0x74, 0x0b, 0xe8, 0x5c, 0x76,
 				    0x34, 0xaf ),
-			   BIGINT ( 0xf3, 0x65, 0x35, 0x41, 0x66, 0x65 ),
-			   BIGINT ( 0xb3, 0x07, 0xe8, 0xb7, 0x01, 0xf6 ) );
-	bigint_reduce_ok ( BIGINT ( 0xfe, 0x30, 0xe1, 0xc6, 0x65, 0x97, 0x48,
-				    0x2e, 0x94, 0xd4 ),
-			   BIGINT ( 0x47, 0xaa, 0x88, 0x00, 0xd0, 0x30, 0x62,
+			   BIGINT ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				    0x00, 0x00, 0x00, 0xb3, 0x07, 0xe8, 0xb7,
+				    0x01, 0xf6 ) );
+	bigint_reduce_ok ( BIGINT ( 0x47, 0xaa, 0x88, 0x00, 0xd0, 0x30, 0x62,
 				    0xfb, 0x5d, 0x55 ),
+			   BIGINT ( 0xfe, 0x30, 0xe1, 0xc6, 0x65, 0x97, 0x48,
+				    0x2e, 0x94, 0xd4 ),
 			   BIGINT ( 0x27, 0x31, 0x49, 0xc3, 0xf5, 0x06, 0x1f,
 				    0x3c, 0x7c, 0xd5 ) );
 	bigint_mod_invert_ok ( BIGINT ( 0x01 ), BIGINT ( 0x01 ) );

From 7c2e68cc87a552c153e13517b0d0d6827f48e95b Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 27 Nov 2024 12:51:04 +0000
Subject: [PATCH 233/237] [crypto] Eliminate temporary working space for
 bigint_mod_invert()

With a slight modification to the algorithm to ignore bits of the
residue that can never contribute to the result, it is possible to
reuse the as-yet uncalculated portions of the inverse to hold the
residue.  This removes the requirement for additional temporary
working space.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       | 63 +++++++++++++++++++++++++--------------
 src/include/ipxe/bigint.h | 24 ++++-----------
 src/tests/bigint_test.c   | 24 +++++++++++----
 3 files changed, 65 insertions(+), 46 deletions(-)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index 4b37c062b..735fcdf61 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -287,27 +287,22 @@ void bigint_reduce_raw ( bigint_element_t *modulus0, bigint_element_t *value0,
  * @v invertend0	Element 0 of odd big integer to be inverted
  * @v inverse0		Element 0 of big integer to hold result
  * @v size		Number of elements in invertend and result
- * @v tmp		Temporary working space
  */
 void bigint_mod_invert_raw ( const bigint_element_t *invertend0,
-			     bigint_element_t *inverse0,
-			     unsigned int size, void *tmp ) {
+			     bigint_element_t *inverse0, unsigned int size ) {
 	const bigint_t ( size ) __attribute__ (( may_alias ))
 		*invertend = ( ( const void * ) invertend0 );
 	bigint_t ( size ) __attribute__ (( may_alias ))
 		*inverse = ( ( void * ) inverse0 );
-	struct {
-		bigint_t ( size ) residue;
-	} *temp = tmp;
-	const unsigned int width = ( 8 * sizeof ( bigint_element_t ) );
+	bigint_element_t accum;
+	bigint_element_t bit;
 	unsigned int i;
 
 	/* Sanity check */
-	assert ( invertend->element[0] & 1 );
+	assert ( bigint_bit_is_set ( invertend, 0 ) );
 
-	/* Initialise temporary working space and output value */
-	memset ( &temp->residue, 0xff, sizeof ( temp->residue ) );
-	memset ( inverse, 0, sizeof ( *inverse ) );
+	/* Initialise output */
+	memset ( inverse, 0xff, sizeof ( *inverse ) );
 
 	/* Compute inverse modulo 2^(width)
 	 *
@@ -315,23 +310,47 @@ void bigint_mod_invert_raw ( const bigint_element_t *invertend0,
 	 * presented in "A New Algorithm for Inversion mod p^k (Koç,
 	 * 2017)".
 	 *
-	 * Each loop iteration calculates one bit of the inverse.  The
-	 * residue value is the two's complement negation of the value
-	 * "b" as used by Koç, to allow for division by two using a
-	 * logical right shift (since we have no arithmetic right
-	 * shift operation for big integers).
+	 * Each inner loop iteration calculates one bit of the
+	 * inverse.  The residue value is the two's complement
+	 * negation of the value "b" as used by Koç, to allow for
+	 * division by two using a logical right shift (since we have
+	 * no arithmetic right shift operation for big integers).
+	 *
+	 * The residue is stored in the as-yet uncalculated portion of
+	 * the inverse.  The size of the residue therefore decreases
+	 * by one element for each outer loop iteration.  Trivial
+	 * inspection of the algorithm shows that any higher bits
+	 * could not contribute to the eventual output value, and so
+	 * we may safely reuse storage this way.
 	 *
 	 * Due to the suffix property of inverses mod 2^k, the result
 	 * represents the least significant bits of the inverse modulo
 	 * an arbitrarily large 2^k.
 	 */
-	for ( i = 0 ; i < ( 8 * sizeof ( *inverse ) ) ; i++ ) {
-		if ( temp->residue.element[0] & 1 ) {
-			inverse->element[ i / width ] |=
-				( 1UL << ( i % width ) );
-			bigint_add ( invertend, &temp->residue );
+	for ( i = size ; i > 0 ; i-- ) {
+		const bigint_t ( i ) __attribute__ (( may_alias ))
+			*addend = ( ( const void * ) invertend );
+		bigint_t ( i ) __attribute__ (( may_alias ))
+			*residue = ( ( void * ) inverse );
+
+		/* Calculate one element's worth of inverse bits */
+		for ( accum = 0, bit = 1 ; bit ; bit <<= 1 ) {
+			if ( bigint_bit_is_set ( residue, 0 ) ) {
+				accum |= bit;
+				bigint_add ( addend, residue );
+			}
+			bigint_shr ( residue );
 		}
-		bigint_shr ( &temp->residue );
+
+		/* Store in the element no longer required to hold residue */
+		inverse->element[ i - 1 ] = accum;
+	}
+
+	/* Correct order of inverse elements */
+	for ( i = 0 ; i < ( size / 2 ) ; i++ ) {
+		accum = inverse->element[i];
+		inverse->element[i] = inverse->element[ size - 1 - i ];
+		inverse->element[ size - 1 - i ] = accum;
 	}
 }
 
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 330d7deec..e55c536c7 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -242,30 +242,17 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	} while ( 0 )
 
 /**
- * Compute inverse of odd big integer modulo its own size
+ * Compute inverse of odd big integer modulo any power of two
  *
  * @v invertend		Odd big integer to be inverted
  * @v inverse		Big integer to hold result
- * @v tmp		Temporary working space
  */
-#define bigint_mod_invert( invertend, inverse, tmp ) do {		\
-	unsigned int size = bigint_size (invertend);			\
+#define bigint_mod_invert( invertend, inverse ) do {			\
+	unsigned int size = bigint_size ( invertend );			\
 	bigint_mod_invert_raw ( (invertend)->element,			\
-				(inverse)->element, size, tmp );	\
+				(inverse)->element, size );		\
 	} while ( 0 )
 
-/**
- * Calculate temporary working space required for modular inversion
- *
- * @v invertend		Odd big integer to be inverted
- * @ret len		Length of temporary working space
- */
-#define bigint_mod_invert_tmp_len( invertend ) ( {			\
-	unsigned int size = bigint_size (invertend);			\
-	sizeof ( struct {						\
-		bigint_t ( size ) temp_residue;				\
-	} ); } )
-
 /**
  * Perform modular multiplication of big integers
  *
@@ -408,8 +395,7 @@ void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
 void bigint_reduce_raw ( bigint_element_t *modulus0, bigint_element_t *value0,
 			 unsigned int size );
 void bigint_mod_invert_raw ( const bigint_element_t *invertend0,
-			     bigint_element_t *inverse0,
-			     unsigned int size, void *tmp );
+			     bigint_element_t *inverse0, unsigned int size );
 void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 			       const bigint_element_t *multiplier0,
 			       const bigint_element_t *modulus0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 61f78fff9..608d8e874 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -197,13 +197,13 @@ void bigint_reduce_sample ( bigint_element_t *modulus0,
 
 void bigint_mod_invert_sample ( const bigint_element_t *invertend0,
 				bigint_element_t *inverse0,
-				unsigned int size, void *tmp ) {
+				unsigned int size ) {
 	const bigint_t ( size ) __attribute__ (( may_alias ))
 		*invertend = ( ( const void * ) invertend0 );
 	bigint_t ( size ) __attribute__ (( may_alias ))
 		*inverse = ( ( void * ) inverse0 );
 
-	bigint_mod_invert ( invertend, inverse, tmp );
+	bigint_mod_invert ( invertend, inverse );
 }
 
 void bigint_mod_multiply_sample ( const bigint_element_t *multiplicand0,
@@ -600,8 +600,6 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 		bigint_required_size ( sizeof ( invertend_raw ) );	\
 	bigint_t ( size ) invertend_temp;				\
 	bigint_t ( size ) inverse_temp;					\
-	size_t tmp_len = bigint_mod_invert_tmp_len ( &invertend_temp );	\
-	uint8_t tmp[tmp_len];						\
 	{} /* Fix emacs alignment */					\
 									\
 	assert ( bigint_size ( &invertend_temp ) ==			\
@@ -610,7 +608,7 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 		      sizeof ( invertend_raw ) );			\
 	DBG ( "Modular invert:\n" );					\
 	DBG_HDA ( 0, &invertend_temp, sizeof ( invertend_temp ) );	\
-	bigint_mod_invert ( &invertend_temp, &inverse_temp, tmp );	\
+	bigint_mod_invert ( &invertend_temp, &inverse_temp );		\
 	DBG_HDA ( 0, &inverse_temp, sizeof ( inverse_temp ) );		\
 	bigint_done ( &inverse_temp, inverse_raw,			\
 		      sizeof ( inverse_raw ) );				\
@@ -1827,6 +1825,10 @@ static void bigint_test_exec ( void ) {
 					0xff, 0xff ),
 			       BIGINT ( 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 					0xff, 0xff ) );
+	bigint_mod_invert_ok ( BIGINT ( 0xa4, 0xcb, 0xbc, 0xc9, 0x9f, 0x7a,
+					0x65, 0xbf ),
+			       BIGINT ( 0xb9, 0xd5, 0xf4, 0x88, 0x0b, 0xf8,
+					0x8a, 0x3f ) );
 	bigint_mod_invert_ok ( BIGINT ( 0x95, 0x6a, 0xc5, 0xe7, 0x2e, 0x5b,
 					0x44, 0xed, 0xbf, 0x7e, 0xfe, 0x8d,
 					0xf4, 0x5a, 0x48, 0xc1 ),
@@ -1839,6 +1841,18 @@ static void bigint_test_exec ( void ) {
 			       BIGINT ( 0xf2, 0x9c, 0x63, 0x29, 0xfa, 0xe4,
 					0xbf, 0x90, 0xa6, 0x9a, 0xec, 0xcf,
 					0x5f, 0xe2, 0x21, 0xcd ) );
+	bigint_mod_invert_ok ( BIGINT ( 0xb9, 0xbb, 0x7f, 0x9c, 0x7a, 0x32,
+					0x43, 0xed, 0x9d, 0xd4, 0x0d, 0x6f,
+					0x32, 0xfa, 0x4b, 0x62, 0x38, 0x3a,
+					0xbf, 0x4c, 0xbd, 0xa8, 0x47, 0xce,
+					0xa2, 0x30, 0x34, 0xe0, 0x2c, 0x09,
+					0x14, 0x89 ),
+			       BIGINT ( 0xfc, 0x05, 0xc4, 0x2a, 0x90, 0x99,
+					0x82, 0xf8, 0x81, 0x1d, 0x87, 0xb8,
+					0xca, 0xe4, 0x95, 0xe2, 0xac, 0x18,
+					0xb3, 0xe1, 0x3e, 0xc6, 0x5a, 0x03,
+					0x51, 0x6f, 0xb7, 0xe3, 0xa5, 0xd6,
+					0xa1, 0xb9 ) );
 	bigint_mod_multiply_ok ( BIGINT ( 0x37 ),
 				 BIGINT ( 0x67 ),
 				 BIGINT ( 0x3f ),

From 96f385d7a48ffe259295991043a86b2cefce1891 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 27 Nov 2024 12:56:22 +0000
Subject: [PATCH 234/237] [crypto] Use inverse size as effective size for
 bigint_mod_invert()

Montgomery reduction requires only the least significant element of an
inverse modulo 2^k, which in turn depends upon only the least
significant element of the invertend.

Use the inverse size (rather than the invertend size) as the effective
size for bigint_mod_invert().  This eliminates around 97% of the loop
iterations for a typical 2048-bit RSA modulus.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/include/ipxe/bigint.h |  2 +-
 src/tests/bigint_test.c   | 15 ++++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index e55c536c7..14f3c5f28 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -248,7 +248,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v inverse		Big integer to hold result
  */
 #define bigint_mod_invert( invertend, inverse ) do {			\
-	unsigned int size = bigint_size ( invertend );			\
+	unsigned int size = bigint_size ( inverse );			\
 	bigint_mod_invert_raw ( (invertend)->element,			\
 				(inverse)->element, size );		\
 	} while ( 0 )
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 608d8e874..bee36de25 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -596,14 +596,14 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	static const uint8_t invertend_raw[] = invertend;		\
 	static const uint8_t expected_raw[] = expected;			\
 	uint8_t inverse_raw[ sizeof ( expected_raw ) ];			\
-	unsigned int size =						\
+	unsigned int invertend_size =					\
 		bigint_required_size ( sizeof ( invertend_raw ) );	\
-	bigint_t ( size ) invertend_temp;				\
-	bigint_t ( size ) inverse_temp;					\
+	unsigned int inverse_size =					\
+		bigint_required_size ( sizeof ( inverse_raw ) );	\
+	bigint_t ( invertend_size ) invertend_temp;			\
+	bigint_t ( inverse_size ) inverse_temp;				\
 	{} /* Fix emacs alignment */					\
 									\
-	assert ( bigint_size ( &invertend_temp ) ==			\
-		 bigint_size ( &inverse_temp ) );			\
 	bigint_init ( &invertend_temp, invertend_raw,			\
 		      sizeof ( invertend_raw ) );			\
 	DBG ( "Modular invert:\n" );					\
@@ -1853,6 +1853,11 @@ static void bigint_test_exec ( void ) {
 					0xb3, 0xe1, 0x3e, 0xc6, 0x5a, 0x03,
 					0x51, 0x6f, 0xb7, 0xe3, 0xa5, 0xd6,
 					0xa1, 0xb9 ) );
+	bigint_mod_invert_ok ( BIGINT ( 0xfe, 0x43, 0xf6, 0xa0, 0x32, 0x02,
+					0x47, 0xaa, 0xaa, 0x0e, 0x33, 0x19,
+					0x2e, 0xe6, 0x22, 0x07 ),
+			       BIGINT ( 0x7b, 0xd1, 0x0f, 0x78, 0x0c, 0x65,
+					0xab, 0xb7 ) );
 	bigint_mod_multiply_ok ( BIGINT ( 0x37 ),
 				 BIGINT ( 0x67 ),
 				 BIGINT ( 0x3f ),

From 4f7dd7fbba205d413cf9b989f7cdc928fa02caf2 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Wed, 27 Nov 2024 13:25:18 +0000
Subject: [PATCH 235/237] [crypto] Add bigint_montgomery() to perform
 Montgomery reduction

Montgomery reduction is substantially faster than direct reduction,
and is better suited for modular exponentiation operations.

Add bigint_montgomery() to perform the Montgomery reduction operation
(often referred to as "REDC"), along with some test vectors.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       | 77 +++++++++++++++++++++++++++++++++++++++
 src/include/ipxe/bigint.h | 21 +++++++++++
 src/tests/bigint_test.c   | 76 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 174 insertions(+)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index 735fcdf61..6d75fbe9b 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -354,6 +354,83 @@ void bigint_mod_invert_raw ( const bigint_element_t *invertend0,
 	}
 }
 
+/**
+ * Perform Montgomery reduction (REDC) of a big integer product
+ *
+ * @v modulus0		Element 0 of big integer modulus
+ * @v modinv0		Element 0 of the inverse of the modulus modulo 2^k
+ * @v mont0		Element 0 of big integer Montgomery product
+ * @v result0		Element 0 of big integer to hold result
+ * @v size		Number of elements in modulus and result
+ *
+ * Note that only the least significant element of the inverse modulo
+ * 2^k is required, and that the Montgomery product will be
+ * overwritten.
+ */
+void bigint_montgomery_raw ( const bigint_element_t *modulus0,
+			     const bigint_element_t *modinv0,
+			     bigint_element_t *mont0,
+			     bigint_element_t *result0, unsigned int size ) {
+	const bigint_t ( size ) __attribute__ (( may_alias ))
+		*modulus = ( ( const void * ) modulus0 );
+	const bigint_t ( 1 ) __attribute__ (( may_alias ))
+		*modinv = ( ( const void * ) modinv0 );
+	union {
+		bigint_t ( size * 2 ) full;
+		struct {
+			bigint_t ( size ) low;
+			bigint_t ( size ) high;
+		} __attribute__ (( packed ));
+	} __attribute__ (( may_alias )) *mont = ( ( void * ) mont0 );
+	bigint_t ( size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
+	bigint_element_t negmodinv = -modinv->element[0];
+	bigint_element_t multiple;
+	bigint_element_t carry;
+	unsigned int i;
+	unsigned int j;
+	int overflow;
+	int underflow;
+
+	/* Sanity checks */
+	assert ( bigint_bit_is_set ( modulus, 0 ) );
+
+	/* Perform multiprecision Montgomery reduction */
+	for ( i = 0 ; i < size ; i++ ) {
+
+		/* Determine scalar multiple for this round */
+		multiple = ( mont->low.element[i] * negmodinv );
+
+		/* Multiply value to make it divisible by 2^(width*i) */
+		carry = 0;
+		for ( j = 0 ; j < size ; j++ ) {
+			bigint_multiply_one ( multiple, modulus->element[j],
+					      &mont->full.element[ i + j ],
+					      &carry );
+		}
+
+		/* Since value is now divisible by 2^(width*i), we
+		 * know that the current low element must have been
+		 * zeroed.  We can store the multiplication carry out
+		 * in this element, avoiding the need to immediately
+		 * propagate the carry through the remaining elements.
+		 */
+		assert ( mont->low.element[i] == 0 );
+		mont->low.element[i] = carry;
+	}
+
+	/* Add the accumulated carries */
+	overflow = bigint_add ( &mont->low, &mont->high );
+
+	/* Conditionally subtract the modulus once */
+	memcpy ( result, &mont->high, sizeof ( *result ) );
+	underflow = bigint_subtract ( modulus, result );
+	bigint_swap ( result, &mont->high, ( underflow & ~overflow ) );
+
+	/* Sanity check */
+	assert ( ! bigint_is_geq ( result, modulus ) );
+}
+
 /**
  * Perform modular multiplication of big integers
  *
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 14f3c5f28..6c9730252 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -253,6 +253,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 				(inverse)->element, size );		\
 	} while ( 0 )
 
+/**
+ * Perform Montgomery reduction (REDC) of a big integer product
+ *
+ * @v modulus		Big integer modulus
+ * @v modinv		Big integer inverse of the modulus modulo 2^k
+ * @v mont		Big integer Montgomery product
+ * @v result		Big integer to hold result
+ *
+ * Note that the Montgomery product will be overwritten.
+ */
+#define bigint_montgomery( modulus, modinv, mont, result ) do {		\
+	unsigned int size = bigint_size (modulus);			\
+	bigint_montgomery_raw ( (modulus)->element, (modinv)->element,	\
+				(mont)->element, (result)->element,	\
+				size );					\
+	} while ( 0 )
+
 /**
  * Perform modular multiplication of big integers
  *
@@ -396,6 +413,10 @@ void bigint_reduce_raw ( bigint_element_t *modulus0, bigint_element_t *value0,
 			 unsigned int size );
 void bigint_mod_invert_raw ( const bigint_element_t *invertend0,
 			     bigint_element_t *inverse0, unsigned int size );
+void bigint_montgomery_raw ( const bigint_element_t *modulus0,
+			     const bigint_element_t *modinv0,
+			     bigint_element_t *mont0,
+			     bigint_element_t *result0, unsigned int size );
 void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 			       const bigint_element_t *multiplier0,
 			       const bigint_element_t *modulus0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index bee36de25..1f2f5f244 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -206,6 +206,23 @@ void bigint_mod_invert_sample ( const bigint_element_t *invertend0,
 	bigint_mod_invert ( invertend, inverse );
 }
 
+void bigint_montgomery_sample ( const bigint_element_t *modulus0,
+				const bigint_element_t *modinv0,
+				bigint_element_t *mont0,
+				bigint_element_t *result0,
+				unsigned int size ) {
+	const bigint_t ( size ) __attribute__ (( may_alias ))
+		*modulus = ( ( const void * ) modulus0 );
+	const bigint_t ( 1 ) __attribute__ (( may_alias ))
+		*modinv = ( ( const void * ) modinv0 );
+	bigint_t ( 2 * size ) __attribute__ (( may_alias ))
+		*mont = ( ( void * ) mont0 );
+	bigint_t ( size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
+
+	bigint_montgomery ( modulus, modinv, mont, result );
+}
+
 void bigint_mod_multiply_sample ( const bigint_element_t *multiplicand0,
 				  const bigint_element_t *multiplier0,
 				  const bigint_element_t *modulus0,
@@ -617,6 +634,46 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 		      sizeof ( inverse_raw ) ) == 0 );			\
 	} while ( 0 )
 
+/**
+ * Report result of Montgomery reduction (REDC) test
+ *
+ * @v modulus		Big integer modulus
+ * @v mont		Big integer Montgomery product
+ * @v expected		Big integer expected result
+ */
+#define bigint_montgomery_ok( modulus, mont, expected ) do {		\
+	static const uint8_t modulus_raw[] = modulus;			\
+	static const uint8_t mont_raw[] = mont;				\
+	static const uint8_t expected_raw[] = expected;			\
+	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
+	unsigned int size =						\
+		bigint_required_size ( sizeof ( modulus_raw ) );	\
+	bigint_t ( size ) modulus_temp;					\
+	bigint_t ( 1 ) modinv_temp;					\
+	bigint_t ( 2 * size ) mont_temp;				\
+	bigint_t ( size ) result_temp;					\
+	{} /* Fix emacs alignment */					\
+									\
+	assert ( ( sizeof ( modulus_raw ) %				\
+		   sizeof ( bigint_element_t ) ) == 0 );		\
+	bigint_init ( &modulus_temp, modulus_raw,			\
+		      sizeof ( modulus_raw ) );				\
+	bigint_init ( &mont_temp, mont_raw, sizeof ( mont_raw ) );	\
+	bigint_mod_invert ( &modulus_temp, &modinv_temp );		\
+	DBG ( "Montgomery:\n" );					\
+	DBG_HDA ( 0, &modulus_temp, sizeof ( modulus_temp ) );		\
+	DBG_HDA ( 0, &modinv_temp, sizeof ( modinv_temp ) );		\
+	DBG_HDA ( 0, &mont_temp, sizeof ( mont_temp ) );		\
+	bigint_montgomery ( &modulus_temp, &modinv_temp, &mont_temp,	\
+			    &result_temp );				\
+	DBG_HDA ( 0, &result_temp, sizeof ( result_temp ) );		\
+	bigint_done ( &result_temp, result_raw,				\
+		      sizeof ( result_raw ) );				\
+									\
+	ok ( memcmp ( result_raw, expected_raw,				\
+		      sizeof ( result_raw ) ) == 0 );			\
+	} while ( 0 )
+
 /**
  * Report result of big integer modular multiplication test
  *
@@ -1858,6 +1915,25 @@ static void bigint_test_exec ( void ) {
 					0x2e, 0xe6, 0x22, 0x07 ),
 			       BIGINT ( 0x7b, 0xd1, 0x0f, 0x78, 0x0c, 0x65,
 					0xab, 0xb7 ) );
+	bigint_montgomery_ok ( BIGINT ( 0x74, 0xdf, 0xd1, 0xb8, 0x14, 0xf7,
+					0x05, 0x83 ),
+			       BIGINT ( 0x50, 0x6a, 0x38, 0x55, 0x9f, 0xb9,
+					0x9d, 0xba, 0xff, 0x23, 0x86, 0x65,
+					0xe3, 0x2c, 0x3f, 0x17 ),
+			       BIGINT ( 0x45, 0x1f, 0x51, 0x44, 0x6f, 0x3c,
+					0x09, 0x6b ) );
+	bigint_montgomery_ok ( BIGINT ( 0x2e, 0x32, 0x90, 0x69, 0x6e, 0xa8,
+					0x47, 0x4c, 0xad, 0xe4, 0xe7, 0x4c,
+					0x03, 0xcb, 0xe6, 0x55 ),
+			       BIGINT ( 0x1e, 0x43, 0xf9, 0xc2, 0x61, 0xdd,
+					0xe8, 0xbf, 0xb8, 0xea, 0xe0, 0xdb,
+					0xed, 0x66, 0x80, 0x1e, 0xe8, 0xf8,
+					0xd1, 0x1d, 0xca, 0x8d, 0x45, 0xe9,
+					0xc5, 0xeb, 0x77, 0x21, 0x34, 0xe0,
+					0xf5, 0x5a ),
+			       BIGINT ( 0x03, 0x38, 0xfb, 0xb6, 0xf3, 0x80,
+					0x91, 0xb2, 0x68, 0x2f, 0x81, 0x44,
+					0xbf, 0x43, 0x0a, 0x4e ) );
 	bigint_mod_multiply_ok ( BIGINT ( 0x37 ),
 				 BIGINT ( 0x67 ),
 				 BIGINT ( 0x3f ),

From 83ac98ce22b5b735cba4d1a21db8cc8e8648dfa4 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Mon, 25 Nov 2024 15:59:22 +0000
Subject: [PATCH 236/237] [crypto] Use Montgomery reduction for modular
 exponentiation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Speed up modular exponentiation by using Montgomery reduction rather
than direct modular reduction.

Montgomery reduction in base 2^n requires the modulus to be coprime to
2^n, which would limit us to requiring that the modulus is an odd
number.  Extend the implementation to include support for
exponentiation with even moduli via Garner's algorithm as described in
"Montgomery reduction with even modulus" (Koç, 1994).

Since almost all use cases for modular exponentation require a large
prime (and hence odd) modulus, the support for even moduli could
potentially be removed in future.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       | 147 ++++++++++++++++++++++++++++++++++----
 src/crypto/dhe.c          |   3 +-
 src/crypto/rsa.c          |   3 +-
 src/include/ipxe/bigint.h |  10 +--
 src/tests/bigint_test.c   |  30 +++++++-
 5 files changed, 164 insertions(+), 29 deletions(-)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index 6d75fbe9b..39e1a25cd 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -505,25 +505,142 @@ void bigint_mod_exp_raw ( const bigint_element_t *base0,
 		*exponent = ( ( const void * ) exponent0 );
 	bigint_t ( size ) __attribute__ (( may_alias )) *result =
 		( ( void * ) result0 );
-	size_t mod_multiply_len = bigint_mod_multiply_tmp_len ( modulus );
+	const unsigned int width = ( 8 * sizeof ( bigint_element_t ) );
 	struct {
-		bigint_t ( size ) base;
-		bigint_t ( exponent_size ) exponent;
-		uint8_t mod_multiply[mod_multiply_len];
+		union {
+			bigint_t ( 2 * size ) padded_modulus;
+			struct {
+				bigint_t ( size ) modulus;
+				bigint_t ( size ) stash;
+			};
+		};
+		union {
+			bigint_t ( 2 * size ) full;
+			bigint_t ( size ) low;
+		} product;
 	} *temp = tmp;
-	static const uint8_t start[1] = { 0x01 };
+	const uint8_t one[1] = { 1 };
+	bigint_t ( 1 ) modinv;
+	bigint_element_t submask;
+	unsigned int subsize;
+	unsigned int scale;
+	unsigned int max;
+	unsigned int bit;
 
-	memcpy ( &temp->base, base, sizeof ( temp->base ) );
-	memcpy ( &temp->exponent, exponent, sizeof ( temp->exponent ) );
-	bigint_init ( result, start, sizeof ( start ) );
+	/* Sanity check */
+	assert ( sizeof ( *temp ) == bigint_mod_exp_tmp_len ( modulus ) );
 
-	while ( ! bigint_is_zero ( &temp->exponent ) ) {
-		if ( bigint_bit_is_set ( &temp->exponent, 0 ) ) {
-			bigint_mod_multiply ( result, &temp->base, modulus,
-					      result, temp->mod_multiply );
+	/* Handle degenerate case of zero modulus */
+	if ( ! bigint_max_set_bit ( modulus ) ) {
+		memset ( result, 0, sizeof ( *result ) );
+		return;
+	}
+
+	/* Factor modulus as (N * 2^scale) where N is odd */
+	bigint_grow ( modulus, &temp->padded_modulus );
+	for ( scale = 0 ; ( ! bigint_bit_is_set ( &temp->modulus, 0 ) ) ;
+	      scale++ ) {
+		bigint_shr ( &temp->modulus );
+	}
+	subsize = ( ( scale + width - 1 ) / width );
+	submask = ( ( 1UL << ( scale % width ) ) - 1 );
+	if ( ! submask )
+		submask = ~submask;
+
+	/* Calculate inverse of (scaled) modulus N modulo element size */
+	bigint_mod_invert ( &temp->modulus, &modinv );
+
+	/* Calculate (R^2 mod N) via direct reduction of (R^2 - N) */
+	memset ( &temp->product.full, 0, sizeof ( temp->product.full ) );
+	bigint_subtract ( &temp->padded_modulus, &temp->product.full );
+	bigint_reduce ( &temp->padded_modulus, &temp->product.full );
+	bigint_copy ( &temp->product.low, &temp->stash );
+
+	/* Initialise result = Montgomery(1, R^2 mod N) */
+	bigint_montgomery ( &temp->modulus, &modinv,
+			    &temp->product.full, result );
+
+	/* Convert base into Montgomery form */
+	bigint_multiply ( base, &temp->stash, &temp->product.full );
+	bigint_montgomery ( &temp->modulus, &modinv, &temp->product.full,
+			    &temp->stash );
+
+	/* Calculate x1 = base^exponent modulo N */
+	max = bigint_max_set_bit ( exponent );
+	for ( bit = 1 ; bit <= max ; bit++ ) {
+
+		/* Square (and reduce) */
+		bigint_multiply ( result, result, &temp->product.full );
+		bigint_montgomery ( &temp->modulus, &modinv,
+				    &temp->product.full, result );
+
+		/* Multiply (and reduce) */
+		bigint_multiply ( &temp->stash, result, &temp->product.full );
+		bigint_montgomery ( &temp->modulus, &modinv,
+				    &temp->product.full, &temp->product.low );
+
+		/* Conditionally swap the multiplied result */
+		bigint_swap ( result, &temp->product.low,
+			      bigint_bit_is_set ( exponent, ( max - bit ) ) );
+	}
+
+	/* Convert back out of Montgomery form */
+	bigint_grow ( result, &temp->product.full );
+	bigint_montgomery ( &temp->modulus, &modinv, &temp->product.full,
+			    result );
+
+	/* Handle even moduli via Garner's algorithm */
+	if ( subsize ) {
+		const bigint_t ( subsize ) __attribute__ (( may_alias ))
+			*subbase = ( ( const void * ) base );
+		bigint_t ( subsize ) __attribute__ (( may_alias ))
+			*submodulus = ( ( void * ) &temp->modulus );
+		bigint_t ( subsize ) __attribute__ (( may_alias ))
+			*substash = ( ( void * ) &temp->stash );
+		bigint_t ( subsize ) __attribute__ (( may_alias ))
+			*subresult = ( ( void * ) result );
+		union {
+			bigint_t ( 2 * subsize ) full;
+			bigint_t ( subsize ) low;
+		} __attribute__ (( may_alias ))
+			*subproduct = ( ( void * ) &temp->product.full );
+
+		/* Calculate x2 = base^exponent modulo 2^k */
+		bigint_init ( substash, one, sizeof ( one ) );
+		for ( bit = 1 ; bit <= max ; bit++ ) {
+
+			/* Square (and reduce) */
+			bigint_multiply ( substash, substash,
+					  &subproduct->full );
+			bigint_copy ( &subproduct->low, substash );
+
+			/* Multiply (and reduce) */
+			bigint_multiply ( subbase, substash,
+					  &subproduct->full );
+
+			/* Conditionally swap the multiplied result */
+			bigint_swap ( substash, &subproduct->low,
+				      bigint_bit_is_set ( exponent,
+							  ( max - bit ) ) );
 		}
-		bigint_shr ( &temp->exponent );
-		bigint_mod_multiply ( &temp->base, &temp->base, modulus,
-				      &temp->base, temp->mod_multiply );
+
+		/* Calculate N^-1 modulo 2^k */
+		bigint_mod_invert ( submodulus, &subproduct->low );
+		bigint_copy ( &subproduct->low, submodulus );
+
+		/* Calculate y = (x2 - x1) * N^-1 modulo 2^k */
+		bigint_subtract ( subresult, substash );
+		bigint_multiply ( substash, submodulus, &subproduct->full );
+		subproduct->low.element[ subsize - 1 ] &= submask;
+		bigint_grow ( &subproduct->low, &temp->stash );
+
+		/* Reconstruct N */
+		bigint_mod_invert ( submodulus, &subproduct->low );
+		bigint_copy ( &subproduct->low, submodulus );
+
+		/* Calculate x = x1 + N * y */
+		bigint_multiply ( &temp->modulus, &temp->stash,
+				  &temp->product.full );
+		bigint_add ( &temp->product.low, result );
 	}
 }
diff --git a/src/crypto/dhe.c b/src/crypto/dhe.c
index 2da107d24..a249f9b40 100644
--- a/src/crypto/dhe.c
+++ b/src/crypto/dhe.c
@@ -57,8 +57,7 @@ int dhe_key ( const void *modulus, size_t len, const void *generator,
 	unsigned int size = bigint_required_size ( len );
 	unsigned int private_size = bigint_required_size ( private_len );
 	bigint_t ( size ) *mod;
-	bigint_t ( private_size ) *exp;
-	size_t tmp_len = bigint_mod_exp_tmp_len ( mod, exp );
+	size_t tmp_len = bigint_mod_exp_tmp_len ( mod );
 	struct {
 		bigint_t ( size ) modulus;
 		bigint_t ( size ) generator;
diff --git a/src/crypto/rsa.c b/src/crypto/rsa.c
index 19472c121..44041da3e 100644
--- a/src/crypto/rsa.c
+++ b/src/crypto/rsa.c
@@ -109,8 +109,7 @@ static int rsa_alloc ( struct rsa_context *context, size_t modulus_len,
 	unsigned int size = bigint_required_size ( modulus_len );
 	unsigned int exponent_size = bigint_required_size ( exponent_len );
 	bigint_t ( size ) *modulus;
-	bigint_t ( exponent_size ) *exponent;
-	size_t tmp_len = bigint_mod_exp_tmp_len ( modulus, exponent );
+	size_t tmp_len = bigint_mod_exp_tmp_len ( modulus );
 	struct {
 		bigint_t ( size ) modulus;
 		bigint_t ( exponent_size ) exponent;
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 6c9730252..3ca871962 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -322,18 +322,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Calculate temporary working space required for moduluar exponentiation
  *
  * @v modulus		Big integer modulus
- * @v exponent		Big integer exponent
  * @ret len		Length of temporary working space
  */
-#define bigint_mod_exp_tmp_len( modulus, exponent ) ( {			\
+#define bigint_mod_exp_tmp_len( modulus ) ( {				\
 	unsigned int size = bigint_size (modulus);			\
-	unsigned int exponent_size = bigint_size (exponent);		\
-	size_t mod_multiply_len =					\
-		bigint_mod_multiply_tmp_len (modulus);			\
 	sizeof ( struct {						\
-		bigint_t ( size ) temp_base;				\
-		bigint_t ( exponent_size ) temp_exponent;		\
-		uint8_t mod_multiply[mod_multiply_len];			\
+		bigint_t ( size ) temp[4];				\
 	} ); } )
 
 #include <bits/bigint.h>
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 1f2f5f244..f3291f6a6 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -746,8 +746,7 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	bigint_t ( size ) modulus_temp;					\
 	bigint_t ( exponent_size ) exponent_temp;			\
 	bigint_t ( size ) result_temp;					\
-	size_t tmp_len = bigint_mod_exp_tmp_len ( &modulus_temp,	\
-						  &exponent_temp );	\
+	size_t tmp_len = bigint_mod_exp_tmp_len ( &modulus_temp );	\
 	uint8_t tmp[tmp_len];						\
 	{} /* Fix emacs alignment */					\
 									\
@@ -2070,6 +2069,14 @@ static void bigint_test_exec ( void ) {
 			    BIGINT ( 0xb9 ),
 			    BIGINT ( 0x39, 0x68, 0xba, 0x7d ),
 			    BIGINT ( 0x17 ) );
+	bigint_mod_exp_ok ( BIGINT ( 0x71, 0x4d, 0x02, 0xe9 ),
+			    BIGINT ( 0x00, 0x00, 0x00, 0x00 ),
+			    BIGINT ( 0x91, 0x7f, 0x4e, 0x3a, 0x5d, 0x5c ),
+			    BIGINT ( 0x00, 0x00, 0x00, 0x00 ) );
+	bigint_mod_exp_ok ( BIGINT ( 0x2b, 0xf5, 0x07, 0xaf ),
+			    BIGINT ( 0x6e, 0xb5, 0xda, 0x5a ),
+			    BIGINT ( 0x00, 0x00, 0x00, 0x00, 0x00 ),
+			    BIGINT ( 0x00, 0x00, 0x00, 0x01 ) );
 	bigint_mod_exp_ok ( BIGINT ( 0x2e ),
 			    BIGINT ( 0xb7 ),
 			    BIGINT ( 0x39, 0x07, 0x1b, 0x49, 0x5b, 0xea,
@@ -2774,6 +2781,25 @@ static void bigint_test_exec ( void ) {
 				     0xfa, 0x83, 0xd4, 0x7c, 0xe9, 0x77,
 				     0x46, 0x91, 0x3a, 0x50, 0x0d, 0x6a,
 				     0x25, 0xd0 ) );
+	bigint_mod_exp_ok ( BIGINT ( 0x5b, 0x80, 0xc5, 0x03, 0xb3, 0x1e,
+				     0x46, 0x9b, 0xa3, 0x0a, 0x70, 0x43,
+				     0x51, 0x2a, 0x4a, 0x44, 0xcb, 0x87,
+				     0x3e, 0x00, 0x2a, 0x48, 0x46, 0xf5,
+				     0xb3, 0xb9, 0x73, 0xa7, 0x77, 0xfc,
+				     0x2a, 0x1d ),
+			    BIGINT ( 0x5e, 0x8c, 0x80, 0x03, 0xe7, 0xb0,
+				     0x45, 0x23, 0x8f, 0xe0, 0x77, 0x02,
+				     0xc0, 0x7e, 0xfb, 0xc4, 0xbe, 0x7b,
+				     0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				     0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				     0x00, 0x00 ),
+			    BIGINT ( 0x71, 0xd9, 0x38, 0xb6 ),
+			    BIGINT ( 0x52, 0xfc, 0x73, 0x55, 0x2f, 0x86,
+				     0x0f, 0xde, 0x04, 0xbc, 0x6d, 0xb8,
+				     0xfd, 0x48, 0xf8, 0x8c, 0x91, 0x1c,
+				     0xa0, 0x8a, 0x70, 0xa8, 0xc6, 0x20,
+				     0x0a, 0x0d, 0x3b, 0x2a, 0x92, 0x65,
+				     0x9c, 0x59 ) );
 }
 
 /** Big integer self-test */

From 5202f83345e17c56da4935205f0a97bd4aa20714 Mon Sep 17 00:00:00 2001
From: Michael Brown <mcb30@ipxe.org>
Date: Thu, 28 Nov 2024 15:05:06 +0000
Subject: [PATCH 237/237] [crypto] Remove obsolete bigint_mod_multiply()

There is no further need for a standalone modular multiplication
primitive, since the only consumer is modular exponentiation (which
now uses Montgomery multiplication instead).

Remove the now obsolete bigint_mod_multiply().

Signed-off-by: Michael Brown <mcb30@ipxe.org>
---
 src/crypto/bigint.c       |  53 -----------
 src/include/ipxe/bigint.h |  36 --------
 src/tests/bigint_test.c   | 188 --------------------------------------
 3 files changed, 277 deletions(-)

diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index 39e1a25cd..b357ea29f 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -38,10 +38,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 static struct profiler bigint_mod_profiler __profiler =
 	{ .name = "bigint_mod" };
 
-/** Modular multiplication overall profiler */
-static struct profiler bigint_mod_multiply_profiler __profiler =
-	{ .name = "bigint_mod_multiply" };
-
 /**
  * Conditionally swap big integers (in constant time)
  *
@@ -431,55 +427,6 @@ void bigint_montgomery_raw ( const bigint_element_t *modulus0,
 	assert ( ! bigint_is_geq ( result, modulus ) );
 }
 
-/**
- * Perform modular multiplication of big integers
- *
- * @v multiplicand0	Element 0 of big integer to be multiplied
- * @v multiplier0	Element 0 of big integer to be multiplied
- * @v modulus0		Element 0 of big integer modulus
- * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements in base, modulus, and result
- * @v tmp		Temporary working space
- */
-void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
-			       const bigint_element_t *multiplier0,
-			       const bigint_element_t *modulus0,
-			       bigint_element_t *result0,
-			       unsigned int size, void *tmp ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *modulus =
-		( ( const void * ) modulus0 );
-	bigint_t ( size ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
-	struct {
-		bigint_t ( size * 2 ) result;
-		bigint_t ( size * 2 ) modulus;
-	} *temp = tmp;
-
-	/* Start profiling */
-	profile_start ( &bigint_mod_multiply_profiler );
-
-	/* Sanity check */
-	assert ( sizeof ( *temp ) == bigint_mod_multiply_tmp_len ( modulus ) );
-
-	/* Perform multiplication */
-	bigint_multiply ( multiplicand, multiplier, &temp->result );
-
-	/* Reduce result */
-	bigint_grow ( modulus, &temp->modulus );
-	bigint_reduce ( &temp->modulus, &temp->result );
-	bigint_shrink ( &temp->result, result );
-
-	/* Sanity check */
-	assert ( ! bigint_is_geq ( result, modulus ) );
-
-	/* Stop profiling */
-	profile_stop ( &bigint_mod_multiply_profiler );
-}
-
 /**
  * Perform modular exponentiation of big integers
  *
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 3ca871962..3058547a6 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -270,37 +270,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 				size );					\
 	} while ( 0 )
 
-/**
- * Perform modular multiplication of big integers
- *
- * @v multiplicand	Big integer to be multiplied
- * @v multiplier	Big integer to be multiplied
- * @v modulus		Big integer modulus
- * @v result		Big integer to hold result
- * @v tmp		Temporary working space
- */
-#define bigint_mod_multiply( multiplicand, multiplier, modulus,		\
-			     result, tmp ) do {				\
-	unsigned int size = bigint_size (multiplicand);			\
-	bigint_mod_multiply_raw ( (multiplicand)->element,		\
-				  (multiplier)->element,		\
-				  (modulus)->element,			\
-				  (result)->element, size, tmp );	\
-	} while ( 0 )
-
-/**
- * Calculate temporary working space required for moduluar multiplication
- *
- * @v modulus		Big integer modulus
- * @ret len		Length of temporary working space
- */
-#define bigint_mod_multiply_tmp_len( modulus ) ( {			\
-	unsigned int size = bigint_size (modulus);			\
-	sizeof ( struct {						\
-		bigint_t ( size * 2 ) temp_result;			\
-		bigint_t ( size * 2 ) temp_modulus;			\
-	} ); } )
-
 /**
  * Perform modular exponentiation of big integers
  *
@@ -411,11 +380,6 @@ void bigint_montgomery_raw ( const bigint_element_t *modulus0,
 			     const bigint_element_t *modinv0,
 			     bigint_element_t *mont0,
 			     bigint_element_t *result0, unsigned int size );
-void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
-			       const bigint_element_t *multiplier0,
-			       const bigint_element_t *modulus0,
-			       bigint_element_t *result0,
-			       unsigned int size, void *tmp );
 void bigint_mod_exp_raw ( const bigint_element_t *base0,
 			  const bigint_element_t *modulus0,
 			  const bigint_element_t *exponent0,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index f3291f6a6..dc74740e6 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -223,24 +223,6 @@ void bigint_montgomery_sample ( const bigint_element_t *modulus0,
 	bigint_montgomery ( modulus, modinv, mont, result );
 }
 
-void bigint_mod_multiply_sample ( const bigint_element_t *multiplicand0,
-				  const bigint_element_t *multiplier0,
-				  const bigint_element_t *modulus0,
-				  bigint_element_t *result0,
-				  unsigned int size,
-				  void *tmp ) {
-	const bigint_t ( size ) *multiplicand __attribute__ (( may_alias ))
-		= ( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) *multiplier __attribute__ (( may_alias ))
-		= ( ( const void * ) multiplier0 );
-	const bigint_t ( size ) *modulus __attribute__ (( may_alias ))
-		= ( ( const void * ) modulus0 );
-	bigint_t ( size ) *result __attribute__ (( may_alias ))
-		= ( ( void * ) result0 );
-
-	bigint_mod_multiply ( multiplicand, multiplier, modulus, result, tmp );
-}
-
 void bigint_mod_exp_sample ( const bigint_element_t *base0,
 			     const bigint_element_t *modulus0,
 			     const bigint_element_t *exponent0,
@@ -674,56 +656,6 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 		      sizeof ( result_raw ) ) == 0 );			\
 	} while ( 0 )
 
-/**
- * Report result of big integer modular multiplication test
- *
- * @v multiplicand	Big integer to be multiplied
- * @v multiplier	Big integer to be multiplied
- * @v modulus		Big integer modulus
- * @v expected		Big integer expected result
- */
-#define bigint_mod_multiply_ok( multiplicand, multiplier, modulus,	\
-				expected ) do {				\
-	static const uint8_t multiplicand_raw[] = multiplicand;		\
-	static const uint8_t multiplier_raw[] = multiplier;		\
-	static const uint8_t modulus_raw[] = modulus;			\
-	static const uint8_t expected_raw[] = expected;			\
-	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
-	unsigned int size =						\
-		bigint_required_size ( sizeof ( multiplicand_raw ) );	\
-	bigint_t ( size ) multiplicand_temp;				\
-	bigint_t ( size ) multiplier_temp;				\
-	bigint_t ( size ) modulus_temp;					\
-	bigint_t ( size ) result_temp;					\
-	size_t tmp_len = bigint_mod_multiply_tmp_len ( &modulus_temp ); \
-	uint8_t tmp[tmp_len];						\
-	{} /* Fix emacs alignment */					\
-									\
-	assert ( bigint_size ( &multiplier_temp ) ==			\
-		 bigint_size ( &multiplicand_temp ) );			\
-	assert ( bigint_size ( &multiplier_temp ) ==			\
-		 bigint_size ( &modulus_temp ) );			\
-	assert ( bigint_size ( &multiplier_temp ) ==			\
-		 bigint_size ( &result_temp ) );			\
-	bigint_init ( &multiplicand_temp, multiplicand_raw,		\
-		      sizeof ( multiplicand_raw ) );			\
-	bigint_init ( &multiplier_temp, multiplier_raw,			\
-		      sizeof ( multiplier_raw ) );			\
-	bigint_init ( &modulus_temp, modulus_raw,			\
-		      sizeof ( modulus_raw ) );				\
-	DBG ( "Modular multiply:\n" );					\
-	DBG_HDA ( 0, &multiplicand_temp, sizeof ( multiplicand_temp ) );\
-	DBG_HDA ( 0, &multiplier_temp, sizeof ( multiplier_temp ) );	\
-	DBG_HDA ( 0, &modulus_temp, sizeof ( modulus_temp ) );		\
-	bigint_mod_multiply ( &multiplicand_temp, &multiplier_temp,	\
-			      &modulus_temp, &result_temp, tmp );	\
-	DBG_HDA ( 0, &result_temp, sizeof ( result_temp ) );		\
-	bigint_done ( &result_temp, result_raw, sizeof ( result_raw ) );\
-									\
-	ok ( memcmp ( result_raw, expected_raw,				\
-		      sizeof ( result_raw ) ) == 0 );			\
-	} while ( 0 )
-
 /**
  * Report result of big integer modular exponentiation test
  *
@@ -1933,126 +1865,6 @@ static void bigint_test_exec ( void ) {
 			       BIGINT ( 0x03, 0x38, 0xfb, 0xb6, 0xf3, 0x80,
 					0x91, 0xb2, 0x68, 0x2f, 0x81, 0x44,
 					0xbf, 0x43, 0x0a, 0x4e ) );
-	bigint_mod_multiply_ok ( BIGINT ( 0x37 ),
-				 BIGINT ( 0x67 ),
-				 BIGINT ( 0x3f ),
-				 BIGINT ( 0x3a ) );
-	bigint_mod_multiply_ok ( BIGINT ( 0x45, 0x94 ),
-				 BIGINT ( 0xbd, 0xd2 ),
-				 BIGINT ( 0xca, 0xc7 ),
-				 BIGINT ( 0xac, 0xc1 ) );
-	bigint_mod_multiply_ok ( BIGINT ( 0x8e, 0xcd, 0x74 ),
-				 BIGINT ( 0xe2, 0xf1, 0xea ),
-				 BIGINT ( 0x59, 0x51, 0x53 ),
-				 BIGINT ( 0x22, 0xdd, 0x1c ) );
-	bigint_mod_multiply_ok ( BIGINT ( 0xc2, 0xa8, 0x40, 0x6f ),
-				 BIGINT ( 0x29, 0xd7, 0xf4, 0x77 ),
-				 BIGINT ( 0xbb, 0xbd, 0xdb, 0x3d ),
-				 BIGINT ( 0x8f, 0x39, 0xd0, 0x47 ) );
-	bigint_mod_multiply_ok ( BIGINT ( 0x2e, 0xcb, 0x74, 0x7c, 0x64, 0x60,
-					  0xb3, 0x44, 0xf3, 0x23, 0x49, 0x4a,
-					  0xc6, 0xb6, 0xbf, 0xea, 0x80, 0xd8,
-					  0x34, 0xc5, 0x54, 0x22, 0x09 ),
-				 BIGINT ( 0x61, 0x2c, 0x5a, 0xc5, 0xde, 0x07,
-					  0x65, 0x8e, 0xab, 0x88, 0x1a, 0x2e,
-					  0x7a, 0x95, 0x17, 0xe3, 0x3b, 0x17,
-					  0xe4, 0x21, 0xb0, 0xb4, 0x57 ),
-				 BIGINT ( 0x8e, 0x46, 0xa5, 0x87, 0x7b, 0x7f,
-					  0xc4, 0xd7, 0x31, 0xb1, 0x94, 0xe7,
-					  0xe7, 0x1c, 0x7e, 0x7a, 0xc2, 0x6c,
-					  0xce, 0xcb, 0xc8, 0x5d, 0x70 ),
-				 BIGINT ( 0x1e, 0xd1, 0x5b, 0x3d, 0x1d, 0x17,
-					  0x7c, 0x31, 0x95, 0x13, 0x1b, 0xd8,
-					  0xee, 0x0a, 0xb0, 0xe1, 0x5b, 0x13,
-					  0xad, 0x83, 0xe9, 0xf8, 0x7f ) );
-	bigint_mod_multiply_ok ( BIGINT ( 0x56, 0x37, 0xab, 0x07, 0x8b, 0x25,
-					  0xa7, 0xc2, 0x50, 0x30, 0x43, 0xfc,
-					  0x63, 0x8b, 0xdf, 0x84, 0x68, 0x85,
-					  0xca, 0xce, 0x44, 0x5c, 0xb1, 0x14,
-					  0xa4, 0xb5, 0xba, 0x43, 0xe0, 0x31,
-					  0x45, 0x6b, 0x82, 0xa9, 0x0b, 0x9e,
-					  0x3a, 0xb0, 0x39, 0x09, 0x2a, 0x68,
-					  0x2e, 0x0f, 0x09, 0x54, 0x47, 0x04,
-					  0xdb, 0xcf, 0x4a, 0x3a, 0x2d, 0x7b,
-					  0x7d, 0x50, 0xa4, 0xc5, 0xeb, 0x13,
-					  0xdd, 0x49, 0x61, 0x7d, 0x18, 0xa1,
-					  0x0d, 0x6b, 0x58, 0xba, 0x9f, 0x7c,
-					  0x81, 0x34, 0x9e, 0xf9, 0x9c, 0x9e,
-					  0x28, 0xa8, 0x1c, 0x15, 0x16, 0x20,
-					  0x3c, 0x0a, 0xb1, 0x11, 0x06, 0x93,
-					  0xbc, 0xd8, 0x4e, 0x49, 0xae, 0x7b,
-					  0xb4, 0x02, 0x8b, 0x1c, 0x5b, 0x18,
-					  0xb4, 0xac, 0x7f, 0xdd, 0x70, 0xef,
-					  0x87, 0xac, 0x1b, 0xac, 0x25, 0xa3,
-					  0xc9, 0xa7, 0x3a, 0xc5, 0x76, 0x68,
-					  0x09, 0x1f, 0xa1, 0x48, 0x53, 0xb6,
-					  0x13, 0xac ),
-				 BIGINT ( 0xef, 0x5c, 0x1f, 0x1a, 0x44, 0x64,
-					  0x66, 0xcf, 0xdd, 0x3f, 0x0b, 0x27,
-					  0x81, 0xa7, 0x91, 0x7a, 0x35, 0x7b,
-					  0x0f, 0x46, 0x5e, 0xca, 0xbf, 0xf8,
-					  0x50, 0x5e, 0x99, 0x7c, 0xc6, 0x64,
-					  0x43, 0x00, 0x9f, 0xb2, 0xda, 0xfa,
-					  0x42, 0x15, 0x9c, 0xa3, 0xd6, 0xc8,
-					  0x64, 0xa7, 0x65, 0x4a, 0x98, 0xf7,
-					  0xb3, 0x96, 0x5f, 0x42, 0xf9, 0x73,
-					  0xe1, 0x75, 0xc3, 0xc4, 0x0b, 0x5d,
-					  0x5f, 0xf3, 0x04, 0x8a, 0xee, 0x59,
-					  0xa6, 0x1b, 0x06, 0x38, 0x0b, 0xa2,
-					  0x9f, 0xb4, 0x4f, 0x6d, 0x50, 0x5e,
-					  0x37, 0x37, 0x21, 0x83, 0x9d, 0xa3,
-					  0x12, 0x16, 0x4d, 0xab, 0x36, 0x51,
-					  0x21, 0xb1, 0x74, 0x66, 0x40, 0x9a,
-					  0xd3, 0x72, 0xcc, 0x18, 0x40, 0x53,
-					  0x89, 0xff, 0xd7, 0x00, 0x8d, 0x7e,
-					  0x93, 0x81, 0xdb, 0x29, 0xb6, 0xd7,
-					  0x23, 0x2b, 0x67, 0x2f, 0x11, 0x98,
-					  0x49, 0x87, 0x2f, 0x46, 0xb7, 0x33,
-					  0x6d, 0x12 ),
-				 BIGINT ( 0x67, 0x7a, 0x17, 0x6a, 0xd2, 0xf8,
-					  0x49, 0xfb, 0x7c, 0x95, 0x25, 0x54,
-					  0xf0, 0xab, 0x5b, 0xb3, 0x0e, 0x01,
-					  0xab, 0xd3, 0x65, 0x6f, 0x7e, 0x18,
-					  0x05, 0xed, 0x9b, 0xc4, 0x90, 0x6c,
-					  0xd0, 0x6d, 0x94, 0x79, 0x28, 0xd6,
-					  0x24, 0x77, 0x9a, 0x08, 0xd2, 0x2f,
-					  0x7c, 0x2d, 0xa0, 0x0c, 0x14, 0xbe,
-					  0x7b, 0xee, 0x9e, 0x48, 0x88, 0x3c,
-					  0x8f, 0x9f, 0xb9, 0x7a, 0xcb, 0x98,
-					  0x76, 0x61, 0x0d, 0xee, 0xa2, 0x42,
-					  0x67, 0x1b, 0x2c, 0x42, 0x8f, 0x41,
-					  0xcc, 0x78, 0xba, 0xba, 0xaa, 0xa2,
-					  0x92, 0xb0, 0x6e, 0x0c, 0x4e, 0xe1,
-					  0xa5, 0x13, 0x7d, 0x8a, 0x8f, 0x81,
-					  0x95, 0x8a, 0xdf, 0x57, 0x93, 0x88,
-					  0x27, 0x4f, 0x1a, 0x59, 0xa4, 0x74,
-					  0x22, 0xa9, 0x78, 0xe5, 0xed, 0xb1,
-					  0x09, 0x26, 0x59, 0xde, 0x88, 0x21,
-					  0x8d, 0xa2, 0xa8, 0x58, 0x10, 0x7b,
-					  0x65, 0x96, 0xbf, 0x69, 0x3b, 0xc5,
-					  0x55, 0xf8 ),
-				 BIGINT ( 0x15, 0xf7, 0x00, 0xeb, 0xc7, 0x5a,
-					  0x6f, 0xf0, 0x50, 0xf3, 0x21, 0x8a,
-					  0x8e, 0xa6, 0xf6, 0x67, 0x56, 0x7d,
-					  0x07, 0x45, 0x89, 0xdb, 0xd7, 0x7e,
-					  0x9e, 0x28, 0x7f, 0xfb, 0xed, 0xca,
-					  0x2c, 0xbf, 0x47, 0x77, 0x99, 0x95,
-					  0xf3, 0xd6, 0x9d, 0xc5, 0x57, 0x81,
-					  0x7f, 0x97, 0xf2, 0x6b, 0x24, 0xee,
-					  0xce, 0xc5, 0x9b, 0xe6, 0x42, 0x2d,
-					  0x37, 0xb7, 0xeb, 0x3d, 0xb5, 0xf7,
-					  0x1e, 0x86, 0xc2, 0x40, 0x44, 0xc9,
-					  0x85, 0x5a, 0x1a, 0xc0, 0xac, 0x9e,
-					  0x78, 0x69, 0x00, 0x7b, 0x93, 0x65,
-					  0xd7, 0x7f, 0x0c, 0xd6, 0xba, 0x4f,
-					  0x06, 0x00, 0x61, 0x05, 0xb2, 0x44,
-					  0xb4, 0xe7, 0xbb, 0x3b, 0x96, 0xb0,
-					  0x6d, 0xe8, 0x43, 0xd2, 0x03, 0xb7,
-					  0x0a, 0xc4, 0x6d, 0x30, 0xd8, 0xd5,
-					  0xe6, 0x54, 0x65, 0xdd, 0xa9, 0x1b,
-					  0x50, 0xc0, 0xb9, 0x95, 0xb0, 0x7d,
-					  0x7c, 0xca, 0x63, 0xf8, 0x72, 0xbe,
-					  0x3b, 0x00 ) );
 	bigint_mod_exp_ok ( BIGINT ( 0xcd ),
 			    BIGINT ( 0xbb ),
 			    BIGINT ( 0x25 ),