Add a route when a gateway is outside a subnet (#488)

When we set a gateway outside a subnet in Linux, this fails with:
```
ip-route-add-default: netlink response indicates error, rc = -101 
```

To resolve the error, we should first add a route to the gateway outside
the subnet, which this PR does.

Sources/Containerization/LinuxContainer.swift

@@ -457,9 +457,14 @@ extension LinuxContainer {
                     // 3. If a gateway IP address is present, add the default route.
                     for (index, i) in self.interfaces.enumerated() {
                         let name = "eth\(index)"
+                        self.logger?.debug("setting up interface \(name) with address \(i.ipv4Address)")
                         try await agent.addressAdd(name: name, ipv4Address: i.ipv4Address)
                         try await agent.up(name: name, mtu: i.mtu)
                         if let ipv4Gateway = i.ipv4Gateway {
+                            if !i.ipv4Address.contains(ipv4Gateway) {
+                                self.logger?.debug("gateway \(ipv4Gateway) is outside subnet \(i.ipv4Address), adding a route first")
+                                try await agent.routeAddLink(name: name, dstIPv4Addr: ipv4Gateway, srcIPv4Addr: i.ipv4Address.address)
+                            }
                             try await agent.routeAddDefault(name: name, ipv4Gateway: ipv4Gateway)
                         }
                     }

Sources/Containerization/LinuxPod.swift

@@ -401,9 +401,14 @@ extension LinuxPod {
                     // 3. If a gateway IP address is present, add the default route.
                     for (index, i) in self.interfaces.enumerated() {
                         let name = "eth\(index)"
+                        self.logger?.debug("setting up interface \(name) with address \(i.ipv4Address)")
                         try await agent.addressAdd(name: name, ipv4Address: i.ipv4Address)
                         try await agent.up(name: name, mtu: i.mtu)
                         if let ipv4Gateway = i.ipv4Gateway {
+                            if !i.ipv4Address.contains(ipv4Gateway) {
+                                self.logger?.debug("gateway \(ipv4Gateway) is outside subnet \(i.ipv4Address), adding a route first")
+                                try await agent.routeAddLink(name: name, dstIPv4Addr: ipv4Gateway, srcIPv4Addr: nil)
+                            }
                             try await agent.routeAddDefault(name: name, ipv4Gateway: ipv4Gateway)
                         }
                     }

Sources/Containerization/VirtualMachineAgent.swift

@@ -89,6 +89,7 @@ public protocol VirtualMachineAgent: Sendable {
     func up(name: String, mtu: UInt32?) async throws
     func down(name: String) async throws
     func addressAdd(name: String, ipv4Address: CIDRv4) async throws
+    func routeAddLink(name: String, dstIPv4Addr: IPv4Address, srcIPv4Addr: IPv4Address?) async throws
     func routeAddDefault(name: String, ipv4Gateway: IPv4Address) async throws
     func configureDNS(config: DNS, location: String) async throws
     func configureHosts(config: Hosts, location: String) async throws

Sources/Containerization/Vminitd.swift

@@ -382,6 +382,19 @@ extension Vminitd {
             })
     }
 
+    /// Add a route in the sandbox's environment.
+    public func routeAddLink(name: String, dstIPv4Addr: IPv4Address, srcIPv4Addr: IPv4Address? = nil) async throws {
+        let dstCIDR = "\(dstIPv4Addr.description)/32"
+        _ = try await client.ipRouteAddLink(
+            .with {
+                $0.interface = name
+                $0.dstIpv4Addr = dstCIDR
+                if let srcIPv4Addr {
+                    $0.srcIpv4Addr = srcIPv4Addr.description
+                }
+            })
+    }
+
     /// Set the default route in the sandbox's environment.
     public func routeAddDefault(name: String, ipv4Gateway: IPv4Address) async throws {
         _ = try await client.ipRouteAddDefault(

Sources/ContainerizationNetlink/NetlinkSession.swift

@@ -267,15 +267,20 @@ public struct NetlinkSession {
     public func routeAdd(
         interface: String,
         dstIpv4Addr: CIDRv4,
-        srcIpv4Addr: IPv4Address
+        srcIpv4Addr: IPv4Address?
     ) throws {
-        // ip route add [dest-cidr] dev [interface] src [src-addr] proto kernel
+        // ip route add [dest-cidr] dev [interface] [src [src-addr]] proto kernel
         let interfaceIndex = try getInterfaceIndex(interface)
 
         let dstAddrBytes = dstIpv4Addr.address.bytes
         let dstAddrAttrSize = RTAttribute.size + dstAddrBytes.count
-        let srcAddrBytes = srcIpv4Addr.bytes
-        let srcAddrAttrSize = RTAttribute.size + srcAddrBytes.count
+        let srcAddrAttrSize: Int
+        if let srcIpv4Addr {
+            let srcAddrBytes = srcIpv4Addr.bytes
+            srcAddrAttrSize = RTAttribute.size + srcAddrBytes.count
+        } else {
+            srcAddrAttrSize = 0
+        }
         let interfaceAttrSize = RTAttribute.size + MemoryLayout<UInt32>.size
         let requestSize =
             NetlinkMessageHeader.size + RouteInfo.size + dstAddrAttrSize + srcAddrAttrSize + interfaceAttrSize
@@ -308,10 +313,14 @@ public struct NetlinkSession {
             throw BindError.sendMarshalFailure(type: "RTAttribute", field: "RTA_DST")
         }
 
-        let srcAddrAttr = RTAttribute(len: UInt16(dstAddrAttrSize), type: RouteAttributeType.PREFSRC)
-        requestOffset = try srcAddrAttr.appendBuffer(&requestBuffer, offset: requestOffset)
-        guard var requestOffset = requestBuffer.copyIn(buffer: srcAddrBytes, offset: requestOffset) else {
-            throw BindError.sendMarshalFailure(type: "RTAttribute", field: "RTA_PREFSRC")
+        if let srcIpv4Addr {
+            let srcAddrBytes = srcIpv4Addr.bytes
+            let srcAddrAttr = RTAttribute(len: UInt16(srcAddrAttrSize), type: RouteAttributeType.PREFSRC)
+            requestOffset = try srcAddrAttr.appendBuffer(&requestBuffer, offset: requestOffset)
+            guard let newOffset = requestBuffer.copyIn(buffer: srcAddrBytes, offset: requestOffset) else {
+                throw BindError.sendMarshalFailure(type: "RTAttribute", field: "RTA_PREFSRC")
+            }
+            requestOffset = newOffset
         }
 
         let interfaceAttr = RTAttribute(len: UInt16(interfaceAttrSize), type: RouteAttributeType.OIF)

Tests/ContainerizationNetlinkTests/NetlinkSessionTest.swift

@@ -311,8 +311,8 @@ struct NetlinkSessionTest {
         let expectedAddRequest =
             "3400000018000506000000000cc00cc0"  // Netlink header (16 B)
             + "02180000fe02fd0100000000"  // struct rtmsg (12 B): AF_INET, dst/24,
-            //   table=RT_TABLE_MAIN (0xfe), proto=RTPROT_BOOT (0x02),
-            //   scope=RT_SCOPE_UNIVERSE (0xfd), type=RTN_UNICAST (0x01)
+            //   table=RT_TABLE_MAIN (0xfe), proto=RTPROT_KERNEL (0x02),
+            //   scope=RT_SCOPE_LINK (0xfd), type=RTN_UNICAST (0x01)
             + "08000100c0a84000"  // RTA_DST     192.168.64.0
             + "08000700c0a84003"  // RTA_PREFSRC 192.168.64.3
             + "0800040002000000"  // RTA_OIF     ifindex 2 (eth0)
@@ -340,6 +340,55 @@ struct NetlinkSessionTest {
         #expect(expectedAddRequest == mockSocket.requests[1].hexEncodedString())
     }
 
+    @Test func testNetworkRouteAddIpLinkWithoutSrc() throws {
+        let mockSocket = try MockNetlinkSocket()
+        mockSocket.pid = 0xc00c_c00c
+
+        // Lookup interface by name, truncated response with no attributes (not needed at present).
+        let expectedLookupRequest =
+            "3400000012000100000000000cc00cc0"  // Netlink header (16 B)
+            + "110000000000000001000000ffffffff"  // struct ifinfomsg (16 B)
+            + "08001d00090000000c0003006574683000000000"  // RT attrs: IFLA_EXT_MASK + IFLA_IFNAME ("eth0")
+        mockSocket.responses.append(
+            [UInt8](
+                hex:
+                    "2000000010000000000000000cc00cc0"  // Netlink header (16 B)
+                    + "00000100020000004310010000000000"  // struct ifinfomsg (16 B) – no attributes
+            )
+        )
+
+        // Add link route without RTA_PREFSRC.
+        let expectedAddRequest =
+            "2c00000018000506000000000cc00cc0"  // Netlink header (16 B)
+            + "02180000fe02fd0100000000"  // struct rtmsg (12 B): AF_INET, dst/24,
+            //   table=RT_TABLE_MAIN (0xfe), proto=RTPROT_KERNEL (0x02),
+            //   scope=RT_SCOPE_LINK (0xfd), type=RTN_UNICAST (0x01)
+            + "08000100c0a84000"  // RTA_DST     192.168.64.0
+            + "0800040002000000"  // RTA_OIF     ifindex 2 (eth0)
+        mockSocket.responses.append(
+            [UInt8](
+                hex:
+                    "2400000002000001000000000cc00cc0"  // Netlink header (16 B)
+                    + "00000000280000001400050600000000"  // nlmsg_err payload (16 B)
+                    + "1f000000"  // first 4 B of echoed offending header
+            )
+        )
+
+        let session = NetlinkSession(socket: mockSocket)
+        try session.routeAdd(
+            interface: "eth0",
+            dstIpv4Addr: try CIDRv4("192.168.64.0/24"),
+            srcIpv4Addr: nil
+        )
+
+        #expect(mockSocket.requests.count == 2)
+        #expect(mockSocket.responseIndex == 2)
+        mockSocket.requests[0][8..<12] = [0, 0, 0, 0]
+        #expect(expectedLookupRequest == mockSocket.requests[0].hexEncodedString())
+        mockSocket.requests[1][8..<12] = [0, 0, 0, 0]
+        #expect(expectedAddRequest == mockSocket.requests[1].hexEncodedString())
+    }
+
     @Test func testNetworkLinkGetMultipleMessagesInSingleBuffer() throws {
         let mockSocket = try MockNetlinkSocket()
         mockSocket.pid = 0x8765_4321

vminitd/Sources/vminitd/Server+GRPC.swift

@@ -958,7 +958,7 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContextAsyncProvid
             let socket = try DefaultNetlinkSocket()
             let session = NetlinkSession(socket: socket, log: log)
             let dstIpv4Addr = try CIDRv4(request.dstIpv4Addr)
-            let srcIpv4Addr = try IPv4Address(request.srcIpv4Addr)
+            let srcIpv4Addr = request.srcIpv4Addr.isEmpty ? nil : try IPv4Address(request.srcIpv4Addr)
             try session.routeAdd(
                 interface: request.interface,
                 dstIpv4Addr: dstIpv4Addr,