Skip to content

Commit b5d71df

Browse files
nvazqueznvazquez
authored
FR02 - DPDK client and server mode (#2)
* Initial approach * Refactor and unit tests * Dont add vhost user mode value as an extra config on the VM XML * SQL text field and DPDK classes refactor * Fix NullPointerException after refactor
1 parent 369fb25 commit b5d71df

15 files changed

Lines changed: 609 additions & 79 deletions

File tree

‎engine/schema/resources/META-INF/db/schema-41100to41110.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,6 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis
9191
-- XCP-NG 7.4
9292
INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, storage_motion_supported) values (UUID(), 'XenServer', 'XCP-ng 7.4.0', 500, 13, 1);
9393
INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', 'XCP-ng 7.4.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='7.4.0';
94+
95+
-- DPDK client and server mode support
96+
ALTER TABLE `cloud`.`service_offering_details` CHANGE COLUMN `value` `value` TEXT NOT NULL;

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/dpdk/DPDKDriver.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.hypervisor.kvm.dpdk;
20+
21+
import com.cloud.utils.component.Adapter;
22+
23+
import java.util.Map;
24+
25+
public interface DPDKDriver extends Adapter {
26+
27+
/**
28+
* Get the next DPDK port name to be created
29+
*/
30+
String getNextDpdkPort();
31+
32+
/**
33+
* Get the latest DPDK port number created on a DPDK enabled host
34+
*/
35+
int getDpdkLatestPortNumberUsed();
36+
37+
/**
38+
* Add OVS port (if it does not exist) to bridge with DPDK support
39+
*/
40+
void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath);
41+
42+
/**
43+
* Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
44+
*/
45+
String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode);
46+
47+
/**
48+
* Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
49+
*/
50+
DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig);
51+
52+
/**
53+
* Check for additional extra 'dpdk-interface' configurations, return them appended
54+
*/
55+
String getExtraDpdkProperties(Map<String, String> extraConfig);
56+
}

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/dpdk/DPDKDriverImpl.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.hypervisor.kvm.dpdk;
20+
21+
import com.cloud.utils.component.AdapterBase;
22+
import com.cloud.utils.script.Script;
23+
import org.apache.commons.lang.StringUtils;
24+
import org.apache.log4j.Logger;
25+
26+
import java.util.Map;
27+
28+
public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
29+
static final String DPDK_PORT_PREFIX = "csdpdk-";
30+
31+
private final String dpdkPortVhostUserType = "dpdkvhostuser";
32+
private final String dpdkPortVhostUserClientType = "dpdkvhostuserclient";
33+
34+
private static final Logger s_logger = Logger.getLogger(DPDKDriver.class);
35+
36+
public DPDKDriverImpl() {
37+
}
38+
39+
/**
40+
* Get the next DPDK port name to be created
41+
*/
42+
public String getNextDpdkPort() {
43+
int portNumber = getDpdkLatestPortNumberUsed();
44+
return DPDK_PORT_PREFIX + String.valueOf(portNumber + 1);
45+
}
46+
47+
/**
48+
* Get the latest DPDK port number created on a DPDK enabled host
49+
*/
50+
public int getDpdkLatestPortNumberUsed() {
51+
s_logger.debug("Checking the last DPDK port created");
52+
String cmd = "ovs-vsctl show | grep Port | grep " + DPDK_PORT_PREFIX + " | " +
53+
"awk '{ print $2 }' | sort -rV | head -1";
54+
String port = Script.runSimpleBashScript(cmd);
55+
int portNumber = 0;
56+
if (StringUtils.isNotBlank(port)) {
57+
String unquotedPort = port.replace("\"", "");
58+
String dpdkPortNumber = unquotedPort.split(DPDK_PORT_PREFIX)[1];
59+
portNumber = Integer.valueOf(dpdkPortNumber);
60+
}
61+
return portNumber;
62+
}
63+
64+
/**
65+
* Add OVS port (if it does not exist) to bridge with DPDK support
66+
*/
67+
public void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath) {
68+
String type = vHostUserMode == DPDKHelper.VHostUserMode.SERVER ?
69+
dpdkPortVhostUserType :
70+
dpdkPortVhostUserClientType;
71+
72+
StringBuilder stringBuilder = new StringBuilder();
73+
stringBuilder.append(String.format("ovs-vsctl add-port %s %s " +
74+
"vlan_mode=access tag=%s " +
75+
"-- set Interface %s type=%s", bridgeName, port, vlan, port, type));
76+
77+
if (vHostUserMode == DPDKHelper.VHostUserMode.CLIENT) {
78+
stringBuilder.append(String.format(" options:vhost-server-path=%s/%s",
79+
dpdkOvsPath, port));
80+
}
81+
82+
String cmd = stringBuilder.toString();
83+
s_logger.debug("DPDK property enabled, executing: " + cmd);
84+
Script.runSimpleBashScript(cmd);
85+
}
86+
87+
/**
88+
* Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
89+
*/
90+
public String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode) {
91+
return dpdKvHostUserMode == DPDKHelper.VHostUserMode.CLIENT ? "server" : "client";
92+
}
93+
94+
/**
95+
* Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
96+
*/
97+
public DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig) {
98+
return extraConfig.containsKey(DPDKHelper.DPDK_VHOST_USER_MODE) ?
99+
DPDKHelper.VHostUserMode.fromValue(extraConfig.get(DPDKHelper.DPDK_VHOST_USER_MODE)) :
100+
DPDKHelper.VHostUserMode.SERVER;
101+
}
102+
103+
/**
104+
* Check for additional extra 'dpdk-interface' configurations, return them appended
105+
*/
106+
public String getExtraDpdkProperties(Map<String, String> extraConfig) {
107+
StringBuilder stringBuilder = new StringBuilder();
108+
for (String key : extraConfig.keySet()) {
109+
if (key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX)) {
110+
stringBuilder.append(extraConfig.get(key));
111+
}
112+
}
113+
return stringBuilder.toString();
114+
}
115+
}

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@
4747
import javax.xml.parsers.DocumentBuilderFactory;
4848
import javax.xml.parsers.ParserConfigurationException;
4949

50+
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
5051
import com.cloud.resource.RequestWrapper;
51-
import org.apache.cloudstack.api.ApiConstants;
5252
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
5353
import org.apache.cloudstack.storage.to.VolumeObjectTO;
5454
import org.apache.cloudstack.utils.hypervisor.HypervisorUtils;
@@ -528,9 +528,6 @@ protected enum BridgeType {
528528

529529
protected boolean dpdkSupport = false;
530530
protected String dpdkOvsPath;
531-
protected static final String DPDK_NUMA = ApiConstants.EXTRA_CONFIG + "-dpdk-numa";
532-
protected static final String DPDK_HUGE_PAGES = ApiConstants.EXTRA_CONFIG + "-dpdk-hugepages";
533-
protected static final String DPDK_INTERFACE_PREFIX = ApiConstants.EXTRA_CONFIG + "-dpdk-interface-";
534531

535532
private String getEndIpFromStartIp(final String startIp, final int numIps) {
536533
final String[] tokens = startIp.split("[.]");
@@ -2047,7 +2044,7 @@ public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
20472044
vm.setPlatformEmulator(vmTO.getPlatformEmulator());
20482045

20492046
Map<String, String> extraConfig = vmTO.getExtraConfig();
2050-
if (dpdkSupport && (!extraConfig.containsKey(DPDK_NUMA) || !extraConfig.containsKey(DPDK_HUGE_PAGES))) {
2047+
if (dpdkSupport && (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA) || !extraConfig.containsKey(DPDKHelper.DPDK_HUGE_PAGES))) {
20512048
s_logger.info("DPDK is enabled but it needs extra configurations for CPU NUMA and Huge Pages for VM deployment");
20522049
}
20532050

@@ -2084,7 +2081,7 @@ public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
20842081
grd.setVcpuNum(vcpus);
20852082
vm.addComp(grd);
20862083

2087-
if (!extraConfig.containsKey(DPDK_NUMA)) {
2084+
if (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA)) {
20882085
final CpuModeDef cmd = new CpuModeDef();
20892086
cmd.setMode(_guestCpuMode);
20902087
cmd.setModel(_guestCpuModel);
@@ -2222,7 +2219,7 @@ protected void addExtraConfigComponent(Map<String, String> extraConfig, LibvirtV
22222219
if (MapUtils.isNotEmpty(extraConfig)) {
22232220
StringBuilder extraConfigBuilder = new StringBuilder();
22242221
for (String key : extraConfig.keySet()) {
2225-
if (!key.startsWith(DPDK_INTERFACE_PREFIX)) {
2222+
if (!key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX) && !key.equals(DPDKHelper.DPDK_VHOST_USER_MODE)) {
22262223
extraConfigBuilder.append(extraConfig.get(key));
22272224
}
22282225
}

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -952,6 +952,7 @@ enum HostNicType {
952952
private String _dpdkSourcePath;
953953
private String _dpdkSourcePort;
954954
private String _dpdkExtraLines;
955+
private String _interfaceMode;
955956

956957
public void defBridgeNet(String brName, String targetBrName, String macAddr, NicModel model) {
957958
defBridgeNet(brName, targetBrName, macAddr, model, 0);
@@ -966,14 +967,16 @@ public void defBridgeNet(String brName, String targetBrName, String macAddr, Nic
966967
_networkRateKBps = networkRateKBps;
967968
}
968969

969-
public void defDpdkNet(String dpdkSourcePath, String dpdkPort, String macAddress, NicModel model, Integer networkRateKBps, String extra) {
970+
public void defDpdkNet(String dpdkSourcePath, String dpdkPort, String macAddress, NicModel model,
971+
Integer networkRateKBps, String extra, String interfaceMode) {
970972
_netType = GuestNetType.VHOSTUSER;
971973
_dpdkSourcePath = dpdkSourcePath;
972974
_dpdkSourcePort = dpdkPort;
973975
_macAddr = macAddress;
974976
_model = model;
975977
_networkRateKBps = networkRateKBps;
976978
_dpdkExtraLines = extra;
979+
_interfaceMode = interfaceMode;
977980
}
978981

979982
public void defDirectNet(String sourceName, String targetName, String macAddr, NicModel model, String sourceMode) {
@@ -1120,7 +1123,8 @@ public String toString() {
11201123
} else if (_netType == GuestNetType.DIRECT) {
11211124
netBuilder.append("<source dev='" + _sourceName + "' mode='" + _netSourceMode + "'/>\n");
11221125
} else if (_netType == GuestNetType.VHOSTUSER) {
1123-
netBuilder.append("<source type='unix' path='"+ _dpdkSourcePath + _dpdkSourcePort + "' mode='client'/>\n");
1126+
netBuilder.append("<source type='unix' path='"+ _dpdkSourcePath + _dpdkSourcePort +
1127+
"' mode='" + _interfaceMode + "'/>\n");
11241128
}
11251129
if (_networkName != null) {
11261130
netBuilder.append("<target dev='" + _networkName + "'/>\n");

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java‎

Lines changed: 19 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424

2525
import javax.naming.ConfigurationException;
2626

27+
import com.cloud.hypervisor.kvm.dpdk.DPDKDriver;
28+
import com.cloud.hypervisor.kvm.dpdk.DPDKDriverImpl;
29+
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
2730
import com.cloud.utils.exception.CloudRuntimeException;
2831
import org.apache.commons.lang.StringUtils;
2932
import org.apache.log4j.Logger;
@@ -41,8 +44,7 @@
4144
public class OvsVifDriver extends VifDriverBase {
4245
private static final Logger s_logger = Logger.getLogger(OvsVifDriver.class);
4346
private int _timeout;
44-
45-
protected static final String DPDK_PORT_PREFIX = "csdpdk-";
47+
private DPDKDriver dpdkDriver;
4648

4749
@Override
4850
public void configure(Map<String, Object> params) throws ConfigurationException {
@@ -55,6 +57,11 @@ public void configure(Map<String, Object> params) throws ConfigurationException
5557
networkScriptsDir = "scripts/vm/network/vnet";
5658
}
5759

60+
String dpdk = (String) params.get("openvswitch.dpdk.enabled");
61+
if (StringUtils.isNotBlank(dpdk) && Boolean.parseBoolean(dpdk)) {
62+
dpdkDriver = new DPDKDriverImpl();
63+
}
64+
5865
String value = (String)params.get("scripts.timeout");
5966
_timeout = NumbersUtil.parseInt(value, 30 * 60) * 1000;
6067
}
@@ -80,55 +87,6 @@ public void getPifs() {
8087
s_logger.debug("done looking for pifs, no more bridges");
8188
}
8289

83-
/**
84-
* Get the latest DPDK port number created on a DPDK enabled host
85-
*/
86-
protected int getDpdkLatestPortNumberUsed() {
87-
s_logger.debug("Checking the last DPDK port created");
88-
String cmd = "ovs-vsctl show | grep Port | grep " + DPDK_PORT_PREFIX + " | " +
89-
"awk '{ print $2 }' | sort -rV | head -1";
90-
String port = Script.runSimpleBashScript(cmd);
91-
int portNumber = 0;
92-
if (StringUtils.isNotBlank(port)) {
93-
String unquotedPort = port.replace("\"", "");
94-
String dpdkPortNumber = unquotedPort.split(DPDK_PORT_PREFIX)[1];
95-
portNumber = Integer.valueOf(dpdkPortNumber);
96-
}
97-
return portNumber;
98-
}
99-
100-
/**
101-
* Get the next DPDK port name to be created
102-
*/
103-
protected String getNextDpdkPort() {
104-
int portNumber = getDpdkLatestPortNumberUsed();
105-
return DPDK_PORT_PREFIX + String.valueOf(portNumber + 1);
106-
}
107-
108-
/**
109-
* Add OVS port (if it does not exist) to bridge with DPDK support
110-
*/
111-
protected void addDpdkPort(String bridgeName, String port, String vlan) {
112-
String cmd = String.format("ovs-vsctl add-port %s %s " +
113-
"vlan_mode=access tag=%s " +
114-
"-- set Interface %s type=dpdkvhostuser", bridgeName, port, vlan, port);
115-
s_logger.debug("DPDK property enabled, executing: " + cmd);
116-
Script.runSimpleBashScript(cmd);
117-
}
118-
119-
/**
120-
* Check for additional extra 'dpdk-interface' configurations, return them appended
121-
*/
122-
private String getExtraDpdkProperties(Map<String, String> extraConfig) {
123-
StringBuilder stringBuilder = new StringBuilder();
124-
for (String key : extraConfig.keySet()) {
125-
if (key.startsWith(LibvirtComputingResource.DPDK_INTERFACE_PREFIX)) {
126-
stringBuilder.append(extraConfig.get(key));
127-
}
128-
}
129-
return stringBuilder.toString();
130-
}
131-
13290
@Override
13391
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
13492
s_logger.debug("plugging nic=" + nic);
@@ -158,12 +116,18 @@ public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<S
158116
if (trafficLabel != null && !trafficLabel.isEmpty()) {
159117
if (_libvirtComputingResource.dpdkSupport && !nic.isDpdkDisabled()) {
160118
s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
161-
if (StringUtils.isBlank(_libvirtComputingResource.dpdkOvsPath)) {
119+
String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
120+
if (StringUtils.isBlank(dpdkOvsPath)) {
162121
throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
163122
}
164-
String port = getNextDpdkPort();
165-
addDpdkPort(_pifs.get(trafficLabel), port, vlanId);
166-
intf.defDpdkNet(_libvirtComputingResource.dpdkOvsPath, port, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), 0, getExtraDpdkProperties(extraConfig));
123+
String port = dpdkDriver.getNextDpdkPort();
124+
DPDKHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDPDKvHostUserMode(extraConfig);
125+
dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
126+
String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDPDKVhostUserMode(dpdKvHostUserMode);
127+
intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
128+
getGuestNicModel(guestOsType, nicAdapter), 0,
129+
dpdkDriver.getExtraDpdkProperties(extraConfig),
130+
interfaceMode);
167131
} else {
168132
s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
169133
intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);

0 commit comments

Comments
 (0)