Skip to content

Commit

Permalink
adding phy port id support for the sriov pf and vf
Browse files Browse the repository at this point in the history
  • Loading branch information
@rkamudhan
rkamudhan committed Feb 17, 2017
1 parent a019dfd commit da6280f
Showing 1 changed file with 130 additions and 22 deletions.
152 changes: 130 additions & 22 deletions sriov/sriov.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"runtime"
"strconv"
"strings"
"syscall"

"github.com/containernetworking/cni/pkg/ipam"
"github.com/containernetworking/cni/pkg/ns"
Expand All @@ -37,6 +38,7 @@ type NetConf struct {
CNIDir string `json:"cniDir"`
IF0 string `json:"if0"`
IF0NAME string `json:"if0name"`
L2Mode bool `json:"l2enable"`
Vlan int `json:"vlan"`
}

Expand Down Expand Up @@ -88,7 +90,7 @@ func loadConf(bytes []byte) (*NetConf, error) {

func saveScratchNetConf(containerID, dataDir string, netconf []byte) error {
if err := os.MkdirAll(dataDir, 0700); err != nil {
return fmt.Errorf("failed to create the multus data directory(%q): %v", dataDir, err)
return fmt.Errorf("failed to create the sriov data directory(%q): %v", dataDir, err)
}

path := filepath.Join(dataDir, containerID)
Expand Down Expand Up @@ -191,42 +193,103 @@ func getpciaddress(ifName string, vf int) (string, error) {
return pciaddr, nil
}

func setupVF(conf *NetConf, ifName string, podifName string, cid string, netns ns.NetNS) error {
func getphyPortID(ifName string) (string, error) {
var physportId string

var vfIdx int
var infos []os.FileInfo
var pciAddr string
physportidFile := fmt.Sprintf("/sys/class/net/%s/phys_port_id", ifName)
if _, err := os.Lstat(physportidFile); err != nil {
return physportId, fmt.Errorf("failed to open the phys_port_id file of device %q: %v", ifName, err)
}

m, err := netlink.LinkByName(ifName)
data, err := ioutil.ReadFile(physportidFile)
if err != nil {
return fmt.Errorf("failed to lookup master %q: %v", conf.IF0, err)
// skip nics that don't support phy port id
if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOTSUP {
return physportId, nil
}

return physportId, fmt.Errorf("failed to read the phys_port_id file of device %q: %v", ifName, err)
}

if len(data) == 0 {
return physportId, fmt.Errorf("no data in the file: %q", physportidFile)
}

physportId = strings.TrimSpace(string(data))

return physportId, nil

}

func compareportID(ifName string, portId string) (bool, error) {
physportId, err := getphyPortID(ifName)
if err != nil {
return false, err
}
if strings.Compare(physportId, portId) != 0 {
return false, nil
}

return true, nil

}

func getsriovNumfs(ifName string) (int, error) {
var vfTotal int

sriovFile := fmt.Sprintf("/sys/class/net/%s/device/sriov_numvfs", ifName)
if _, err := os.Lstat(sriovFile); err != nil {
return fmt.Errorf("failed to open the sriov_numfs of device %q: %v", ifName, err)
return vfTotal, fmt.Errorf("failed to open the sriov_numfs of device %q: %v", ifName, err)
}

data, err := ioutil.ReadFile(sriovFile)
if err != nil {
return fmt.Errorf("failed to read the sriov_numfs of device %q: %v", ifName, err)
return vfTotal, fmt.Errorf("failed to read the sriov_numfs of device %q: %v", ifName, err)
}

if len(data) == 0 {
return fmt.Errorf("no data in the file %q", sriovFile)
return vfTotal, fmt.Errorf("no data in the file %q", sriovFile)
}

sriovNumfs := strings.TrimSpace(string(data))
vfTotal, err := strconv.Atoi(sriovNumfs)
vfTotal, err = strconv.Atoi(sriovNumfs)
if err != nil {
return vfTotal, fmt.Errorf("failed to convert sriov_numfs(byte value) to int of device %q: %v", ifName, err)
}

return vfTotal, nil
}

func setupVF(conf *NetConf, ifName string, podifName string, cid string, netns ns.NetNS) error {

var vfIdx int
var infos []os.FileInfo
var pciAddr string
var vfDevName string

m, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to convert sriov_numfs(byte value) to int of device %q: %v", ifName, err)
return fmt.Errorf("failed to lookup master %q: %v", conf.IF0, err)
}

// get the ifname phy port id, if exist
physportId, err := getphyPortID(ifName)
if err != nil {
return fmt.Errorf("failed to get phy port id of %q: %v", ifName, err)
}

// get the ifname sriov vf num
vfTotal, err := getsriovNumfs(ifName)
if err != nil {
return err
}

if vfTotal <= 0 {
return fmt.Errorf("no virtual function in the device %q: %v", ifName)
}

for vf := 0; vf <= (vfTotal - 1); vf++ {
// get network interface of virtual function
vfDir := fmt.Sprintf("/sys/class/net/%s/device/virtfn%d/net", ifName, vf)
if _, err := os.Lstat(vfDir); err != nil {
if vf == (vfTotal - 1) {
Expand All @@ -248,8 +311,35 @@ func setupVF(conf *NetConf, ifName string, podifName string, cid string, netns n
continue
}

if len(infos) == 1 {
// if sriov vf has more than one interface
if len(infos) <= 2 {
var portId bool
var err error

vfIdx = vf
for i := 1; i <= len(infos) && portId != true; i++ {
vfDevName = infos[i-1].Name()

if len(physportId) != 0 {
// compare the phy port id of sriov vf net interface and pf
portId, err = compareportID(infos[i-1].Name(), physportId)
if err != nil {
return err
}

}
}

//return error, if no VF is available for the NIC support port ID
if !portId && len(physportId) != 0 && (vf == (vfTotal - 1)) {
return fmt.Errorf("no Virtual function exist in directory %s, last vf is virtfn%d", vfDir, vf)
}

if !portId && len(physportId) != 0 {
continue
}

// get the sriov vf net interface pci
pciAddr, err = getpciaddress(ifName, vfIdx)
if err != nil {
return fmt.Errorf("err in getting pci address - %q", err)
Expand All @@ -260,18 +350,14 @@ func setupVF(conf *NetConf, ifName string, podifName string, cid string, netns n
}
}

// VF NIC name
if len(infos) != 1 {
return fmt.Errorf("no virutal network resources avaiable for the %q", conf.IF0)
}

vfDevName := infos[0].Name()
if conf.DPDKMode != false {
conf.DPDKConf.PCIaddr = pciAddr
conf.DPDKConf.Ifname = podifName
// save the DPDK net conf in cniDir
if err = savedpdkConf(cid, conf.CNIDir, conf); err != nil {
return err
}
// bind the sriov vf to the DPDK driver
return enabledpdkmode(&conf.DPDKConf, vfDevName, true)
}

Expand Down Expand Up @@ -301,16 +387,27 @@ func setupVF(conf *NetConf, ifName string, podifName string, cid string, netns n
return fmt.Errorf("failed to rename %d vf of the device %q to %q: %v", vfIdx, vfDevName, podifName, err)
}

// for L2 mode enable the pod net interface
if conf.L2Mode != false {
err = setUpLink(podifName)
if err != nil {
return fmt.Errorf("failed to set up the pod interface name %q: %v", podifName, err)
}
}

return nil
})
}

func releaseVF(conf *NetConf, podifName string, cid string, netns ns.NetNS) error {
// check for the DPDK mode and release the allocated DPDK resources
if conf.DPDKMode != false {
df := &dpdkConf{}
// get the DPDK net conf in cniDir
if err := df.getdpdkConf(cid, conf.CNIDir, conf); err != nil {
return err
}
// bind the sriov vf to the kernel driver
return enabledpdkmode(df, df.Ifname, false)
}

Expand Down Expand Up @@ -372,12 +469,13 @@ func cmdAdd(args *skel.CmdArgs) error {
return fmt.Errorf("failed to set up pod interface %q from the device %q: %v", args.IfName, n.IF0, err)
}

// run the IPAM plugin and get back the config to apply
// skip the IPAM allocation for the DPDK and L2 mode
var result *types.Result
if n.DPDKMode != false {
if n.DPDKMode != false || n.L2Mode != false {
return result.Print()
}

// run the IPAM plugin and get back the config to apply
result, err = ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return fmt.Errorf("failed to set up IPAM plugin type %q from the device %q: %v", n.IPAM.Type, n.IF0, err)
Expand Down Expand Up @@ -417,7 +515,8 @@ func cmdDel(args *skel.CmdArgs) error {
return err
}

if n.DPDKMode != false {
// skip the IPAM release for the DPDK and L2 mode
if n.DPDKMode != false || n.L2Mode != false {
return nil
}

Expand All @@ -438,6 +537,15 @@ func renameLink(curName, newName string) error {
return netlink.LinkSetName(link, newName)
}

func setUpLink(ifName string) error {
link, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to set up device %q: %v", ifName, err)
}

return netlink.LinkSetUp(link)
}

func main() {
skel.PluginMain(cmdAdd, cmdDel)
}

0 comments on commit da6280f

Please sign in to comment.