Skip to content

Commit

Permalink
support disk waitforattach, getvolumename, default config for nas
Browse files Browse the repository at this point in the history
  • Loading branch information
@fredkan
fredkan committed Apr 21, 2019
1 parent 47561f3 commit 8e63825
Show file tree
Hide file tree
Showing 7 changed files with 124 additions and 32 deletions.
22 changes: 10 additions & 12 deletions package/entrypoint.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ if [ "$lsb_release_exist" = "0" ]; then
host_os="alios"
else
echo "OS is not ubuntu 1604/1404, Centos7"
echo "system information: "$os_info
exit 1
fi

Expand Down Expand Up @@ -64,8 +65,6 @@ install_disk() {
cp /acs/flexvolume /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~disk/disk
chmod 755 /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~disk/disk

restart_kubelet="true"

# update status
else
oldmd5=`md5sum /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~disk/disk | awk '{print $1}'`
Expand All @@ -77,7 +76,6 @@ install_disk() {
cp /acs/flexvolume /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~disk/disk
chmod 755 /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~disk/disk

# restart_kubelet="true"
fi
fi

Expand Down Expand Up @@ -116,8 +114,6 @@ install_nas() {
cp /acs/flexvolume /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~nas/nas
chmod 755 /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~nas/nas

restart_kubelet="true"

# update nas
else
oldmd5=`md5sum /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~nas/nas | awk '{print $1}'`
Expand All @@ -128,8 +124,6 @@ install_nas() {
rm -rf /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~nas/nas
cp /acs/flexvolume /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~nas/nas
chmod 755 /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~nas/nas

# restart_kubelet="true"
fi
fi

Expand Down Expand Up @@ -188,8 +182,6 @@ install_oss() {
cp /acs/flexvolume /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~oss/oss
chmod 755 /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~oss/oss

restart_kubelet="true"

# update oss
else
oldmd5=`md5sum /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~oss/oss | awk '{print $1}'`
Expand All @@ -201,7 +193,6 @@ install_oss() {
cp /acs/flexvolume /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~oss/oss
chmod 755 /host/usr/libexec/kubernetes/kubelet-plugins/volume/exec/alicloud~oss/oss

# restart_kubelet="true"
fi
fi

Expand Down Expand Up @@ -297,10 +288,17 @@ install_cpfs() {
}

# if kubelet not disable controller, exit
enableADController="false"
count=`ps -ef | grep kubelet | grep "enable-controller-attach-detach=false" | grep -v "grep" | wc -l`
if [ "$count" = "0" ]; then
echo "kubelet not running in: enable-controller-attach-detach=false"
exit 1
configInFile=`/acs/nsenter --mount=/proc/1/ns/mnt cat /var/lib/kubelet/config.yaml | grep enableControllerAttachDetach | grep false | grep -v grep | wc -l`
if [ "$configInFile" = "0" ]; then
enableADController=true
fi
fi

if [ "$enableADController" = "true" ]; then
echo "kubelet not running in: enable-controller-attach-detach=false, mount maybe failed"
fi

# install plugins
Expand Down
4 changes: 2 additions & 2 deletions provider/cpfs/cpfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func (p *CpfsPlugin) Unmount(mountPoint string) utils.Result {
umntCmd := fmt.Sprintf("umount %s", mountPoint)
if _, err := utils.Run(umntCmd); err != nil {
log.Errorf("Cpfs, Umount cpfs Fail: %s, %s", err.Error(), mountPoint)
utils.FinishError("Cpfs, Umount nfs Fail: " + err.Error())
utils.FinishError("Cpfs, Umount cpfs Fail: " + err.Error())
}

log.Info("Umount Cpfs Successful:", mountPoint)
Expand All @@ -146,7 +146,7 @@ func (p *CpfsPlugin) Getvolumename(opts interface{}) utils.Result {
}

// Not Support
func (p *CpfsPlugin) Waitforattach(opts interface{}) utils.Result {
func (p *CpfsPlugin) Waitforattach(devicePath string, opts interface{}) utils.Result {
return utils.NotSupport()
}

Expand Down
37 changes: 33 additions & 4 deletions provider/disk/disk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,14 +324,43 @@ func UnmountMountPoint(mountPath string) error {
return fmt.Errorf("Failed to unmount path")
}

// Not Support
// Support
func (p *DiskPlugin) Getvolumename(opts interface{}) utils.Result {
return utils.NotSupport()
opt := opts.(*DiskOptions)
return utils.Result{
Status: "Success",
VolumeName: opt.VolumeName,
}
}

// Not Support
func (p *DiskPlugin) Waitforattach(opts interface{}) utils.Result {
return utils.NotSupport()
func (p *DiskPlugin) Waitforattach(devicePath string, opts interface{}) utils.Result {
opt := opts.(*DiskOptions)
if devicePath == "" {
utils.FinishError("Waitforattach, devicePath is empty, cannot used for Volume: " + opt.VolumeName)
}
if ! utils.IsFileExisting(devicePath) {
utils.FinishError("Waitforattach, devicePath: " + devicePath + " is not exist, cannot used for Volume: " + opt.VolumeName)
}

// check the device is used for system
if devicePath == "/dev/vda" || devicePath == "/dev/vda1" {
utils.FinishError("Waitforattach, devicePath: " + devicePath + " is system device, cannot used for Volume: " + opt.VolumeName)
}
if devicePath == "/dev/vdb1" {
checkCmd := fmt.Sprintf("mount | grep \"/dev/vdb1 on /var/lib/kubelet type\" | wc -l")
if out, err := utils.Run(checkCmd); err != nil {
utils.FinishError("Waitforattach, devicePath: " + devicePath + " is check vdb error for Volume: " + opt.VolumeName)
} else if strings.TrimSpace(out) != "0" {
utils.FinishError("Waitforattach, devicePath: " + devicePath + " is used as DataDisk for kubelet, cannot used fo Volume: " + opt.VolumeName)
}
}

log.Infof("Waitforattach, wait for attach: %s, %s", devicePath, opt.VolumeName)
return utils.Result{
Status: "Success",
Device: devicePath,
}
}

// Not Support
Expand Down
37 changes: 30 additions & 7 deletions provider/driver/k8s.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ import (
"strings"
"time"

"github.com/AliyunContainerService/flexvolume/provider/cpfs"
"github.com/AliyunContainerService/flexvolume/provider/disk"
"github.com/AliyunContainerService/flexvolume/provider/monitor"
"github.com/AliyunContainerService/flexvolume/provider/nas"
"github.com/AliyunContainerService/flexvolume/provider/oss"
"github.com/AliyunContainerService/flexvolume/provider/utils"
log "github.com/sirupsen/logrus"
cpfs "github.com/AliyunContainerService/flexvolume/provider/cpfs"
disk "github.com/AliyunContainerService/flexvolume/provider/disk"
monitor "github.com/AliyunContainerService/flexvolume/provider/monitor"
nas "github.com/AliyunContainerService/flexvolume/provider/nas"
oss "github.com/AliyunContainerService/flexvolume/provider/oss"
utils "github.com/AliyunContainerService/flexvolume/provider/utils"
)

// VolumePlugin interface for plugins
Expand All @@ -22,7 +22,7 @@ type FluxVolumePlugin interface {
Init() utils.Result
Getvolumename(opt interface{}) utils.Result
Attach(opt interface{}, nodeName string) utils.Result
Waitforattach(opt interface{}) utils.Result
Waitforattach(devicePath string, opt interface{}) utils.Result
Mountdevice(mountPath string, opt interface{}) utils.Result
Detach(volumeName string, nodeName string) utils.Result
Mount(opt interface{}, mountPath string) utils.Result
Expand Down Expand Up @@ -115,6 +115,29 @@ func RunPlugin(plugin FluxVolumePlugin) {
mountPath := os.Args[2]
utils.Finish(plugin.Unmount(mountPath))

case "waitforattach":
if len(os.Args) != 4 {
utils.FinishError("waitforattach expected exactly 4 arguments; got: " + strings.Join(os.Args, ","))
}
opt := plugin.NewOptions()
if err := json.Unmarshal([]byte(os.Args[3]), opt); err != nil {
utils.FinishError("waitforattach Options illegal; got: " + os.Args[3])
}

devicePath := os.Args[2]
utils.Finish(plugin.Waitforattach(devicePath, opt))

case "getvolumename":
if len(os.Args) != 3 {
utils.FinishError("getvolumename expected exactly 3 arguments; got: " + strings.Join(os.Args, ","))
}
opt := plugin.NewOptions()
if err := json.Unmarshal([]byte(os.Args[2]), opt); err != nil {
utils.FinishError("GetVolumeName Options illegal; got: " + os.Args[3])
}

utils.Finish(plugin.Getvolumename(opt))

default:
utils.Finish(utils.NotSupport(os.Args))
}
Expand Down
43 changes: 40 additions & 3 deletions provider/nas/nas.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ func (p *NasPlugin) Mount(opts interface{}, mountPath string) utils.Result {
// Add NAS white list if needed
// updateNasWhiteList(opt)

// if system not set nas, config it.
checkSystemNasConfig()

// Create Mount Path
if err := utils.CreateDest(mountPath); err != nil {
utils.FinishError("Nas, Mount error with create Path fail: " + mountPath)
Expand Down Expand Up @@ -117,6 +120,36 @@ func (p *NasPlugin) Mount(opts interface{}, mountPath string) utils.Result {
return utils.Result{Status: "Success"}
}

// check system config,
// if tcp_slot_table_entries not set to 128, just config.
func checkSystemNasConfig() {
updateNasConfig := false
sunRpcFile := "/etc/modprobe.d/sunrpc.conf"
if ! utils.IsFileExisting(sunRpcFile) {
updateNasConfig = true
} else {
chkCmd := fmt.Sprintf("cat %s | grep tcp_slot_table_entries | grep 128 | grep -v grep | wc -l", sunRpcFile)
out, err := utils.Run(chkCmd)
if err != nil {
log.Warnf("Update Nas system config check error: ", err.Error())
return
}
if strings.TrimSpace(out) == "0" {
updateNasConfig = true
}
}

if updateNasConfig {
upCmd := fmt.Sprintf("echo \"options sunrpc tcp_slot_table_entries=128\" >> %s && echo \"options sunrpc tcp_max_slot_table_entries=128\" >> %s && sysctl -w sunrpc.tcp_slot_table_entries=128", sunRpcFile, sunRpcFile)
_, err := utils.Run(upCmd)
if err != nil {
log.Warnf("Update Nas system config error: ", err.Error())
return
}
log.Warnf("Successful update Nas system config")
}
}

func (p *NasPlugin) Unmount(mountPoint string) utils.Result {
log.Infof("Nas Plugin Umount: %s", strings.Join(os.Args, ","))

Expand Down Expand Up @@ -196,13 +229,17 @@ func (p *NasPlugin) Detach(device string, nodeName string) utils.Result {
return utils.NotSupport()
}

// Not Support
// Support
func (p *NasPlugin) Getvolumename(opts interface{}) utils.Result {
return utils.NotSupport()
opt := opts.(*NasOptions)
return utils.Result{
Status: "Success",
VolumeName: opt.VolumeName,
}
}

// Not Support
func (p *NasPlugin) Waitforattach(opts interface{}) utils.Result {
func (p *NasPlugin) Waitforattach(devicePath string, opts interface{}) utils.Result {
return utils.NotSupport()
}

Expand Down
10 changes: 7 additions & 3 deletions provider/oss/oss.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,17 @@ func (p *OssPlugin) Detach(device string, nodeName string) utils.Result {
return utils.NotSupport()
}

// Not Support
// Support
func (p *OssPlugin) Getvolumename(opts interface{}) utils.Result {
return utils.NotSupport()
opt := opts.(*OssOptions)
return utils.Result{
Status: "Success",
VolumeName: opt.VolumeName,
}
}

// Not Support
func (p *OssPlugin) Waitforattach(opts interface{}) utils.Result {
func (p *OssPlugin) Waitforattach(devicePath string, opts interface{}) utils.Result {
return utils.NotSupport()
}

Expand Down
3 changes: 2 additions & 1 deletion provider/utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import (
"strings"
"syscall"

log "github.com/sirupsen/logrus"
"github.com/denverdino/aliyungo/metadata"
log "github.com/sirupsen/logrus"
)

// used for global ak
Expand Down Expand Up @@ -81,6 +81,7 @@ type Result struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
Device string `json:"device,omitempty"`
VolumeName string `json:"volumeName"`
}

// run shell command
Expand Down

0 comments on commit 8e63825

Please sign in to comment.