Skip to content

Commit

Permalink
test: use mock puller and mounter in the node server tests (#83)
Browse files Browse the repository at this point in the history
* edit: docs to run node server with containerd image
* refactor: remove commented out code
* chore: switch over from criapi `v1alpha2` -> `v1`
* chore: fix import path name `csi-driver-image` -> `container-image-csi-driver`
* docs: replace `warm metal` -> `container-image-csi-driver`
* test: fix socket path `/tmp/csi/csi.sock` -> `/tmp/csi.sock`
  • Loading branch information
vadasambar authored Feb 14, 2024
1 parent 4d6fa73 commit 82d2e85
Show file tree
Hide file tree
Showing 13 changed files with 186 additions and 76 deletions.
9 changes: 9 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,15 @@ sanity:
e2e:
cd ./test/e2e && KUBECONFIG=~/.kube/config go run .

# to run unit tests
# PHONY signifies the make recipe is not a file
# more info: https://stackoverflow.com/questions/2145590/what-is-the-purpose-of-phony-in-a-makefile
.PHONY: unit-tests
unit-tests:
# -count=1 forces re-test everytime (instead of caching the results)
# more info: https://stackoverflow.com/questions/48882691/force-retesting-or-disable-test-caching
go test -count=1 ./cmd/plugin

.PHONY: integration
integration:
./test/integration/test-backward-compatability.sh
Expand Down
2 changes: 1 addition & 1 deletion cmd/plugin/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func main() {
*runtimeAddr = addr.String()
}

var mounter *backend.SnapshotMounter
var mounter backend.Mounter
if len(*runtimeAddr) > 0 {
addr, err := url.Parse(*runtimeAddr)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/plugin/node_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ const (

type ImagePullStatus int

func NewNodeServer(driver *csicommon.CSIDriver, mounter *backend.SnapshotMounter, imageSvc cri.ImageServiceClient, secretStore secret.Store, asyncImagePullMount bool) *NodeServer {
func NewNodeServer(driver *csicommon.CSIDriver, mounter backend.Mounter, imageSvc cri.ImageServiceClient, secretStore secret.Store, asyncImagePullMount bool) *NodeServer {
return &NodeServer{
DefaultNodeServer: csicommon.NewDefaultNodeServer(driver),
mounter: mounter,
Expand All @@ -52,7 +52,7 @@ func NewNodeServer(driver *csicommon.CSIDriver, mounter *backend.SnapshotMounter

type NodeServer struct {
*csicommon.DefaultNodeServer
mounter *backend.SnapshotMounter
mounter backend.Mounter
secretStore secret.Store
asyncImagePullMount bool
mountExecutor *mountexecutor.MountExecutor
Expand Down
67 changes: 35 additions & 32 deletions cmd/plugin/node_server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,24 @@ import (
"github.com/warm-metal/container-image-csi-driver/pkg/backend/containerd"
"github.com/warm-metal/container-image-csi-driver/pkg/cri"
"github.com/warm-metal/container-image-csi-driver/pkg/metrics"
"github.com/warm-metal/container-image-csi-driver/pkg/test/utils"
csicommon "github.com/warm-metal/csi-drivers/pkg/csi-common"
"google.golang.org/grpc"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/credentialprovider"
)

// Check test/integration/node-server/README.md for how to run this test correctly
func TestNodePublishVolumeAsync(t *testing.T) {
socketAddr := "unix:///run/containerd/containerd.sock"
addr, err := url.Parse(socketAddr)
assert.NoError(t, err)

criClient, err := cri.NewRemoteImageService(socketAddr, time.Minute)
assert.NoError(t, err)
assert.NotNil(t, criClient)
criClient := &utils.MockImageServiceClient{
PulledImages: make(map[string]bool),
ImagePullTime: time.Second * 5,
}

mounter := containerd.NewMounter(addr.Path)
assert.NotNil(t, mounter)
mounter := &utils.MockMounter{
ImageSvcClient: *criClient,
Mounted: make(map[string]bool),
}

driver := csicommon.NewCSIDriver(driverName, driverVersion, "fake-node")
assert.NotNil(t, driver)
Expand Down Expand Up @@ -71,20 +70,23 @@ func TestNodePublishVolumeAsync(t *testing.T) {

server := csicommon.NewNonBlockingGRPCServer()

addr, err = url.Parse(*endpoint)
assert.NoError(t, err)

os.Remove("/csi/csi.sock")
endpoint := "unix:///tmp/csi.sock"

// automatically deleted when the server is stopped
f, err := os.Create("/csi/csi.sock")
f, err := os.Create("/tmp/csi.sock")
assert.NoError(t, err)
assert.NotNil(t, f)

defer os.Remove("/tmp/csi.sock")

addr, err := url.Parse("unix:///tmp/csi.sock")
assert.NoError(t, err)

var wg sync.WaitGroup
wg.Add(1)
go func() {
server.Start(*endpoint,

server.Start(endpoint,
nil,
nil,
ns)
Expand Down Expand Up @@ -154,16 +156,14 @@ func TestNodePublishVolumeAsync(t *testing.T) {

// Check test/integration/node-server/README.md for how to run this test correctly
func TestNodePublishVolumeSync(t *testing.T) {
socketAddr := "unix:///run/containerd/containerd.sock"
addr, err := url.Parse(socketAddr)
assert.NoError(t, err)

criClient, err := cri.NewRemoteImageService(socketAddr, time.Minute)
assert.NoError(t, err)
assert.NotNil(t, criClient)

mounter := containerd.NewMounter(addr.Path)
assert.NotNil(t, mounter)
criClient := &utils.MockImageServiceClient{
PulledImages: make(map[string]bool),
ImagePullTime: time.Second * 5,
}
mounter := &utils.MockMounter{
ImageSvcClient: *criClient,
Mounted: make(map[string]bool),
}

driver := csicommon.NewCSIDriver(driverName, driverVersion, "fake-node")
assert.NotNil(t, driver)
Expand Down Expand Up @@ -197,20 +197,23 @@ func TestNodePublishVolumeSync(t *testing.T) {

server := csicommon.NewNonBlockingGRPCServer()

addr, err = url.Parse(*endpoint)
assert.NoError(t, err)

os.Remove("/csi/csi.sock")
endpoint := "unix:///tmp/csi.sock"

// automatically deleted when the server is stopped
f, err := os.Create("/csi/csi.sock")
f, err := os.Create("/tmp/csi.sock")
assert.NoError(t, err)
assert.NotNil(t, f)

defer os.Remove("/tmp/csi/csi.sock")

addr, err := url.Parse("unix:///tmp/csi.sock")
assert.NoError(t, err)

var wg sync.WaitGroup
wg.Add(1)
go func() {
server.Start(*endpoint,

server.Start(endpoint,
nil,
nil,
ns)
Expand Down
2 changes: 1 addition & 1 deletion pkg/backend/containerd/containerd.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type snapshotMounter struct {
cli *containerd.Client
}

func NewMounter(socketPath string) *backend.SnapshotMounter {
func NewMounter(socketPath string) backend.Mounter {
c, err := containerd.New(socketPath, containerd.WithDefaultNamespace("k8s.io"))
if err != nil {
klog.Fatalf("containerd connection is broken because the mounted unix socket somehow dose not work,"+
Expand Down
12 changes: 6 additions & 6 deletions pkg/backend/mounter.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
)

type SnapshotMounter struct {
runtime ContainerRuntime
runtime ContainerRuntimeMounter

guard sync.Mutex
// mapping from targets to key of read-only snapshots
Expand All @@ -21,7 +21,7 @@ type SnapshotMounter struct {
roSnapshotTargetsMap map[SnapshotKey]map[MountTarget]struct{}
}

func NewMounter(runtime ContainerRuntime) *SnapshotMounter {
func NewMounter(runtime ContainerRuntimeMounter) *SnapshotMounter {
mounter := &SnapshotMounter{
runtime: runtime,
targetRoSnapshotMap: make(map[MountTarget]SnapshotKey),
Expand Down Expand Up @@ -173,7 +173,7 @@ func (s *SnapshotMounter) Mount(
klog.Fatalf("invalid image id of image %q", image)
}

key = genSnapshotKey(imageID)
key = GenSnapshotKey(imageID)
klog.Infof("refer read-only snapshot of image %q with key %q", image, key)
if err := s.refROSnapshot(ctx, target, imageID, key, createSnapshotMetaData(target)); err != nil {
return err
Expand All @@ -189,7 +189,7 @@ func (s *SnapshotMounter) Mount(
}()
} else {
// For read-write volumes, they must be ephemeral volumes, that which volumeIDs are unique strings.
key = genSnapshotKey(volumeId)
key = GenSnapshotKey(volumeId)
klog.Infof("create read-write snapshot of image %q with key %q", image, key)
if err := s.runtime.PrepareRWSnapshot(ctx, imageID, key, nil); err != nil {
return err
Expand Down Expand Up @@ -221,13 +221,13 @@ func (s *SnapshotMounter) Unmount(ctx context.Context, volumeId string, target M

klog.Infof("delete the read-write snapshot")
// Must be a read-write snapshot
return s.runtime.DestroySnapshot(ctx, genSnapshotKey(volumeId))
return s.runtime.DestroySnapshot(ctx, GenSnapshotKey(volumeId))
}

func (s *SnapshotMounter) ImageExists(ctx context.Context, image docker.Named) bool {
return s.runtime.ImageExists(ctx, image)
}

func genSnapshotKey(parent string) SnapshotKey {
func GenSnapshotKey(parent string) SnapshotKey {
return SnapshotKey(fmt.Sprintf("csi-image.warm-metal.tech-%s", parent))
}
16 changes: 15 additions & 1 deletion pkg/backend/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ type MountOptions struct {
type SnapshotKey string
type MountTarget string

type ContainerRuntime interface {
// ContainerRuntimeMounter is a container runtime specific interface
type ContainerRuntimeMounter interface {
Mount(ctx context.Context, key SnapshotKey, target MountTarget, ro bool) error
Unmount(ctx context.Context, target MountTarget) error

Expand Down Expand Up @@ -43,3 +44,16 @@ type ContainerRuntime interface {
// The snapshot key must also be saved in the returned map with the key "FakeMetaDataSnapshotKey".
ListSnapshots(ctx context.Context) ([]SnapshotMetadata, error)
}

// Mounter is a generic interface used for mounting images
type Mounter interface {
// Mount mounts a specific image
Mount(
ctx context.Context, volumeId string, target MountTarget, image docker.Named, ro bool) (err error)

// Unmount unmounts a specific image
Unmount(ctx context.Context, volumeId string, target MountTarget) error

// ImageExists checks if the image already exists on the local machine
ImageExists(ctx context.Context, image docker.Named) bool
}
4 changes: 2 additions & 2 deletions pkg/mountexecutor/mountexecutor.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ const (
// MountExecutorOptions are options passed to mount executor
type MountExecutorOptions struct {
AsyncMount bool
Mounter *backend.SnapshotMounter
Mounter backend.Mounter
}

// MountOptions are options for a single mount request
Expand All @@ -44,7 +44,7 @@ type MountOptions struct {
type MountExecutor struct {
asyncMount bool
mutex *sync.Mutex
mounter *backend.SnapshotMounter
mounter backend.Mounter
asyncErrs map[docker.Named]error
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/pullexecutor/pullexecutor.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ type PullExecutorOptions struct {
AsyncPull bool
ImageServiceClient cri.ImageServiceClient
SecretStore secret.Store
Mounter *backend.SnapshotMounter
Mounter backend.Mounter
}

// PullOptions are the options for a single pull request
Expand All @@ -50,7 +50,7 @@ type PullExecutor struct {
mutex *sync.Mutex
asyncErrs map[docker.Named]error
secretStore secret.Store
mounter *backend.SnapshotMounter
mounter backend.Mounter
}

// NewPullExecutor initializes a new pull executor object
Expand Down
111 changes: 111 additions & 0 deletions pkg/test/utils/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package utils

import (
"context"
"fmt"
"time"

"github.com/containerd/containerd/reference/docker"
"github.com/warm-metal/container-image-csi-driver/pkg/backend"
"google.golang.org/grpc"
criapi "k8s.io/cri-api/pkg/apis/runtime/v1"
)

type MockImageServiceClient struct {
PulledImages map[string]bool
ImagePullTime time.Duration
}

type MockMounter struct {
ImageSvcClient MockImageServiceClient
Mounted map[string]bool
}

const hundredMB = 104857600

func (m *MockMounter) Mount(
ctx context.Context, volumeId string, target backend.MountTarget, image docker.Named, ro bool) (err error) {
m.Mounted[volumeId] = true
return nil
}

// Unmount unmounts a specific image
func (m *MockMounter) Unmount(ctx context.Context, volumeId string, target backend.MountTarget) error {
if m.Mounted[volumeId] {
delete(m.Mounted, volumeId)
return nil
}
return fmt.Errorf("image mount not found")
}

// ImageExists checks if the image already exists on the local machine
func (m *MockMounter) ImageExists(ctx context.Context, image docker.Named) bool {
return m.ImageSvcClient.PulledImages[image.Name()]
}

func (c *MockImageServiceClient) ListImages(ctx context.Context, in *criapi.ListImagesRequest, opts ...grpc.CallOption) (*criapi.ListImagesResponse, error) {
resp := new(criapi.ListImagesResponse)
resp.Images = []*criapi.Image{}

for k := range c.PulledImages {
resp.Images = append(resp.Images, &criapi.Image{
Id: k,
// 100MB
Size_: hundredMB,
Spec: &criapi.ImageSpec{
Image: k,
},
})
}

return resp, nil
}

func (c *MockImageServiceClient) ImageStatus(ctx context.Context, in *criapi.ImageStatusRequest, opts ...grpc.CallOption) (*criapi.ImageStatusResponse, error) {
resp := new(criapi.ImageStatusResponse)
resp.Image = &criapi.Image{
Id: in.Image.Image,
Size_: hundredMB,
Spec: &criapi.ImageSpec{
Image: in.Image.Image,
},
}
return resp, nil
}

func (c *MockImageServiceClient) PullImage(ctx context.Context, in *criapi.PullImageRequest, opts ...grpc.CallOption) (*criapi.PullImageResponse, error) {
resp := new(criapi.PullImageResponse)
resp.ImageRef = in.Image.Image
time.Sleep(c.ImagePullTime)

return resp, nil
}

func (c *MockImageServiceClient) RemoveImage(ctx context.Context, in *criapi.RemoveImageRequest, opts ...grpc.CallOption) (*criapi.RemoveImageResponse, error) {
resp := new(criapi.RemoveImageResponse)
delete(c.PulledImages, in.Image.Image)
return resp, nil
}

func (c *MockImageServiceClient) ImageFsInfo(ctx context.Context, in *criapi.ImageFsInfoRequest, opts ...grpc.CallOption) (*criapi.ImageFsInfoResponse, error) {
resp := new(criapi.ImageFsInfoResponse)
resp.ImageFilesystems = []*criapi.FilesystemUsage{}

for _ = range c.PulledImages {
resp.ImageFilesystems = append(resp.ImageFilesystems, &criapi.FilesystemUsage{
Timestamp: time.Now().Unix(),
FsId: &criapi.FilesystemIdentifier{
Mountpoint: "target",
},
UsedBytes: &criapi.UInt64Value{
Value: hundredMB,
},
InodesUsed: &criapi.UInt64Value{
// random value
Value: 10,
},
})
}

return resp, nil
}
File renamed without changes.
Loading

0 comments on commit 82d2e85

Please sign in to comment.