diff --git a/Makefile b/Makefile index cd3009bcb..88bb786c5 100755 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ DEFAULT_DOCKER_VERSION := 1.12.6 V2PLUGIN_DOCKER_VERSION := 1.13.1 SHELL := /bin/bash # TODO: contivmodel should be removed once its code passes golint and misspell -EXCLUDE_DIRS := bin docs Godeps scripts vagrant vendor install contivmodel +EXCLUDE_DIRS := bin docs Godeps scripts vagrant vendor install contivmodel venv PKG_DIRS := $(filter-out $(EXCLUDE_DIRS),$(subst /,,$(sort $(dir $(wildcard */))))) TO_BUILD := ./netplugin/ ./netmaster/ ./netctl/netctl/ ./mgmtfn/k8splugin/contivk8s/ ./mgmtfn/mesosplugin/netcontiv/ HOST_GOBIN := `if [ -n "$$(go env GOBIN)" ]; then go env GOBIN; else dirname $$(which go); fi` @@ -131,9 +131,18 @@ update: start: CONTIV_DOCKER_VERSION="$${CONTIV_DOCKER_VERSION:-$(DEFAULT_DOCKER_VERSION)}" CONTIV_NODE_OS=${CONTIV_NODE_OS} vagrant up +# =================================================================== +# kubernetes helper targets + +k8s-check-python-deps: + @command -v pip >/dev/null \ + || echo No pip available, make sure netaddr and parse packages are installed + @command -v pip >/dev/null && pip show -q netaddr && pip show -q parse \ + || (echo parse and netaddr python packages are required; exit 1) + # =================================================================== # kubernetes cluster bringup/cleanup targets -k8s-cluster: +k8s-cluster: k8s-check-python-deps cd vagrant/k8s/ && CONTIV_K8S_USE_KUBEADM=1 ./setup_cluster.sh k8s-l3-cluster: @@ -147,11 +156,24 @@ k8s-l3-destroy: # =================================================================== # kubernetes test targets -k8s-test: k8s-cluster - cd vagrant/k8s/ && vagrant ssh k8master -c 'bash -lc "cd /opt/gopath/src/github.com/contiv/netplugin && make run-build"' - cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python && PYTHONIOENCODING=utf-8 ./createcfg.py -scheduler 'k8s' -binpath contiv/bin -install_mode 'kubeadm' - CONTIV_K8S_USE_KUBEADM=1 CONTIV_NODES=3 go test -v -timeout 540m ./test/systemtests -check.v -check.abort -check.f $(K8S_SYSTEM_TESTS_TO_RUN) - cd vagrant/k8s && vagrant destroy -f +k8s-build: FIRST_MASTER_HOSTNAME:=k8master +k8s-build: first_master_make_targets:="compile archive" +k8s-build: export VAGRANT_CWD=$(PWD)/vagrant/k8s +k8s-build: TO_BUILD := netplugin netmaster mgmtfn test/systemtests +k8s-build: make-on-first-master-dep + +# for k8s-test, run-build will be run on the node instead of local +k8s-test: FIRST_MASTER_HOSTNAME:=k8master +k8s-test: first_master_make_targets:=run-build +k8s-test: export VAGRANT_CWD:=$(PWD)/vagrant/k8s +k8s-test: export CONTIV_K8S_USE_KUBEADM:=1 CONTIV_NODES:=3 PYTHONIOENCODING:=utf-8 +k8s-test: k8s-cluster make-on-first-master-dep +k8s-test-now: + cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python \ + && ./createcfg.py -scheduler 'k8s' -binpath contiv/bin -install_mode 'kubeadm' + go test -v -timeout 540m ./test/systemtests -check.v -check.abort -check.f \ + $(K8S_SYSTEM_TESTS_TO_RUN) + #cd vagrant/k8s && vagrant destroy -f k8s-l3-test: k8s-l3-cluster cd vagrant/k8s/ && vagrant ssh k8master -c 'bash -lc "cd /opt/gopath/src/github.com/contiv/netplugin && make run-build"' @@ -204,7 +226,7 @@ integ-test: stop clean start ssh-build ubuntu-tests: CONTIV_NODE_OS=ubuntu make clean build unit-test system-test stop -system-test:start +system-test: start @echo "system-test: running the following system tests:" $(SYSTEM_TESTS_TO_RUN) cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python && PYTHONIOENCODING=utf-8 ./createcfg.py go test -v -timeout 480m ./test/systemtests -check.v -check.abort -check.f $(SYSTEM_TESTS_TO_RUN) @@ -293,7 +315,7 @@ host-plugin-create: host-plugin-update: host-plugin-remove unarchive host-plugin-create # same behavior as host-plugin-update but runs locally with docker 1.13+ plugin-update: tar - $(call make-on-node1, host-plugin-update) + $(call make-on-first-master, host-plugin-update) # cleanup all containers, recreate and start the v2plugin on all hosts # uses the latest compiled binaries @@ -315,21 +337,23 @@ host-pluginfs-unpack: --exclude=etc/terminfo/v/vt220 # Runs make targets on the first netplugin vagrant node -# this is used as a macro like $(call make-on-node1, compile checks) -make-on-node1 = vagrant ssh netplugin-node1 -c '\ +# this is used as a macro like $(call make-on-first-master, compile checks) + +FIRST_MASTER_HOSTNAME ?= netplugin-node1 +make-on-first-master = vagrant ssh $(FIRST_MASTER_HOSTNAME) -c '\ bash -lc "source /etc/profile.d/envvar.sh \ && cd /opt/gopath/src/github.com/contiv/netplugin && make $(1)"' -# Calls macro make-on-node1 but can be used as a dependecy by setting -# the variable "node1-make-targets" -make-on-node1-dep: - $(call make-on-node1, $(node1-make-targets)) +# Calls macro make-on-first-master but can be used as a dependecy by setting +# the variable "first_master_make_targets" +make-on-first-master-dep: + $(call make-on-first-master, $(first_master_make_targets)) # assumes the v2plugin archive is available, installs the v2plugin and resets # everything on the vm to clean state v2plugin-install: @echo Installing v2plugin - $(call make-on-node1, install-shell-completion host-pluginfs-unpack \ + $(call make-on-first-master, install-shell-completion host-pluginfs-unpack \ host-plugin-restart host-swarm-restart) # Just like demo-v2plugin except builds are done locally and cached @@ -341,8 +365,8 @@ demo-v2plugin-from-local: tar host-pluginfs-create start v2plugin-install # then creates and enables v2plugin demo-v2plugin: export CONTIV_DOCKER_VERSION ?= $(V2PLUGIN_DOCKER_VERSION) demo-v2plugin: export CONTIV_DOCKER_SWARM := swarm_mode -demo-v2plugin: node1-make-targets := host-pluginfs-create -demo-v2plugin: ssh-build make-on-node1-dep v2plugin-install +demo-v2plugin: first_master_make_targets := host-pluginfs-create +demo-v2plugin: ssh-build make-on-first-master-dep v2plugin-install # release a v2 plugin from the VM host-plugin-release: tar host-pluginfs-create host-pluginfs-unpack host-plugin-create diff --git a/mgmtfn/k8splugin/driver.go b/mgmtfn/k8splugin/driver.go index c79a047c7..c8d76ea39 100644 --- a/mgmtfn/k8splugin/driver.go +++ b/mgmtfn/k8splugin/driver.go @@ -404,7 +404,7 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int ep, err := createEP(epReq) if err != nil { log.Errorf("Error creating ep. Err: %v", err) - setErrorResp(&resp, "Error creating EP", err) + setErrorResp(&resp, "Error creating EP '"+epReq.Name+"'", err) return resp, err } diff --git a/scripts/netContain/scripts/contivNet.sh b/scripts/netContain/scripts/contivNet.sh index fdcf881fd..98675385e 100755 --- a/scripts/netContain/scripts/contivNet.sh +++ b/scripts/netContain/scripts/contivNet.sh @@ -131,7 +131,6 @@ if [ $netmaster == true ]; then sleep 5 done elif [ $netplugin == true ]; then - echo "Starting netplugin" if [[ "$cluster_store" =~ ^etcd://.+ ]]; then store_arg="--etcd-endpoints $(echo $cluster_store | sed s/etcd/http/)" elif [[ "$cluster_store" =~ ^consul://.+ ]]; then @@ -146,7 +145,10 @@ elif [ $netplugin == true ]; then if [ "$vlan_if" != "" ]; then vlan_if_param="--vlan-if" fi + echo "Starting netplugin $(date)" + set -x /contiv/bin/netplugin $debug $store_arg $vtep_ip_param $vtep_ip $vlan_if_param $vlan_if --plugin-mode $plugin || true + set +x echo "CRITICAL : Netplugin has exited. Trying to respawn in 5s" fi sleep 5 diff --git a/state/etcdstatedriver.go b/state/etcdstatedriver.go index 8012d046f..d74058f8d 100644 --- a/state/etcdstatedriver.go +++ b/state/etcdstatedriver.go @@ -127,7 +127,7 @@ func (d *EtcdStateDriver) Read(key string) ([]byte, error) { } if client.IsKeyNotFound(err) { - return []byte{}, core.Errorf("key not found") + return []byte{}, core.Errorf("key '%s' not found", key) } if err.Error() == client.ErrClusterUnavailable.Error() { diff --git a/test/systemtests/How-to-Run.md b/test/systemtests/How-to-Run.md index fc9d3ca7a..cbf9b8f51 100644 --- a/test/systemtests/How-to-Run.md +++ b/test/systemtests/How-to-Run.md @@ -1,3 +1,5 @@ +# A guide to running netplugin systemtests on Vagrant and Baremetal platforms: + Current framework can run system-tests for ``` @@ -9,7 +11,6 @@ Baremetal- Swarm -- ACI Swarm -- Non-ACI ``` -A guide to running netplugin systemtests on Vagrant and Baremetal platforms: Customize the example JSON file `netplugin/systemtests/cfg.json.example` according to your environment and rename it to `netplugin/systemtests/cfg.json`. A typical file for vagrant with swarm looks like: ``` @@ -46,7 +47,8 @@ Customize the example JSON file `netplugin/systemtests/cfg.json.example` accordi ] ``` -Testing with Vagrant: + +### Testing with Vagrant: * Make a suitable JSON file on your local machine (inside the systemtests directory). * From the netplugin directory of your machine (outside the vagrant nodes), run: @@ -54,7 +56,24 @@ Testing with Vagrant: ``` make system-test ``` -Testing with Baremetal with Swarm: + + +### Testing k8s with Vagrant: + +To run all the k8s system tests: + +``` +make k8s-test +``` + +To work on a single system test, such as TestNetworkAddDeleteNoGatewayVLAN: + +``` + make K8S_SYSTEM_TESTS_TO_RUN=TestNetworkAddDeleteNoGatewayVLAN start k8s-test +``` + + +### Testing with Baremetal with Swarm: For ACI testing , We need to have connectivity to APIC and ACI Fabric Switches from Baremetal VMs and Hosts. * You need to complete Pre-requisites, Step 1, Step 2, Step3 metioned here : https://github.com/contiv/demo/tree/master/net @@ -84,7 +103,9 @@ godep go test -v -timeout 240m ./systemtests -check.v -check.f "TestACI" This will run all the test functions which have the string TestACI ``` -Troubleshooting + + +### Troubleshooting * First delete all netmaster, netctl, netplugin, contivk8s binaries from $GOBIN directory from all Nodes in the Cluster * You can perform following steps to clear etcd states diff --git a/test/systemtests/kubeadm_test.go b/test/systemtests/kubeadm_test.go index 9cb26740c..bea572752 100755 --- a/test/systemtests/kubeadm_test.go +++ b/test/systemtests/kubeadm_test.go @@ -9,9 +9,10 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "os" "time" + + "github.com/Sirupsen/logrus" ) type kubePod struct { @@ -618,7 +619,9 @@ func (k *kubePod) runCommandUntilNoNetmasterError() error { } processCheckCmd := `kubectl -n kube-system exec ` + podName + ` -- pgrep netmaster` - return k8sMaster.runCommandUntilNoError(processCheckCmd) + err = k8sMaster.runCommandUntilNoError(processCheckCmd) + logrus.Infof("netmaster status check complete on: %s", k.node.Name()) + return err } func (k *kubePod) runCommandUntilNoNetpluginError() error { @@ -633,7 +636,8 @@ func (k *kubePod) runCommandUntilNoNetpluginError() error { } processCheckCmd := `kubectl -n kube-system exec ` + podName + ` -- pgrep netplugin` - return k8sMaster.runCommandUntilNoError(processCheckCmd) + err = k8sMaster.runCommandUntilNoError(processCheckCmd) + return err } func (k *kubePod) rotateNetmasterLog() error { diff --git a/test/systemtests/network_test.go b/test/systemtests/network_test.go index ac071b9d9..999fef805 100755 --- a/test/systemtests/network_test.go +++ b/test/systemtests/network_test.go @@ -259,9 +259,13 @@ func (s *systemtestSuite) testNetworkAddDeleteNoGateway(c *C, encap string) { for _, name := range netNames { var err error - // There seem to be a docker bug in creating external connectivity if we run - // containers in parallel. So, running it serially for this test - containers[name], err = s.runContainersSerial(numContainer, false, name, "", nil) + if s.basicInfo.Scheduler == "k8s" { + containers[name], err = s.runContainers(numContainer, false, name, "", nil, nil) + } else { + // There seem to be a docker bug in creating external connectivity if we run + // containers in parallel. So, running it serially for this test + containers[name], err = s.runContainersSerial(numContainer, false, name, "", nil) + } c.Assert(err, IsNil) } diff --git a/test/systemtests/util_test.go b/test/systemtests/util_test.go index 59926dbdd..ecba9e677 100755 --- a/test/systemtests/util_test.go +++ b/test/systemtests/util_test.go @@ -186,7 +186,7 @@ func (s *systemtestSuite) runContainers(num int, withService bool, networkName s for i := 0; i < num; i++ { go func(i int) { nodeNum := i % len(s.nodes) - var name string + var name, cname string mutex.Lock() if len(names) > 0 { @@ -197,6 +197,9 @@ func (s *systemtestSuite) runContainers(num int, withService bool, networkName s if name == "" { name = fmt.Sprintf("%s-srv%d-%d", strings.Replace(networkName, "/", "-", -1), i, nodeNum) + cname = name + } else { + cname = fmt.Sprintf("%s-%d", name, i) } var serviceName string @@ -205,7 +208,6 @@ func (s *systemtestSuite) runContainers(num int, withService bool, networkName s serviceName = name } - cname := fmt.Sprintf("%s-%d", name, i) spec := containerSpec{ imageName: "contiv/alpine", networkName: networkName, @@ -1249,8 +1251,8 @@ func (s *systemtestSuite) SetUpSuiteVagrant(c *C) { c.Assert(s.vagrant.Setup(false, []string{"CONTIV_L3=1 VAGRANT_CWD=" + topDir + "/src/github.com/contiv/netplugin/vagrant/k8s/"}, contivNodes), IsNil) // Sleep to give enough time for the netplugin pods to come up - logrus.Infof("Sleeping for 1 minute for pods to come up") - time.Sleep(time.Minute) + //logrus.Infof("Sleeping for 1 minute for pods to come up") + // time.Sleep(time.Minute) case swarmScheduler: c.Assert(s.vagrant.Setup(false, append([]string{"CONTIV_NODES=3 CONTIV_L3=1"}, s.basicInfo.SwarmEnv), contivNodes+contivL3Nodes), IsNil) @@ -1276,8 +1278,8 @@ func (s *systemtestSuite) SetUpSuiteVagrant(c *C) { c.Assert(s.vagrant.Setup(false, []string{"VAGRANT_CWD=" + topDir + "/src/github.com/contiv/netplugin/vagrant/k8s/"}, contivNodes), IsNil) // Sleep to give enough time for the netplugin pods to come up - logrus.Infof("Sleeping for 1 minute for pods to come up") - time.Sleep(time.Minute) + //logrus.Infof("Sleeping for 1 minute for pods to come up") + //time.Sleep(time.Minute) case swarmScheduler: c.Assert(s.vagrant.Setup(false, append([]string{}, s.basicInfo.SwarmEnv), contivNodes), IsNil)