External Secret Storage Implementation (caraml-dev#85)

* Rename storage to repository

* Run gofmt

* Run gofmt

* Add secret storage repository

* Fix mount_type typo

* Remove kv_version in vault config

* Use ememory storage for keto local testing

* Use ememory storage for keto local testing

* Fix lint

* Fix lint

* Update config package and its tests to support default secret storage

* Add pathPrefix config

* Add basic implementation of vault secret storage

* Add authentication to vault

* Update secret repository interfaces and remove encryption

* Update secret storage repository interfaces

* Fix incorrect teardown in SecretStorageTestSuite

* Add Service Account Email field to secret storage for iam auth

* Update secret storage client registry

* Update secret service implementation and tests

* Remove encryption key from config

* Add default secret storage initialization

* Update secrets_api.go

* Fix secrets_api_test.go

* Add unit test for FindByID

* Remove EncryptionKey from tests

* Add reference to the corresponding project in secret model

* Add end to end test

* Update dependency

* Update CI

* Add map.go

* Add secret_storage_repository.go mock

* Fix lint

* Add mocks

* Fix lint

* Remove it-test-api-ci

* Add SetAll and DeleteAll to secretstorage.Client interface

* Remove old vault code

* Add secret storage API

* Fromat projects_api.go

* Update entity model

* Properly delete secret key

* Allow user to specify secret storage when creating or updating secret

* Update local development configuration

* Update README.md

* Add code comments

* Update swagger and client code

* Update swagger and client code

* Revert "Update swagger and client code"

This reverts commit f27ad2c.

* Revert "Update swagger and client code"

This reverts commit 63beda4.

* Update swagger

* Update generated client code

* Change the ordering in db migration

* Change basePath in example

* Update MLP_API_BASEPATH

* Fix incorrect ordering

* Use InternalServerError

* Fix comparison

* Handle global secret storage migration

* Drop types in db migration downgrade

* Rename JoinMaps to JoinStringMaps

* Use EuiFieldPassword as secret data input

* Add godocs

* Remove redundant check

* Refactor

* Refactor secretRepository

* Disallow migrating to internal secret storage

* Properly handle default secret storage selection

* Reformat

* Fix lint ui

---------

Co-authored-by: Pradithya Aria <[email protected]>

Author: pradithya

.env.development

@@ -1,4 +1,5 @@
 DATABASE_HOST=localhost
 DATABASE_USER=mlp
 DATABASE_PASSWORD=mlp
-DATABASE_NAME=mlp
+DATABASE_NAME=mlp
+REACT_APP_OAUTH_CLIENT_ID="[OAUTH_CLIENT_ID]"

.github/workflows/ci-pipeline.yml

@@ -99,26 +99,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       GOPATH: ${{ github.workspace }}/.go
-    services:
-      postgres:
-        image: postgres:14.5
-        env:
-          POSTGRES_DB: mlp
-          POSTGRES_USER: mlp
-          POSTGRES_PASSWORD: mlp
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-      keto:
-        image: oryd/keto:v0.4
-        env:
-          DSN: memory
-        ports:
-          - 4466:4466
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
@@ -137,13 +117,7 @@ jobs:
         run: make lint-api
 
       - name: Run Integration Test
-        env:
-          DATABASE_HOST: localhost
-          DATABASE_NAME: mlp
-          DATABASE_USER: mlp
-          DATABASE_PASSWORD: mlp
-          KETO_URL: http://localhost:4466
-        run: make it-test-api-ci
+        run: make it-test-api
 
   e2e-test:
     runs-on: ubuntu-latest
@@ -249,7 +223,7 @@ jobs:
 
       - name: Run E2E tests
         env:
-          MLP_API_BASEPATH: http://mlp.127.0.0.1.nip.io/v1
+          MLP_API_BASEPATH: http://mlp.127.0.0.1.nip.io
         shell: bash
         run: |
           for example in api/client/examples/*; do

Makefile

@@ -46,7 +46,8 @@ lint-ui:
 .PHONY: fmt
 fmt:
 	@echo "Formatting code..."
-	gofmt -s -w ${SRC_ROOT}
+	@goimports -w -local github.com/caraml-dev/mlp $(shell find . -type f -name '*.go' -not -path "**/vendor/*")
+	@gofmt -s -w .
 
 .PHONY: lint-api
 lint-api: setup
@@ -65,19 +66,13 @@ test-api: init-dep-api
 	@cd ${API_PATH} && go test -v -race -cover -coverprofile cover.out ${API_ALL_PACKAGES}
 	@cd ${API_PATH} && go tool cover -func cover.out
 
-.PHONY: it-test-api-local
-it-test-api-local: local-db start-keto
+.PHONY: it-test-api
+it-test-api: local-db start-keto start-vault
 	@echo "> API integration testing locally..."
 	@cd ${API_PATH} && go test -race -short -cover -coverprofile cover.out -tags integration ${API_ALL_PACKAGES}
 	@cd ${API_PATH} && go tool cover -func cover.out
 	@make stop-docker
 
-.PHONY: it-test-api-ci
-it-test-api-ci:
-	@echo "> API integration testing ..."
-	@cd ${API_PATH} && go test -race -short -cover -coverprofile cover.out -tags integration ${API_ALL_PACKAGES}
-	@cd ${API_PATH} && go tool cover -func cover.out
-
 # ============================================================
 # Building recipes
 # ============================================================
@@ -115,9 +110,9 @@ build-image: version
 # Run recipes
 # ============================================================
 .PHONY: run
-run: build-api local-db
+run: local-env
 	@echo "> Running application ..."
-	@./bin/${BIN_NAME} --config config-dev.yaml
+	@go run api/cmd/main.go --config config-dev.yaml
 
 .PHONY: start-ui
 start-ui:
@@ -142,8 +137,15 @@ clean-bin:
 
 generate-client:
 	@echo "> Generating API client ..."
-	@swagger-codegen generate -i static/swagger.yaml -l go -o client -DpackageName=client
-	@goimports -l -w client
+	@docker run --rm -v $(shell pwd):/local swaggerapi/swagger-codegen-cli:2.4.14 generate \
+         -i /local/api/static/swagger.yaml \
+         -l go \
+         -o /local/api/client \
+         -DpackageName=client
+	$(MAKE) fmt
+
+.PHONY: local-env
+local-env: local-db start-keto start-vault
 
 .PHONY: local-db
 local-db:
@@ -155,6 +157,11 @@ start-keto:
 	@echo "> Starting up keto server ..."
 	@docker-compose up -d keto
 
+.PHONY: start-vault
+start-vault:
+	@echo "> Starting up vault server ..."
+	@docker-compose up -d vault
+
 .PHONY: stop-docker
 stop-docker:
 	@echo "> Stopping Docker compose ..."

README.md

@@ -22,7 +22,7 @@ The MLP Server provides REST API used across MLP Products. It exposes a shared c
 
 1. [Google Oauth credential](https://developers.google.com/identity/protocols/oauth2/javascript-implicit-flow)
 
-    MLP uses Google Sign-in to authenticate the user to access the API and UI. After you get the client ID, specify it into `OAUTH_CLIENT_ID` in `.env.development` file.
+    MLP uses Google Sign-in to authenticate the user to access the API and UI. After you get the client ID, specify it into `REACT_APP_OAUTH_CLIENT_ID` in `.env.development` file.
 
 ### From Docker Compose
 
@@ -36,14 +36,19 @@ MLP will now be reachable at <http://localhost:8080>.
 
 ### From source
 
-To build and run MLP from the source code, you need to have [Go](https://golang.org/doc/install), [Node.js](https://nodejs.org/), and [Yarn](https://yarnpkg.com/) installed. You will also need a running Postgresql database. MLP uses Docker to make the task of setting up databases a little easier. You can run `make local-db` to starting up a Postgres Docker container.
+To build and run MLP from the source code, you need to have [Go](https://golang.org/doc/install), [Node.js](https://nodejs.org/), and [Yarn](https://yarnpkg.com/) installed. 
+You will also need a running Postgresql database, Keto, and Vault servers. 
+MLP uses Docker to make the task of setting up the dependencies a little easier. You can run `make local-env` to starting up all those dependencies.
 
 ```shell script
-mkdir -p $GOPATH/src/github.com/caraml-dev
-cd $GOPATH/src/github.com/caraml-dev
-git clone [email protected]/caraml-dev/mlp.git mlp
-cd mlp
-make local-db
+make local-env
+make run
+```
+
+OR
+
+```shell script
+# `make` will execute `make local-env` and `make run`
 make
 ```
 

api/api/api_test.go

@@ -0,0 +1,162 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/caraml-dev/mlp/api/config"
+	"github.com/caraml-dev/mlp/api/it/database"
+	"github.com/caraml-dev/mlp/api/models"
+)
+
+type APITestSuite struct {
+	suite.Suite
+
+	// handler under test
+	route http.Handler
+
+	// db cleanup function
+	cleanupFn func()
+
+	// main project to be used for testing
+	// most test entities (secret and secret_storage) are created under this
+	mainProject *models.Project
+	// other project to be used for testing
+	otherProject *models.Project
+
+	// interal secret storage reference
+	internalSecretStorage *models.SecretStorage
+	// default secret storage, the test configures it to vault
+	defaultSecretStorage *models.SecretStorage
+	// secret storage owned by the project
+	projectSecretStorage *models.SecretStorage
+	// existing secrets owned by the project
+	// in total there are 5 secrets
+	// the first 2 secrets are stored in internalSecretStorage
+	// the rest of secrets are stored in defaultSecretStorage
+	existingSecrets []*models.Secret
+}
+
+func (s *APITestSuite) SetupTest() {
+	db, cleanupFn, err := database.CreateTestDatabase()
+	s.Require().NoError(err, "Failed to connect to test database")
+	s.cleanupFn = cleanupFn
+
+	// create app context which also specify vault as the default secret storage
+	appCtx, err := NewAppContext(db, &config.Config{
+		Port: 0,
+		Authorization: &config.AuthorizationConfig{
+			Enabled: false,
+		},
+		Mlflow: &config.MlflowConfig{
+			TrackingURL: "http://mlflow:5000",
+		},
+		DefaultSecretStorage: &config.SecretStorage{
+			Name: "vault",
+			Type: string(models.VaultSecretStorageType),
+			Config: models.SecretStorageConfig{
+				VaultConfig: &models.VaultConfig{
+					URL:        "http://localhost:8200",
+					Role:       "my-role",
+					MountPath:  "secret",
+					PathPrefix: fmt.Sprintf("api-test/%d/{{ .Project }}", time.Now().Unix()),
+					AuthMethod: models.TokenAuthMethod,
+					Token:      "root",
+				},
+			},
+		},
+	})
+	s.Require().NoError(err, "Failed to create app context")
+
+	// create project and otherProject
+	s.mainProject, err = appCtx.ProjectsService.CreateProject(&models.Project{
+		Name: "test-project",
+	})
+	s.Require().NoError(err, "Failed to create project")
+
+	s.otherProject, err = appCtx.ProjectsService.CreateProject(&models.Project{
+		Name: "other-project",
+	})
+	s.Require().NoError(err, "Failed to create other project")
+
+	// get reference to internal secret storage
+	// internal secret storage always have ID 1
+	s.internalSecretStorage, err = appCtx.SecretStorageService.FindByID(1)
+	s.Require().NoError(err, "Failed to find internal secret storage")
+
+	// get reference to default secret storage
+	s.defaultSecretStorage = appCtx.DefaultSecretStorage
+
+	// create a secret storage owned by the project
+	s.projectSecretStorage, err = appCtx.SecretStorageService.Create(&models.SecretStorage{
+		Name:      "project-secret-storage",
+		Type:      models.VaultSecretStorageType,
+		Scope:     models.ProjectSecretStorageScope,
+		ProjectID: &s.mainProject.ID,
+		Config: models.SecretStorageConfig{
+			VaultConfig: &models.VaultConfig{
+				URL:        "http://localhost:8200",
+				Role:       "my-role",
+				MountPath:  "secret",
+				PathPrefix: fmt.Sprintf("project-secret-test/%d", time.Now().Unix()),
+				AuthMethod: models.TokenAuthMethod,
+				Token:      "root",
+			},
+		},
+	})
+	s.Require().NoError(err, "Failed to create project secret storage")
+
+	//
+	s.existingSecrets = make([]*models.Secret, 0)
+	for i := 0; i < 5; i++ {
+		secretStorageID := s.defaultSecretStorage.ID
+		// the first 2 secrets will be created in internal secret storage
+		if i < 2 {
+			secretStorageID = s.internalSecretStorage.ID
+		}
+
+		secret, err := appCtx.SecretService.Create(&models.Secret{
+			ProjectID:       s.mainProject.ID,
+			SecretStorageID: &secretStorageID,
+			Name:            fmt.Sprintf("secret-%d", i),
+			Data:            fmt.Sprintf("secret-data-%d", i),
+		})
+		s.Require().NoError(err, "Failed to create secret")
+		s.existingSecrets = append(s.existingSecrets, secret)
+	}
+
+	// initialize controllers and http handler / route
+
+	controllers := []Controller{
+		&ApplicationsController{AppContext: appCtx},
+		&ProjectsController{AppContext: appCtx},
+		&SecretsController{AppContext: appCtx},
+		&SecretStoragesController{AppContext: appCtx},
+	}
+
+	r := NewRouter(appCtx, controllers)
+
+	route := mux.NewRouter()
+	route.PathPrefix(basePath).Handler(
+		http.StripPrefix(
+			strings.TrimSuffix(basePath, "/"),
+			r,
+		),
+	)
+
+	s.route = route
+}
+
+func (s *APITestSuite) TearDownTest() {
+	s.cleanupFn()
+}
+
+func TestAPI(t *testing.T) {
+	suite.Run(t, new(APITestSuite))
+}