Merge pull request #391 from IBM-Cloud/feat/pagination-cache

feat: added methods for caching pagination urls into config

Author: Aerex

.travis.yml

@@ -1,7 +1,7 @@
 language: go
-dist: bionic
+dist: focal
 go:
-- '1.17.x'
+- '1.21.x'
 addons:
   apt:
     packages:

bluemix/configuration/config_helpers/helpers.go

@@ -21,23 +21,6 @@ func ConfigDir() string {
 	return defaultConfigDirOld()
 }
 
-// func MigrateFromOldConfig() error {
-// 	new := defaultConfigDirNew()
-// 	if file_helpers.FileExists(new) {
-// 		return nil
-// 	}
-
-// 	old := defaultConfigDirOld()
-// 	if !file_helpers.FileExists(old) {
-// 		return nil
-// 	}
-
-// 	if err := file_helpers.CopyDir(old, new); err != nil {
-// 		return err
-// 	}
-// 	return os.RemoveAll(old)
-// }
-
 func defaultConfigDirNew() string {
 	return filepath.Join(homeDir(), ".ibmcloud")
 }

bluemix/configuration/config_helpers/helpers_test.go

@@ -103,82 +103,3 @@ func TestConfigDir_IbmCloudConfigHomeSet_Exists(t *testing.T) {
 	os.Setenv("IBMCLOUD_CONFIG_HOME", userHome)
 	assert.Equal(userHome, ConfigDir())
 }
-
-// func TestMigrateFromOldConfig(t *testing.T) {
-// 	assert := assert.New(t)
-
-// 	err := prepareBluemixHome()
-// 	assert.NoError(err)
-// 	defer clearBluemixHome()
-
-// 	err = os.MkdirAll(oldConfigDir(), 0700)
-// 	assert.NoError(err)
-// 	oldConfigPath := filepath.Join(oldConfigDir(), "config.json")
-// 	err = ioutil.WriteFile(oldConfigPath, []byte("old"), 0600)
-// 	assert.NoError(err)
-
-// 	err = MigrateFromOldConfig()
-// 	assert.NoError(err)
-
-// 	newConfigPath := filepath.Join(newConfigDir(), "config.json")
-// 	assert.True(file_helpers.FileExists(newConfigPath))
-// 	content, err := ioutil.ReadFile(newConfigPath)
-// 	assert.NoError(err)
-// 	assert.Equal([]byte("old"), content, "should copy old config file")
-
-// 	assert.False(file_helpers.FileExists(oldConfigDir()), "old config dir should be deleted")
-// }
-
-// func TestMigrateFromOldConfig_NewConfigExist(t *testing.T) {
-// 	assert := assert.New(t)
-
-// 	err := prepareBluemixHome()
-// 	assert.NoError(err)
-// 	defer clearBluemixHome()
-
-// 	err = os.MkdirAll(oldConfigDir(), 0700)
-// 	assert.NoError(err)
-// 	oldConfigPath := filepath.Join(oldConfigDir(), "config.json")
-// 	err = ioutil.WriteFile(oldConfigPath, []byte("old"), 0600)
-// 	assert.NoError(err)
-
-// 	err = os.MkdirAll(newConfigDir(), 0700)
-// 	assert.NoError(err)
-// 	newConfigPath := filepath.Join(newConfigDir(), "config.json")
-// 	err = ioutil.WriteFile(newConfigPath, []byte("new"), 0600)
-// 	assert.NoError(err)
-
-// 	err = MigrateFromOldConfig()
-// 	assert.NoError(err)
-
-// 	content, err := ioutil.ReadFile(newConfigPath)
-// 	assert.NoError(err)
-// 	assert.Equal([]byte("new"), content, "should not copy old config file")
-// }
-
-// func TestMigrateFromOldConfig_OldConfigNotExist(t *testing.T) {
-// 	assert := assert.New(t)
-
-// 	err := prepareBluemixHome()
-// 	assert.NoError(err)
-// 	defer clearBluemixHome()
-
-// 	err = MigrateFromOldConfig()
-// 	assert.NoError(err)
-// }
-
-// func prepareBluemixHome() error {
-// 	temp, err := ioutil.TempDir("", "IBMCloudSDKConfigTest")
-// 	if err != nil {
-// 		return err
-// 	}
-// 	os.Setenv("BLUEMIX_HOME", temp)
-// 	return nil
-// }
-
-// func clearBluemixHome() {
-// 	if homeDir := os.Getenv("BLUEMIX_HOME"); homeDir != "" {
-// 		os.RemoveAll(homeDir)
-// 		os.Unsetenv("BLUEMIX_HOME")
-// 	}
-// }

bluemix/configuration/core_config/bx_config.go

@@ -2,6 +2,7 @@ package core_config
 
 import (
 	"encoding/json"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -63,6 +64,7 @@ type BXConfigData struct {
 	UpdateCheckInterval         time.Duration
 	UpdateRetryCheckInterval    time.Duration
 	UpdateNotificationInterval  time.Duration
+	PaginationURLs              []models.PaginationURL
 	raw                         raw
 }
 
@@ -752,9 +754,43 @@ func (c *bxConfig) ClearSession() {
 		c.data.IsLoggedInAsCRI = false
 		c.data.ResourceGroup = models.ResourceGroup{}
 		c.data.LoginAt = time.Time{}
+		c.data.PaginationURLs = []models.PaginationURL{}
 	})
 }
 
+func (c *bxConfig) SetPaginationURLs(paginationURLs []models.PaginationURL) {
+	c.write(func() {
+		c.data.PaginationURLs = paginationURLs
+	})
+}
+
+func (c *bxConfig) ClearPaginationURLs() {
+	c.write(func() {
+		c.data.PaginationURLs = []models.PaginationURL{}
+	})
+}
+
+func (c *bxConfig) AddPaginationURL(index int, url string) {
+	urls := c.PaginationURLs()
+
+	urls = append(urls, models.PaginationURL{
+		LastIndex: index,
+		NextURL:   url,
+	})
+
+	// sort by last index for easier retrieval
+	sort.Sort(models.ByLastIndex(urls))
+	c.SetPaginationURLs(urls)
+}
+
+func (c *bxConfig) PaginationURLs() (paginationURLs []models.PaginationURL) {
+	c.read(func() {
+		paginationURLs = c.data.PaginationURLs
+	})
+
+	return
+}
+
 func (c *bxConfig) UnsetAPI() {
 	c.write(func() {
 		c.data.APIEndpoint = ""

bluemix/configuration/core_config/bx_config_test.go

@@ -469,6 +469,64 @@ func TestLastUpdateSessionTime(t *testing.T) {
 
 }
 
+func TestPaginationURLs(t *testing.T) {
+	config := prepareConfigForCLI(`{}`, t)
+
+	// check initial state
+	paginationURLs := config.PaginationURLs()
+	assert.Empty(t, paginationURLs)
+
+	expected := []models.PaginationURL{
+		{
+			NextURL:   "https://api.example.com?token=dd3784000d9744acb2a23ad121a7bb4b",
+			LastIndex: 50,
+		},
+	}
+	config.SetPaginationURLs(expected)
+
+	paginationURLs = config.PaginationURLs()
+	assert.Equal(t, 1, len(paginationURLs))
+	assert.Equal(t, expected[0].LastIndex, paginationURLs[0].LastIndex)
+	assert.Equal(t, expected[0].NextURL, paginationURLs[0].NextURL)
+
+	t.Cleanup(cleanupConfigFiles)
+
+}
+
+func TestAddPaginationURL(t *testing.T) {
+	config := prepareConfigForCLI(`{}`, t)
+	assert := assert.New(t)
+	unsortedUrls := []models.PaginationURL{
+		{
+			NextURL:   "/v2/example.com/stuff?limit=200",
+			LastIndex: 200,
+		},
+		{
+			NextURL:   "/v2/example.com/stuff?limit=100",
+			LastIndex: 50,
+		},
+		{
+			NextURL:   "/v2/example.com/stuff?limit=100",
+			LastIndex: 100,
+		},
+	}
+
+	for _, p := range unsortedUrls {
+		config.AddPaginationURL(p.LastIndex, p.NextURL)
+	}
+
+	// expect url to be sorted in ascending order by LastIndex
+	sortedUrls := config.PaginationURLs()
+
+	assert.Equal(3, len(sortedUrls))
+	assert.Equal(sortedUrls[0].LastIndex, unsortedUrls[1].LastIndex)
+	assert.Equal(sortedUrls[0].NextURL, unsortedUrls[1].NextURL)
+	assert.Equal(sortedUrls[1].LastIndex, unsortedUrls[2].LastIndex)
+	assert.Equal(sortedUrls[1].NextURL, unsortedUrls[2].NextURL)
+	assert.Equal(sortedUrls[2].LastIndex, unsortedUrls[0].LastIndex)
+	assert.Equal(sortedUrls[2].NextURL, unsortedUrls[0].NextURL)
+}
+
 func checkUsageStats(enabled bool, timeStampExist bool, config core_config.Repository, t *testing.T) {
 	assert.Equal(t, config.UsageStatsEnabled(), enabled)
 	assert.Equal(t, config.UsageStatsEnabledLastUpdate().IsZero(), !timeStampExist)

bluemix/configuration/core_config/repository.go

@@ -125,6 +125,11 @@ type Repository interface {
 
 	SetLastSessionUpdateTime()
 	LastSessionUpdateTime() (session int64)
+
+	SetPaginationURLs(paginationURLs []models.PaginationURL)
+	ClearPaginationURLs()
+	AddPaginationURL(lastIndex int, nextURL string)
+	PaginationURLs() []models.PaginationURL
 }
 
 // Deprecated
@@ -368,6 +373,22 @@ func (c repository) SetLastSessionUpdateTime() {
 	c.bxConfig.SetLastSessionUpdateTime()
 }
 
+func (c repository) PaginationURLs() []models.PaginationURL {
+	return c.bxConfig.PaginationURLs()
+}
+
+func (c repository) AddPaginationURL(index int, url string) {
+	c.bxConfig.AddPaginationURL(index, url)
+}
+
+func (c repository) SetPaginationURLs(paginationURLs []models.PaginationURL) {
+	c.bxConfig.SetPaginationURLs(paginationURLs)
+}
+
+func (c repository) ClearPaginationURLs() {
+	c.bxConfig.ClearPaginationURLs()
+}
+
 func NewCoreConfig(errHandler func(error)) ReadWriter {
 	// config_helpers.MigrateFromOldConfig() // error ignored
 	return NewCoreConfigFromPath(config_helpers.CFConfigFilePath(), config_helpers.ConfigFilePath(), errHandler)

bluemix/models/pagination.go

@@ -0,0 +1,15 @@
+package models
+
+// ByLastIndex sorts PaginationURLs by LastIndex
+type ByLastIndex []PaginationURL
+
+func (a ByLastIndex) Len() int { return len(a) }
+
+func (a ByLastIndex) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a ByLastIndex) Less(i, j int) bool { return a[i].LastIndex < a[j].LastIndex }
+
+type PaginationURL struct {
+	LastIndex int
+	NextURL   string
+}

common/rest/request.go

@@ -51,12 +51,14 @@ import (
 	"net/textproto"
 	"net/url"
 	"strings"
+
+	"github.com/IBM-Cloud/ibm-cloud-cli-sdk/bluemix/models"
 )
 
 const (
-	contentType               = "Content-Type"
-	jsonContentType           = "application/json"
-	formUrlEncodedContentType = "application/x-www-form-urlencoded"
+	ContentType               = "Content-Type"
+	JSONContentType           = "application/json"
+	FormUrlEncodedContentType = "application/x-www-form-urlencoded"
 )
 
 // File represents a file upload in HTTP request
@@ -138,6 +140,17 @@ func OptionsRequest(rawUrl string) *Request {
 	return NewRequest(rawUrl).Method("OPTIONS")
 }
 
+// CachedPaginationNextURL will attempt to return a cached URL with last index
+// if there exists a URL with a last index smaller than the offset provided
+func CachedPaginationNextURL(paginationURLs []models.PaginationURL, offset int) models.PaginationURL {
+	for _, p := range paginationURLs {
+		if p.LastIndex < offset {
+			return p
+		}
+	}
+	return models.PaginationURL{}
+}
+
 // Add adds the key, value pair to the request header. It appends to any
 // existing values associated with key.
 func (r *Request) Add(key string, value string) *Request {
@@ -272,7 +285,7 @@ func (r *Request) buildFormMultipart() (io.Reader, error) {
 		}
 	}
 
-	r.header.Set(contentType, w.FormDataContentType())
+	r.header.Set(ContentType, w.FormDataContentType())
 	return b, nil
 }
 
@@ -296,7 +309,7 @@ func escapeQuotes(s string) string {
 }
 
 func (r *Request) buildFormFields() (io.Reader, error) {
-	r.header.Set(contentType, formUrlEncodedContentType)
+	r.header.Set(ContentType, FormUrlEncodedContentType)
 	return strings.NewReader(r.formParams.Encode()), nil
 }