Smugmug API Mock

CONTRIBUTING.md

@@ -26,3 +26,9 @@ Contributing is more than welcome :smile:
 I'm dropping here the link to
 [package documentation](https://pkg.go.dev/github.com/tommyblue/smugmug-backup?tab=doc), in
 case you need that.
+
+### Mocking the API server
+
+This package contains a mock server that can be executed with `go run ./cmd/api_mock` so that you
+don't need to have a real SmugMug account. The server will listen on http://127.0.0.1:3000  
+To use it run the smugmug-backup binary with the `-mock` flag.

api.go

@@ -160,7 +160,11 @@ func (w *Worker) saveVideo(image albumImage, folder string) error {
 
 func (w *Worker) setChTime(image albumImage, dest string) error {
 	// Try first with the date in the image, to avoid making an additional call
-	created, err := time.Parse(time.RFC3339, image.DateTimeOriginal)
+	dt := image.DateTimeOriginal
+	if dt == "" {
+		dt = image.DateTimeUploaded
+	}
+	created, err := time.Parse(time.RFC3339, dt)
 	if err != nil || created.IsZero() {
 		created = w.imageTimestamp(image)
 	}

cmd/api_mock/albumimages_1.json

@@ -0,0 +1,29 @@
+{
+  "Response": {
+    "Uri": "/api/v2/album/aX3TYu!images",
+    "AlbumImage": [
+      {
+
+        "ArchivedMD5": "258597waaf35aed642z08c351426b5e4",
+        "ArchivedSize": 476277,
+        "ArchivedUri": "http://localhost:3000/photos/photo.jpg",
+        "Caption": "",
+        "DateTimeUploaded": "2024-01-05T20:04:39+00:00",
+        "FileName": "somePhoto.jpg",
+        "ImageKey": "iTsf4K3",
+        "IsVideo": false,
+        "Keywords": "",
+        "Processing": false,
+        "UploadKey": "11112222333",
+        "Uris": {
+          "ImageMetadata": {
+            "Uri": "/api/v2/image/iTsf4K3-0!metadata"
+          }
+        }
+      }
+    ],
+    "Pages": {
+      "NextPage": ""
+    }
+  }
+}

cmd/api_mock/authuser.json

@@ -0,0 +1,7 @@
+{
+  "Response": {
+    "User": {
+      "NickName": "myUser"
+    }
+  }
+}

cmd/api_mock/main.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	_ "embed"
+	"encoding/json"
+	"net/http"
+
+	chi "github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
+	log "github.com/sirupsen/logrus"
+)
+
+//go:embed authuser.json
+var authuser []byte
+
+//go:embed user.json
+var user []byte
+
+//go:embed useralbums_1.json
+var useralbums_1 []byte
+
+//go:embed albumimages_1.json
+var albumimages_1 []byte
+
+//go:embed photo.jpg
+var photo []byte
+
+func parseJson(content []byte) any {
+	var dest interface{}
+	if err := json.Unmarshal(content, &dest); err != nil {
+		log.Fatal(err)
+	}
+
+	return dest
+}
+
+func main() {
+	r := chi.NewRouter()
+	r.Use(middleware.Logger)
+
+	api := chi.NewRouter()
+
+	api.Route("/v2", func(r chi.Router) {
+		// User details. Get the first page of the user albums
+		r.Get("/user/{username}", func(w http.ResponseWriter, r *http.Request) {
+			//username := chi.URLParam(r, "username")
+			//resp.Response.User.Uris.UserAlbums.URI = fmt.Sprintf("/api/v2/user/%s!albums", username)
+			responseOk(w, parseJson(user))
+		})
+
+		r.Get("/album/{albumId}!images", func(w http.ResponseWriter, r *http.Request) {
+			//albumId := chi.URLParam(r, "albumId")
+			responseOk(w, parseJson(albumimages_1))
+		})
+
+		r.Get("/user/{username}!albums", func(w http.ResponseWriter, r *http.Request) {
+			responseOk(w, parseJson(useralbums_1))
+		})
+	})
+
+	api.Get("/v2!authuser", func(w http.ResponseWriter, r *http.Request) {
+		responseOk(w, parseJson(authuser))
+	})
+
+	r.Mount("/api", api)
+
+	r.Get("/photos/{fname}.jpg", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "image/jpg")
+		w.Write(photo)
+	})
+
+	r.Get("/", func(w http.ResponseWriter, r *http.Request) {
+		log.Infof("404 URI => %s", r.RequestURI)
+		w.WriteHeader(http.StatusNotFound)
+	})
+
+	addr := "127.0.0.1:3000"
+	log.Infof("Starting server on %s", addr)
+	http.ListenAndServe(addr, r)
+}
+
+func responseOk(w http.ResponseWriter, resp any) {
+	w.Header().Set("Content-Type", "application/json")
+	h := http.StatusOK
+	if err := json.NewEncoder(w).Encode(resp); err != nil {
+		log.Warnf("Error: %v", err)
+		h = http.StatusInternalServerError
+	}
+	w.WriteHeader(h)
+}

cmd/api_mock/user.json

@@ -0,0 +1,15 @@
+{
+  "Response": {
+    "User": {
+      "Uris": {
+        "UserAlbums": {
+          "Uri": "/api/v2/user/myUser!albums",
+          "Locator": "Album",
+          "LocatorType": "Objects",
+          "UriDescription": "All of user's albums",
+          "EndpointType": "UserAlbums"
+        }
+      }
+    }
+  }
+}

cmd/api_mock/useralbums_1.json

@@ -0,0 +1,18 @@
+{
+  "Response": {
+    "Uri": "/api/v2/user/myUser!albums",
+    "Album": [
+      {
+        "UrlPath": "/MyAlbum/2024/01",
+        "Uris": {
+          "AlbumImages": {
+            "Uri": "/api/v2/album/aX3TYu!images"
+          }
+        }
+      }
+    ],
+    "Pages": {
+      "NextPage": ""
+    }
+  }
+}

cmd/smugmug-backup/main.go

@@ -21,6 +21,7 @@ var version = "-- unknown --"
 var flagVersion = flag.Bool("version", false, "print version number")
 var flagStats = flag.Bool("stats", false, fmt.Sprintf("show stats at %s", statsAddr))
 var cfgPath = flag.String("cfg", "", "folder containing configuration file")
+var mockServer = flag.Bool("mock", false, "use the included mock server (must be running on localhost:3000)")
 
 func init() {
 	log.SetFormatter(&log.TextFormatter{})
@@ -56,6 +57,10 @@ func main() {
 		log.WithError(err).Fatal("Configuration error")
 	}
 
+	if *mockServer {
+		cfg.HTTPBaseUrl = "http://localhost:3000"
+	}
+
 	wrk, err := smugmug.New(cfg)
 	if err != nil {
 		log.WithError(err).Fatal("Can't initialize the package")

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/arl/statsviz v0.6.0
+	github.com/go-chi/chi/v5 v5.0.11
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/viper v1.18.2
 )

go.sum

@@ -6,6 +6,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA=
+github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
 github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=

http.go

@@ -12,22 +12,22 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-// maxRetries defines the number of http calls retries (in case of errors) before giving up
-const maxRetries = 3
-const baseAPIURL = "https://api.smugmug.com"
-
 type header struct {
 	name  string
 	value string
 }
 
 type handler struct {
-	oauth *oauthConf
+	baseUrl    string
+	maxRetries int
+	oauth      *oauthConf
 }
 
-func newHTTPHandler(apiKey, apiSecret, userToken, userSecret string) *handler {
+func newHTTPHandler(baseUrl string, maxRetries int, apiKey, apiSecret, userToken, userSecret string) *handler {
 	return &handler{
-		oauth: newOauthConf(apiKey, apiSecret, userToken, userSecret),
+		baseUrl:    baseUrl,
+		maxRetries: maxRetries,
+		oauth:      newOauthConf(apiKey, apiSecret, userToken, userSecret),
 	}
 }
 
@@ -36,7 +36,7 @@ func (s *handler) get(url string, obj interface{}) error {
 	if url == "" {
 		return errors.New("can't get empty url")
 	}
-	return s.getJSON(fmt.Sprintf("%s%s", baseAPIURL, url), obj)
+	return s.getJSON(fmt.Sprintf("%s%s", s.baseUrl, url), obj)
 }
 
 // download the resource (image or video) from the given url to the given destination, checking
@@ -76,7 +76,7 @@ func (s *handler) download(dest, downloadURL string, fileSize int64) (bool, erro
 // getJSON makes a http calls to the given url, trying to decode the JSON response on the given obj
 func (s *handler) getJSON(url string, obj interface{}) error {
 	var result interface{}
-	for i := 1; i <= maxRetries; i++ {
+	for i := 1; i <= s.maxRetries; i++ {
 		log.Debug("Calling ", url)
 		resp, err := s.makeAPICall(url)
 		if err != nil {
@@ -86,7 +86,7 @@ func (s *handler) getJSON(url string, obj interface{}) error {
 		defer resp.Body.Close()
 		if err != nil {
 			log.Errorf("%s: reading response. %s", url, err)
-			if i >= maxRetries {
+			if i >= s.maxRetries {
 				return err
 			}
 		} else {
@@ -103,7 +103,7 @@ func (s *handler) makeAPICall(url string) (*http.Response, error) {
 
 	var resp *http.Response
 	var errorsList []error
-	for i := 1; i <= maxRetries; i++ {
+	for i := 1; i <= s.maxRetries; i++ {
 		req, _ := http.NewRequest("GET", url, nil)
 
 		// Auth header must be generate every time (nonce must change)
@@ -122,7 +122,7 @@ func (s *handler) makeAPICall(url string) (*http.Response, error) {
 		if err != nil {
 			log.Debugf("#%d %s: %s\n", i, url, err)
 			errorsList = append(errorsList, err)
-			if i >= maxRetries {
+			if i >= s.maxRetries {
 				for _, e := range errorsList {
 					log.Error(e)
 				}
@@ -135,7 +135,7 @@ func (s *handler) makeAPICall(url string) (*http.Response, error) {
 
 		if r.StatusCode >= 400 {
 			errorsList = append(errorsList, errors.New(r.Status))
-			if i >= maxRetries {
+			if i >= s.maxRetries {
 				for _, e := range errorsList {
 					log.Error(e)
 				}

json_structs.go

@@ -75,6 +75,7 @@ type albumImage struct {
 	UploadKey        string `json:"UploadKey"`
 	DateTimeOriginal string `json:"DateTimeOriginal"`
 	Caption          string `json:"Caption"`
+	DateTimeUploaded string `json:"DateTimeUploaded"`
 	Keywords         string `json:"Keywords"`
 	Latitude         string `json:"Latitude"`
 	Longitude        string `json:"Longitude"`

smugmug.go

@@ -26,6 +26,8 @@ type Conf struct {
 	ForceVideoDownload  bool   // When true, download videos also if marked as under processing
 	ConcurrentDownloads int    // number of concurrent downloads of images and videos, default is 1
 	ConcurrentAlbums    int    // number of concurrent albums analyzed via API calls
+	HTTPBaseUrl         string // Smugmug API URL, defaults to https://api.smugmug.com
+	HTTPMaxRetries      int    // Max number of retries for HTTP calls, defaults to 3
 
 	username     string
 	metadataFile string
@@ -101,6 +103,8 @@ func ReadConf(cfgPath string) (*Conf, error) {
 	viper.AddConfigPath(".")
 
 	// defaults
+	viper.SetDefault("http.base_url", "https://api.smugmug.com")
+	viper.SetDefault("http.max_retries", 3)
 	viper.SetDefault("store.file_names", "{{.FileName}}")
 	viper.SetDefault("store.concurrent_downloads", 1)
 	viper.SetDefault("store.concurrent_albums", 1)
@@ -130,6 +134,8 @@ func ReadConf(cfgPath string) (*Conf, error) {
 		ForceVideoDownload:  viper.GetBool("store.force_video_download"),
 		ConcurrentDownloads: viper.GetInt("store.concurrent_downloads"),
 		ConcurrentAlbums:    viper.GetInt("store.concurrent_albums"),
+		HTTPBaseUrl:         viper.GetString("http.base_url"),
+		HTTPMaxRetries:      viper.GetInt("http.max_retries"),
 	}
 
 	cfg.overrideEnvConf()
@@ -178,7 +184,7 @@ func New(cfg *Conf) (*Worker, error) {
 		return nil, err
 	}
 
-	handler := newHTTPHandler(cfg.ApiKey, cfg.ApiSecret, cfg.UserToken, cfg.UserSecret)
+	handler := newHTTPHandler(cfg.HTTPBaseUrl, cfg.HTTPMaxRetries, cfg.ApiKey, cfg.ApiSecret, cfg.UserToken, cfg.UserSecret)
 
 	tmpl, err := buildFilenameTemplate(cfg.Filenames)
 	if err != nil {