From 7011ba4c14bba75672548a516291f8408a27658c Mon Sep 17 00:00:00 2001 From: TheDiscordian Date: Tue, 20 Jul 2021 23:58:40 -0400 Subject: [PATCH] Estuary support, closes #49 --- config.yaml.sample | 5 ++++ configuration.go | 20 ++++++++------ main.go | 67 +++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 78 insertions(+), 14 deletions(-) diff --git a/config.yaml.sample b/config.yaml.sample index 319668d..f1b90c5 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -8,6 +8,9 @@ # Path to file where db should be stored (example: "/home/user/.ipfs-sync.db") DB: +# API key for Estuary (optional, find out more at https://estuary.tech) +EstuaryAPIKey: + # Set the dirs to monitor: Dirs: ## Unique identifier for the IPNS key @@ -20,6 +23,8 @@ Dirs: # DontHash: false ## If true, will pin the root directory # Pin: false +## If true, and EstuaryAPIKey is set, will attempt to pin the CID via Estuary as well +# Estuary: false # - ID: Example2 # Dir: /home/user/Pictures/ # Nocopy: false diff --git a/configuration.go b/configuration.go index ff46e15..4f4e115 100644 --- a/configuration.go +++ b/configuration.go @@ -36,6 +36,7 @@ var ( VersionFlag = flag.Bool("version", false, "display version and exit") VerboseFlag = flag.Bool("v", false, "display verbose output") Verbose bool + EstuaryAPIKey string // don't make this a flag version string // passed by -ldflags ) @@ -56,6 +57,7 @@ type DirKey struct { Nocopy bool `yaml:"Nocopy"` DontHash bool `yaml:"DontHash"` Pin bool `yaml:"Pin"` + Estuary bool `yaml:"Estuary"` // probably best to let this be managed automatically CID string @@ -100,14 +102,15 @@ func (ig *IgnoreStruct) String() string { // ConfigFileStruct is used for loading information from the config file. type ConfigFileStruct struct { - BasePath string `yaml:"BasePath"` - EndPoint string `yaml:"EndPoint"` - Dirs []*DirKey `yaml:"Dirs"` - Sync string `yaml:"Sync"` - Ignore []string `yaml:"Ignore"` - DB string `yaml:"DB"` - IgnoreHidden bool `yaml:"IgnoreHidden"` - Timeout string `yaml:"Timeout"` + BasePath string `yaml:"BasePath"` + EndPoint string `yaml:"EndPoint"` + Dirs []*DirKey `yaml:"Dirs"` + Sync string `yaml:"Sync"` + Ignore []string `yaml:"Ignore"` + DB string `yaml:"DB"` + IgnoreHidden bool `yaml:"IgnoreHidden"` + Timeout string `yaml:"Timeout"` + EstuaryAPIKey string `yaml:"EstuaryAPIKey"` } func loadConfig(path string) { @@ -168,6 +171,7 @@ func loadConfig(path string) { DBPath = cfg.DB } IgnoreHidden = cfg.IgnoreHidden + EstuaryAPIKey = cfg.EstuaryAPIKey } // Process flags, and load config. diff --git a/main.go b/main.go index d4b5d01..f53e905 100644 --- a/main.go +++ b/main.go @@ -122,7 +122,7 @@ func filePathWalkDir(root string) ([]string, error) { } // AddDir adds a directory, and returns CID. -func AddDir(path string, nocopy bool, pin bool) (string, error) { +func AddDir(path string, nocopy bool, pin bool, estuary bool) (string, error) { pathSplit := strings.Split(path, string(os.PathSeparator)) dirName := pathSplit[len(pathSplit)-2] files, err := filePathWalkDir(path) @@ -155,7 +155,13 @@ func AddDir(path string, nocopy bool, pin bool) (string, error) { } cid := GetFileCID(dirName) if pin { - err = Pin(cid) + err := Pin(cid) + log.Println("Error pinning", dirName, ":", err) + } + if estuary { + if err := PinEstuary(cid, dirName); err != nil { + log.Println("Error pinning to Estuary:", err) + } } return cid, err } @@ -241,9 +247,9 @@ func AddFile(from, to string, nocopy bool, makedir bool, overwrite bool) (string if Verbose { log.Println("Error on files/cp:", err) log.Println("fpath:", from) - if HandleBadBlockError(err) { - log.Println("files/cp failure due to bad filestore, file not added") - } + } + if HandleBadBlockError(err) { + log.Println("files/cp failure due to bad filestore, file not added") } } return hash.Hash, err @@ -446,6 +452,50 @@ func Publish(cid, key string) error { return err } +type EstuaryFile struct { + Cid string + Name string +} + +func PinEstuary(cid, name string) error { + if EstuaryAPIKey == "" { + return errors.New("Estuary API key is blank, not pinning to Estuary.") + } + var cancel context.CancelFunc + ctx := context.Background() + if TimeoutTime > 0 { + ctx, cancel = context.WithTimeout(ctx, TimeoutTime) + defer cancel() + } + c := &http.Client{} + jsonData, _ := json.Marshal(&EstuaryFile{Cid: cid, Name: name}) + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.estuary.tech/pinning/pins", bytes.NewBuffer(jsonData)) + if err != nil { + return err + } + req.Header.Add("Authorization", "Bearer "+EstuaryAPIKey) + req.Header.Add("Content-Type", "application/json") + resp, err := c.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + errStruct := new(ErrorStruct) + err = json.Unmarshal(body, errStruct) + if err == nil { + if errStruct.Error() != "" { + return errStruct + } + } + + return nil +} + // WatchDog watches for directory updates, periodically updates IPNS records, and updates recursive pins. func WatchDog() { // Init WatchDog @@ -522,7 +572,7 @@ func WatchDog() { log.Println(dk.ID, "not found, generating...") ik := GenerateKey(dk.ID) var err error - dk.CID, err = AddDir(dk.Dir, dk.Nocopy, dk.Pin) + dk.CID, err = AddDir(dk.Dir, dk.Nocopy, dk.Pin, dk.Estuary) if err != nil { log.Panicln("[ERROR] Failed to add directory:", err) } @@ -541,6 +591,11 @@ func WatchDog() { if dk.Pin { UpdatePin(dk.CID, fCID) } + if dk.Estuary { + if err := PinEstuary(fCID, strings.Split(dk.MFSPath, "/")[0]); err != nil { + log.Println("Error pinning to Estuary:", err) + } + } dk.CID = fCID log.Println(dk.MFSPath, "updated...") }