Merge pull request #2 from 01walid/master

Refactoring, Support links mode and some enhancements

Author: gerdriesselmann

.gitignore

@@ -3,3 +3,4 @@
 *.csv
 .idea
 bin
+vendor/

Gopkg.lock

@@ -0,0 +1,38 @@
+
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"
+
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/spf13/cobra"
+
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "1.1.4"
+
+[[constraint]]
+  name = "github.com/fatih/color"
+  version = "1.5.0"
+
+[[constraint]]
+  name = "gopkg.in/cheggaaa/pb.v1"
+  version = "1.0.18"

Gopkg.toml

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+)
+
+// CError a Red-colored Error string (with Ansi escape codes, supports Windows)
+func CError(format string, a ...interface{}) error {
+	red := color.New(color.FgHiRed).SprintfFunc()
+	// Append a new line for a better error readibility.
+	return fmt.Errorf(red(format+"\n"), a...)
+}
+
+// PrintRed prints a red text into the terminal
+func PrintRed(format string, a ...interface{}) {
+	color.Red(format, a...)
+}
+
+// PrintYellow prints a yellow text into the terminal
+func PrintYellow(format string, a ...interface{}) {
+	color.Yellow(format, a...)
+}
+
+// StringYellow format a text with yellow ansi escape codes
+func StringYellow(text string) string {
+	yellow := color.New(color.FgHiYellow).SprintfFunc()
+	return yellow(text)
+}

cmd/colors.go

@@ -0,0 +1,113 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+// Command Line flags
+var (
+	username    string // Username for Audisto API authentication
+	password    string // Password for audisto API authentication
+	crawlID     uint64 // ID of the crawl to download
+	chunkNumber uint64 // Number of Chunk
+	chunkSize   uint64 // Elements in each chunk
+	output      string // Output format
+	filter      string // Possible filter
+	noResume    bool   // Resume or not any previously downloaded file
+	noDetails   bool   // Request or not details from Audisto API
+	order       string // Possible order of results
+	mode        string // pages or links
+	targets     string // "self" or a path to a file containing link target pages (IDs)
+)
+
+// register global flags that apply to the root command
+func registerPersistentFlags(rootCmd *cobra.Command) {
+	pf := rootCmd.PersistentFlags()
+	pf.StringVarP(&username, "username", "u", "", "Audisto API Username (required)")
+	pf.StringVarP(&password, "password", "p", "", "Audisto API Password (required)")
+	pf.Uint64VarP(&crawlID, "crawl", "c", 0, "ID of the crawl to download (required)")
+	pf.StringVarP(&mode, "mode", "m", "pages", "Download mode, set it to 'links' or 'pages' (default)")
+	pf.BoolVarP(&noDetails, "no-details", "d", false, "If passed, details in API request is set to 0")
+	pf.StringVarP(&output, "output", "o", "", "Path for the output file")
+	pf.BoolVarP(&noResume, "no-resume", "r", false, "If passed, download starts again, else the download is resumed")
+	pf.StringVarP(&filter, "filter", "f", "", "Filter all pages by some attributes")
+	pf.StringVarP(&order, "order", "", "", "Order by some attributes")
+	pf.StringVarP(&targets, "targets", "t", "", `"self" or a path to a file containing link target pages (IDs)`)
+}
+
+// check if --username --password and --crawl are being passed with non-empty values
+func requiredFlagsPassed() bool {
+	return username != "" && password != "" && crawlID != 0
+}
+
+// Beside parsing flags and auto-type inferring offered by Cobra package
+// we check for our own flag validations/logic as well
+func customFlagsValidation(cmd *cobra.Command) error {
+
+	// make sure required flags are passed
+	if !requiredFlagsPassed() {
+		return CError("--username, --password and --crawl are required")
+	}
+
+	// normalize flags before proceeding with the validation
+	normalizeFlags()
+
+	// validate mode
+	if mode != "" && mode != "pages" && mode != "links" {
+		msg := "mode has to be 'links' or 'pages', if this flag is dropped, it will default to 'pages'"
+		return CError(msg)
+	}
+
+	// validate targets / mode / filter combinations
+	if targets != "" {
+
+		// do not allow --filter when --targets is being used
+		if filter != "" {
+			return CError("Set either --filter or --targets, but not both")
+		}
+
+		// --mode=pages is only allowed when targets=self
+		if targets == "self" && mode != "pages" {
+			return CError("Set --mode=pages to use --targets=self")
+		}
+
+		// --targets=FILEPATH is only allowed when mode is set to links
+		// we'd also make sure the file exists.
+		if targets != "self" {
+
+			if mode != "links" {
+				return CError("Set --mode=links to use --targets=FILEPATH")
+			}
+
+			if _, err := os.Stat(targets); os.IsNotExist(err) {
+				return CError(fmt.Sprintf("%s file does not exist", targets))
+			}
+		}
+
+	}
+	// returning no error means the validation passed
+	return nil
+}
+
+// trim spaces and lowercase [some] string-based flags
+func normalizeFlags() {
+
+	// trim spaces for 'mode', 'targets', 'output', 'filter' and 'order'
+	mode = strings.TrimSpace(mode)
+	targets = strings.TrimSpace(targets)
+	output = strings.TrimSpace(output)
+	filter = strings.TrimSpace(filter)
+	order = strings.TrimSpace(order)
+
+	// lowercase 'mode'
+	mode = strings.ToLower(mode)
+
+	// lowercase 'targets' when it's being set to 'self'
+	if strings.EqualFold(targets, "self") {
+		targets = "self"
+	}
+}

cmd/flags.go

@@ -0,0 +1,42 @@
+package cmd
+
+import (
+	"github.com/audisto/data-downloader/downloader"
+	"github.com/spf13/cobra"
+)
+
+// RootCmd the root (default) command to data-downloader
+var RootCmd = &cobra.Command{
+	Use:   "data-downloader",
+	Short: "Audisto Data Downloader",
+	Long:  "A simple CLI tool to download data using Audisto API",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		// Run our custom flags validation
+		err := customFlagsValidation(cmd)
+		if err != nil {
+			return err
+		}
+		return performDownload()
+	},
+	Example: getExamples(),
+}
+
+func init() {
+	// register global flags that apply to the root command
+	registerPersistentFlags(RootCmd)
+}
+
+// use Audisto downloader package to initiate/resume API downloads
+func performDownload() error {
+	return downloader.Get(username, password, crawlID, mode, noDetails,
+		chunkNumber, chunkSize, output, filter, noResume, order)
+}
+
+// example command usage hooked into the CLI usage text.
+func getExamples() string {
+	// Todo: change color
+	return StringYellow(`
+$ data-downloader --username="USERNAME" --password="PASSWORD" --crawl=12345 --output="myCrawl.tsv"
+$ data-downloader -u="USERNAME" -p="PASSWORD" -c=12345 -o="myCrawl.tsv" --no-resume -m=links
+`)
+}

cmd/root.go

@@ -0,0 +1,23 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// VERSION Audisto data downloader version number
+const VERSION = "0.4"
+
+func init() {
+	RootCmd.AddCommand(versionCmd)
+}
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Print the version number of data-downloader",
+	Long:  `Print the version number of data-downloader`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("data-downloader v" + VERSION)
+	},
+}