Add actual file upload to Yandex with progress bar

README.md

@@ -2,7 +2,7 @@
 
 ## English
 
-This app is useful when you want to upload single file to Yandex.Disk but don't want to fiddle with sync official client or WebDAV. It is especially useful for automating backups (e. g. in conjuction with Proxmox's ``vzdump``.)
+This app is useful when you want to upload single file to Yandex.Disk but don't want to fiddle with sync official client or WebDAV. It is especially useful for automating backups (e. g. in conjuction with Proxmox's ``vzdump``).
 
 ## Russian
 

domains/yandex/v1/exported.go

@@ -12,6 +12,7 @@ import (
 const YANDEX_APPID = "7d8a0561fdc44c05bb6695b464403f9c"
 const YANDEX_APPPW = "56e12e4ed0d64738bf441a47f68c7146"
 const DEVICE_NAME = "yapusher-cli"
+const MAX_FILE_SIZE = 10 * 1024 * 1024 * 1024 // 10 gigabytes
 
 var (
 	c    *context.Context
@@ -34,16 +35,23 @@ func New(cc *context.Context) {
 		Name:         "uploadPath",
 		Description:  "Path to upload your file on Yandex.Disk. Must exist before uploading.",
 		Type:         "string",
-		DefaultValue: "/",
+		DefaultValue: "",
 	})
 
 	_ = c.Flagger.AddFlag(&flagger.Flag{
 		Name:         "file",
-		Description:  "Path to file that will be uploaded. Max upload size - 50 GB",
+		Description:  "Path to file that will be uploaded. Max upload size - 10 GB",
 		Type:         "string",
 		DefaultValue: "",
 	})
 
+	_ = c.Flagger.AddFlag(&flagger.Flag{
+		Name:         "force",
+		Description:  "Force file to be uploaded even if destination file on Yandex.Disk already exists.",
+		Type:         "bool",
+		DefaultValue: false,
+	})
+
 	dlog.Info().Msg("Domain initialized")
 }
 
@@ -56,7 +64,9 @@ func Process() {
 
 	filePath, _ := c.Flagger.GetStringValue("file")
 	if filePath != "" {
-		uploadFile()
+		uploadPath, _ := c.Flagger.GetStringValue("uploadPath")
+		forceUpload, _ := c.Flagger.GetBoolValue("force")
+		uploadFile(uploadPath, filePath, forceUpload)
 	}
 
 	if !checkAuth() {

domains/yandex/v1/requests.go

@@ -5,13 +5,18 @@ package yandexv1
 
 import (
 	"encoding/json"
+	"github.com/schollz/progressbar/v2"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 )
 
+func reportFileProgress(r int64) {
+}
+
 func sendCode(code int) {
 	baseURL := "https://oauth.yandex.ru/token"
 
@@ -51,11 +56,108 @@ func sendCode(code int) {
 			dlog.Error().Err(err).Msg("Failed to decode response")
 		}
 
-		dlog.Error().Interface("response", errorData).Msg("Got error from Yandex, not authorized. Please retry authorization")
+		dlog.Error().Str("error", errorData.Error).Str("description", errorData.ErrorDescription).Msg("Got error from Yandex, not authorized. Please retry authorization")
 		authorize()
 	}
 
 	os.Exit(0)
 }
 
-func uploadFile() {}
+func uploadFile(uploadPath string, filePath string, overwriteFile bool) {
+	uploadRequestURL := "https://cloud-api.yandex.net/v1/disk/resources/upload"
+
+	// Checking file existence before requesting
+	normalizedFilePath, _ := filepath.Abs(filePath)
+
+	fileInfo, err := os.Stat(normalizedFilePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			dlog.Fatal().Err(err).Msg("File for uploading not found")
+		} else {
+			dlog.Fatal().Err(err).Msg("Failed to stat uploading file")
+		}
+	}
+
+	if fileInfo.Size() > (MAX_FILE_SIZE - 1) {
+		dlog.Fatal().Msg("Requested file is too big")
+	}
+
+	if !fileInfo.Mode().IsRegular() {
+		dlog.Fatal().Msg("Only regular files uploading is supported right now")
+	}
+
+	client := http.Client{}
+	uploadInfo := UploadInfo{}
+
+	// The first request will get from Yandex upload URL
+	req, _ := http.NewRequest("GET", uploadRequestURL, nil)
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("Authorization", "OAuth "+c.Config.Token.AccessToken)
+
+	query := url.Values{}
+	query.Add("path", "disk:/"+uploadPath+"/"+fileInfo.Name())
+	query.Add("overwrite", strconv.FormatBool(overwriteFile))
+
+	req.URL.RawQuery = query.Encode()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		dlog.Fatal().Err(err).Msg("Failed to send request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		err = json.NewDecoder(resp.Body).Decode(&uploadInfo)
+		if err != nil {
+			dlog.Error().Err(err).Msg("Failed to decode response")
+		}
+	} else {
+		errorData := UploadError{}
+		err = json.NewDecoder(resp.Body).Decode(&errorData)
+		if err != nil {
+			dlog.Error().Err(err).Msg("Failed to decode response")
+		}
+
+		dlog.Info().Str("error", errorData.Error).Str("description", errorData.Description).Msg("Failed to upload file")
+		os.Exit(1)
+	}
+
+	file, _ := os.Open(normalizedFilePath)
+	bar := progressbar.NewOptions(
+		int(fileInfo.Size()),
+		progressbar.OptionSetBytes(int(fileInfo.Size())),
+		progressbar.OptionSetRenderBlankState(true),
+	)
+
+	progressReader := &progressReader{
+		r:           file,
+		progressbar: bar,
+	}
+
+	if uploadInfo.URL == "" {
+		dlog.Fatal().Msg("Got empty upload URL. Report a bug at https://github.com/fat0troll/yapusher/issues because this situation is impossible.")
+	}
+
+	uploadReq, _ := http.NewRequest("PUT", uploadInfo.URL, progressReader)
+
+	uploadResp, err := client.Do(uploadReq)
+	if err != nil {
+		dlog.Fatal().Err(err).Msg("Failed to send upload request")
+	}
+	defer uploadResp.Body.Close()
+
+	switch uploadResp.StatusCode {
+	case http.StatusCreated:
+		dlog.Info().Msg("File uploaded successfully")
+	case http.StatusAccepted:
+		dlog.Info().Msg("File uploaded successfully, but it will take time for Yandex.Disk to handle it internally. Be patient and don't try to upload single file many times")
+	case http.StatusRequestEntityTooLarge:
+		dlog.Fatal().Msg("File upload is too large. Report a bug at https://github.com/fat0troll/yapusher/issues because this situation should be handled before upload attempt.")
+	case http.StatusInsufficientStorage:
+		dlog.Fatal().Msg("There is no space left on your Yandex.Disk.")
+	default:
+		dlog.Fatal().Msg("Failed to upload file (error on Yandex's side). Try again later.")
+	}
+
+	os.Exit(0)
+}

domains/yandex/v1/structs.go

@@ -3,7 +3,46 @@
 
 package yandexv1
 
+import (
+	"fmt"
+	"github.com/schollz/progressbar/v2"
+	"io"
+)
+
+type progressReader struct {
+	r           io.Reader
+	atEOF       bool
+	progressbar *progressbar.ProgressBar
+}
+
+func (pr *progressReader) Read(p []byte) (int, error) {
+	n, err := pr.r.Read(p)
+	if err == io.EOF {
+		pr.atEOF = true
+	}
+	pr.report(int64(n))
+	return n, err
+}
+
+func (pr *progressReader) report(n int64) {
+	_ = pr.progressbar.Add64(n)
+	if pr.atEOF {
+		fmt.Print("\n\n")
+	}
+}
+
 type TokenError struct {
 	Error            string `json:"error"`
 	ErrorDescription string `json:"error_description"`
 }
+
+type UploadError struct {
+	Error       string `json:"error"`
+	Description string `json:"description"`
+}
+
+type UploadInfo struct {
+	URL            string `json:"href"`
+	Method         string `json:"method"`
+	URLIsTemplated bool   `json:"templated"`
+}

go.mod

@@ -5,7 +5,11 @@ go 1.12
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/rs/zerolog v1.13.0
+	github.com/schollz/progressbar v1.0.0 // indirect
+	github.com/schollz/progressbar/v2 v2.10.0
+	github.com/stretchr/objx v0.1.1 // indirect
 	github.com/stretchr/testify v1.3.0 // indirect
 	gitlab.com/pztrn/flagger v0.0.0-20190122123836-d429d7149cc9
 	gitlab.com/pztrn/go-uuid v0.0.0-20190208164458-d6cc46783d2b

go.sum

@@ -3,11 +3,19 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f h1:dKccXx7xA56UNqOcFIbuqFjAWPVtP688j5QMgmo6OHU=
 github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f/go.mod h1:4rEELDSfUAlBSyUjPG0JnaNGjf13JySHFeRdD/3dLP0=
+github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/zerolog v1.13.0 h1:hSNcYHyxDWycfePW7pUI8swuFkcSMPKh3E63Pokg1Hk=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
+github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM=
+github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs=
+github.com/schollz/progressbar/v2 v2.10.0 h1:AxYYUjr5fOPlA0Pcqc3R3kDBrqyLFWk2P7LRLdXb3yE=
+github.com/schollz/progressbar/v2 v2.10.0/go.mod h1:l6tn6yU6ZdQoF8lwX/VoAUQ3FjhCbrcZDnl9xeWZzYw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 gitlab.com/pztrn/flagger v0.0.0-20190122123836-d429d7149cc9 h1:qLlIZybSEr3MSCaE+cJZpT6O/1q3IsBPV7GcZJc05vs=

vendor/github.com/mitchellh/colorstring/.travis.yml

@@ -0,0 +1,15 @@
+language: go
+
+go:
+    - 1.0
+    - 1.1
+    - 1.2
+    - 1.3
+    - tip
+
+script:
+    - go test
+
+matrix:
+    allow_failures:
+        - go: tip

vendor/github.com/mitchellh/colorstring/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Mitchell Hashimoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/mitchellh/colorstring/README.md

@@ -0,0 +1,30 @@
+# colorstring [![Build Status](https://travis-ci.org/mitchellh/colorstring.svg)](https://travis-ci.org/mitchellh/colorstring)
+
+colorstring is a [Go](http://www.golang.org) library for outputting colored
+strings to a console using a simple inline syntax in your string to specify
+the color to print as.
+
+For example, the string `[blue]hello [red]world` would output the text
+"hello world" in two colors. The API of colorstring allows for easily disabling
+colors, adding aliases, etc.
+
+## Installation
+
+Standard `go get`:
+
+```
+$ go get github.com/mitchellh/colorstring
+```
+
+## Usage & Example
+
+For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/colorstring).
+
+Usage is easy enough:
+
+```go
+colorstring.Println("[blue]Hello [red]World!")
+```
+
+Additionally, the `Colorize` struct can be used to set options such as
+custom colors, color disabling, etc.

vendor/github.com/mitchellh/colorstring/colorstring.go

@@ -0,0 +1,244 @@
+// colorstring provides functions for colorizing strings for terminal
+// output.
+package colorstring
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+)
+
+// Color colorizes your strings using the default settings.
+//
+// Strings given to Color should use the syntax `[color]` to specify the
+// color for text following. For example: `[blue]Hello` will return "Hello"
+// in blue. See DefaultColors for all the supported colors and attributes.
+//
+// If an unrecognized color is given, it is ignored and assumed to be part
+// of the string. For example: `[hi]world` will result in "[hi]world".
+//
+// A color reset is appended to the end of every string. This will reset
+// the color of following strings when you output this text to the same
+// terminal session.
+//
+// If you want to customize any of this behavior, use the Colorize struct.
+func Color(v string) string {
+	return def.Color(v)
+}
+
+// ColorPrefix returns the color sequence that prefixes the given text.
+//
+// This is useful when wrapping text if you want to inherit the color
+// of the wrapped text. For example, "[green]foo" will return "[green]".
+// If there is no color sequence, then this will return "".
+func ColorPrefix(v string) string {
+	return def.ColorPrefix(v)
+}
+
+// Colorize colorizes your strings, giving you the ability to customize
+// some of the colorization process.
+//
+// The options in Colorize can be set to customize colorization. If you're
+// only interested in the defaults, just use the top Color function directly,
+// which creates a default Colorize.
+type Colorize struct {
+	// Colors maps a color string to the code for that color. The code
+	// is a string so that you can use more complex colors to set foreground,
+	// background, attributes, etc. For example, "boldblue" might be
+	// "1;34"
+	Colors map[string]string
+
+	// If true, color attributes will be ignored. This is useful if you're
+	// outputting to a location that doesn't support colors and you just
+	// want the strings returned.
+	Disable bool
+
+	// Reset, if true, will reset the color after each colorization by
+	// adding a reset code at the end.
+	Reset bool
+}
+
+// Color colorizes a string according to the settings setup in the struct.
+//
+// For more details on the syntax, see the top-level Color function.
+func (c *Colorize) Color(v string) string {
+	matches := parseRe.FindAllStringIndex(v, -1)
+	if len(matches) == 0 {
+		return v
+	}
+
+	result := new(bytes.Buffer)
+	colored := false
+	m := []int{0, 0}
+	for _, nm := range matches {
+		// Write the text in between this match and the last
+		result.WriteString(v[m[1]:nm[0]])
+		m = nm
+
+		var replace string
+		if code, ok := c.Colors[v[m[0]+1:m[1]-1]]; ok {
+			colored = true
+
+			if !c.Disable {
+				replace = fmt.Sprintf("\033[%sm", code)
+			}
+		} else {
+			replace = v[m[0]:m[1]]
+		}
+
+		result.WriteString(replace)
+	}
+	result.WriteString(v[m[1]:])
+
+	if colored && c.Reset && !c.Disable {
+		// Write the clear byte at the end
+		result.WriteString("\033[0m")
+	}
+
+	return result.String()
+}
+
+// ColorPrefix returns the first color sequence that exists in this string.
+//
+// For example: "[green]foo" would return "[green]". If no color sequence
+// exists, then "" is returned. This is especially useful when wrapping
+// colored texts to inherit the color of the wrapped text.
+func (c *Colorize) ColorPrefix(v string) string {
+	return prefixRe.FindString(strings.TrimSpace(v))
+}
+
+// DefaultColors are the default colors used when colorizing.
+//
+// If the color is surrounded in underscores, such as "_blue_", then that
+// color will be used for the background color.
+var DefaultColors map[string]string
+
+func init() {
+	DefaultColors = map[string]string{
+		// Default foreground/background colors
+		"default":   "39",
+		"_default_": "49",
+
+		// Foreground colors
+		"black":         "30",
+		"red":           "31",
+		"green":         "32",
+		"yellow":        "33",
+		"blue":          "34",
+		"magenta":       "35",
+		"cyan":          "36",
+		"light_gray":    "37",
+		"dark_gray":     "90",
+		"light_red":     "91",
+		"light_green":   "92",
+		"light_yellow":  "93",
+		"light_blue":    "94",
+		"light_magenta": "95",
+		"light_cyan":    "96",
+		"white":         "97",
+
+		// Background colors
+		"_black_":         "40",
+		"_red_":           "41",
+		"_green_":         "42",
+		"_yellow_":        "43",
+		"_blue_":          "44",
+		"_magenta_":       "45",
+		"_cyan_":          "46",
+		"_light_gray_":    "47",
+		"_dark_gray_":     "100",
+		"_light_red_":     "101",
+		"_light_green_":   "102",
+		"_light_yellow_":  "103",
+		"_light_blue_":    "104",
+		"_light_magenta_": "105",
+		"_light_cyan_":    "106",
+		"_white_":         "107",
+
+		// Attributes
+		"bold":       "1",
+		"dim":        "2",
+		"underline":  "4",
+		"blink_slow": "5",
+		"blink_fast": "6",
+		"invert":     "7",
+		"hidden":     "8",
+
+		// Reset to reset everything to their defaults
+		"reset":      "0",
+		"reset_bold": "21",
+	}
+
+	def = Colorize{
+		Colors: DefaultColors,
+		Reset:  true,
+	}
+}
+
+var def Colorize
+var parseReRaw = `\[[a-z0-9_-]+\]`
+var parseRe = regexp.MustCompile(`(?i)` + parseReRaw)
+var prefixRe = regexp.MustCompile(`^(?i)(` + parseReRaw + `)+`)
+
+// Print is a convenience wrapper for fmt.Print with support for color codes.
+//
+// Print formats using the default formats for its operands and writes to
+// standard output with support for color codes. Spaces are added between
+// operands when neither is a string. It returns the number of bytes written
+// and any write error encountered.
+func Print(a string) (n int, err error) {
+	return fmt.Print(Color(a))
+}
+
+// Println is a convenience wrapper for fmt.Println with support for color
+// codes.
+//
+// Println formats using the default formats for its operands and writes to
+// standard output with support for color codes. Spaces are always added
+// between operands and a newline is appended. It returns the number of bytes
+// written and any write error encountered.
+func Println(a string) (n int, err error) {
+	return fmt.Println(Color(a))
+}
+
+// Printf is a convenience wrapper for fmt.Printf with support for color codes.
+//
+// Printf formats according to a format specifier and writes to standard output
+// with support for color codes. It returns the number of bytes written and any
+// write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Printf(Color(format), a...)
+}
+
+// Fprint is a convenience wrapper for fmt.Fprint with support for color codes.
+//
+// Fprint formats using the default formats for its operands and writes to w
+// with support for color codes. Spaces are added between operands when neither
+// is a string. It returns the number of bytes written and any write error
+// encountered.
+func Fprint(w io.Writer, a string) (n int, err error) {
+	return fmt.Fprint(w, Color(a))
+}
+
+// Fprintln is a convenience wrapper for fmt.Fprintln with support for color
+// codes.
+//
+// Fprintln formats using the default formats for its operands and writes to w
+// with support for color codes. Spaces are always added between operands and a
+// newline is appended. It returns the number of bytes written and any write
+// error encountered.
+func Fprintln(w io.Writer, a string) (n int, err error) {
+	return fmt.Fprintln(w, Color(a))
+}
+
+// Fprintf is a convenience wrapper for fmt.Fprintf with support for color
+// codes.
+//
+// Fprintf formats according to a format specifier and writes to w with support
+// for color codes. It returns the number of bytes written and any write error
+// encountered.
+func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(w, Color(format), a...)
+}

vendor/github.com/mitchellh/colorstring/go.mod

@@ -0,0 +1 @@
+module github.com/mitchellh/colorstring

vendor/github.com/schollz/progressbar/v2/.travis.yml

@@ -0,0 +1,4 @@
+language: go
+
+go:
+  - tip

vendor/github.com/schollz/progressbar/v2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Zack
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/schollz/progressbar/v2/README.md

@@ -0,0 +1,122 @@
+# progressbar
+
+[![travis](https://travis-ci.org/schollz/progressbar.svg?branch=master)](https://travis-ci.org/schollz/progressbar) 
+[![go report card](https://goreportcard.com/badge/github.com/schollz/progressbar)](https://goreportcard.com/report/github.com/schollz/progressbar) 
+[![coverage](https://img.shields.io/badge/coverage-84%25-brightgreen.svg)](https://gocover.io/github.com/schollz/progressbar)
+[![godocs](https://godoc.org/github.com/schollz/progressbar?status.svg)](https://godoc.org/github.com/schollz/progressbar) 
+
+A very simple thread-safe progress bar which should work on every OS without problems. I needed a progressbar for [croc](https://github.com/schollz/croc) and everything I tried had problems, so I made another one.
+
+![Example of progress bar](https://user-images.githubusercontent.com/6550035/32120326-5f420d42-bb15-11e7-89d4-c502864e78eb.gif)
+
+## Install
+
+```
+go get -u github.com/schollz/progressbar
+```
+
+## Usage 
+
+### Basic usage
+
+```golang
+bar := progressbar.New(100)
+for i := 0; i < 100; i++ {
+    bar.Add(1)
+    time.Sleep(10 * time.Millisecond)
+}
+```
+
+which looks like:
+
+```bash
+ 100% |████████████████████████████████████████| [1s:0s]
+ ```
+
+The times at the end show the elapsed time and the remaining time, respectively.
+
+### Long running processes
+
+For long running processes, you might want to render from a 0% state.
+
+```golang
+// Renders the bar right on construction
+bar := progressbar.NewOptions(100, progressbar.OptionSetRenderBlankState(true))
+```
+
+Alternatively, when you want to delay rendering, but still want to render a 0% state
+```golang
+bar := progressbar.NewOptions(100)
+
+// Render the current state, which is 0% in this case
+bar.RenderBlank()
+
+// Emulate work
+for i := 0; i < 10; i++ {
+    time.Sleep(10 * time.Minute)
+    bar.Add(10)
+}
+```
+
+### Use a custom writer
+
+The default writer is standard output (os.Stdout), but you can set it to whatever satisfies io.Writer.
+```golang
+bar := NewOptions(
+    10,
+    OptionSetTheme(Theme{Saucer: "#", SaucerPadding: "-", BarStart: ">", BarEnd: "<"}),
+    OptionSetWidth(10),
+    OptionSetWriter(&buf),
+)
+
+bar.Add(5)
+result := strings.TrimSpace(buf.String())
+
+// Result equals:
+// 50% >#####-----< [0s:0s]
+
+```
+
+### Progress for I/O operations
+
+The `progressbar` implements an `io.Writer` so it can automatically detect the number of bytes written to a stream, so you can use it as a progressbar for an `io.Reader`.
+
+```golang
+urlToGet := "https://github.com/schollz/croc/releases/download/v4.1.4/croc_v4.1.4_Windows-64bit_GUI.zip"
+req, _ := http.NewRequest("GET", urlToGet, nil)
+resp, _ := http.DefaultClient.Do(req)
+defer resp.Body.Close()
+
+var out io.Writer
+f, _ := os.OpenFile("croc_v4.1.4_Windows-64bit_GUI.zip", os.O_CREATE|os.O_WRONLY, 0644)
+out = f
+defer f.Close()
+
+bar := progressbar.NewOptions(
+    int(resp.ContentLength), 
+    progressbar.OptionSetBytes(int(resp.ContentLength)),
+)
+out = io.MultiWriter(out, bar)
+io.Copy(out, resp.Body)
+```
+
+See the tests for another example.
+
+## Contributing
+
+Pull requests are welcome. Feel free to...
+
+- Revise documentation
+- Add new features
+- Fix bugs
+- Suggest improvements
+
+## Thanks
+
+Thanks [@Dynom](https://github.com/dynom) for massive improvements in version 2.0!
+
+Thanks [@CrushedPixel](https://github.com/CrushedPixel) for adding descriptions and color code support!
+
+## License
+
+MIT

vendor/github.com/schollz/progressbar/v2/go.mod

@@ -0,0 +1,3 @@
+module github.com/schollz/progressbar/v2
+
+require github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286

vendor/github.com/schollz/progressbar/v2/progressbar.go

@@ -0,0 +1,451 @@
+package progressbar
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mitchellh/colorstring"
+)
+
+// ProgressBar is a thread-safe, simple
+// progress bar
+type ProgressBar struct {
+	state  state
+	config config
+
+	lock sync.Mutex
+}
+
+// State is the basic properties of the bar
+type State struct {
+	CurrentPercent float64
+	CurrentBytes   float64
+	MaxBytes       int64
+	SecondsSince   float64
+	SecondsLeft    float64
+	KBsPerSecond   float64
+}
+
+type state struct {
+	currentNum        int64
+	currentPercent    int
+	lastPercent       int
+	currentSaucerSize int
+
+	lastShown time.Time
+	startTime time.Time
+
+	maxLineWidth int
+	currentBytes float64
+}
+
+type config struct {
+	max                  int64 // max number of the counter
+	width                int
+	writer               io.Writer
+	theme                Theme
+	renderWithBlankState bool
+	description          string
+	// whether the output is expected to contain color codes
+	colorCodes bool
+	maxBytes   int64
+	// show the iterations per second
+	showIterationsPerSecond bool
+	showIterationsCount     bool
+
+	// minimum time to wait in between updates
+	throttleDuration time.Duration
+}
+
+// Theme defines the elements of the bar
+type Theme struct {
+	Saucer        string
+	SaucerHead    string
+	SaucerPadding string
+	BarStart      string
+	BarEnd        string
+}
+
+// Option is the type all options need to adhere to
+type Option func(p *ProgressBar)
+
+// OptionSetWidth sets the width of the bar
+func OptionSetWidth(s int) Option {
+	return func(p *ProgressBar) {
+		p.config.width = s
+	}
+}
+
+// OptionSetTheme sets the elements the bar is constructed of
+func OptionSetTheme(t Theme) Option {
+	return func(p *ProgressBar) {
+		p.config.theme = t
+	}
+}
+
+// OptionSetWriter sets the output writer (defaults to os.StdOut)
+func OptionSetWriter(w io.Writer) Option {
+	return func(p *ProgressBar) {
+		p.config.writer = w
+	}
+}
+
+// OptionSetRenderBlankState sets whether or not to render a 0% bar on construction
+func OptionSetRenderBlankState(r bool) Option {
+	return func(p *ProgressBar) {
+		p.config.renderWithBlankState = r
+	}
+}
+
+// OptionSetDescription sets the description of the bar to render in front of it
+func OptionSetDescription(description string) Option {
+	return func(p *ProgressBar) {
+		p.config.description = description
+	}
+}
+
+// OptionEnableColorCodes enables or disables support for color codes
+// using mitchellh/colorstring
+func OptionEnableColorCodes(colorCodes bool) Option {
+	return func(p *ProgressBar) {
+		p.config.colorCodes = colorCodes
+	}
+}
+
+// OptionSetBytes will also print the bytes/second
+func OptionSetBytes(maxBytes int) Option {
+	return OptionSetBytes64(int64(maxBytes))
+}
+
+// OptionSetBytes64 will also print the bytes/second
+func OptionSetBytes64(maxBytes int64) Option {
+	return func(p *ProgressBar) {
+		p.config.maxBytes = maxBytes
+	}
+}
+
+// OptionShowCount will also print current count out of total
+func OptionShowCount() Option {
+	return func(p *ProgressBar) {
+		p.config.showIterationsCount = true
+	}
+}
+
+// OptionShowIts will also print the iterations/second
+func OptionShowIts() Option {
+	return func(p *ProgressBar) {
+		p.config.showIterationsPerSecond = true
+	}
+}
+
+// OptionThrottle will wait the specified duration before updating again. The default
+// duration is 0 seconds.
+func OptionThrottle(duration time.Duration) Option {
+	return func(p *ProgressBar) {
+		p.config.throttleDuration = duration
+	}
+}
+
+var defaultTheme = Theme{Saucer: "█", SaucerPadding: " ", BarStart: "|", BarEnd: "|"}
+
+// NewOptions constructs a new instance of ProgressBar, with any options you specify
+func NewOptions(max int, options ...Option) *ProgressBar {
+	return NewOptions64(int64(max), options...)
+}
+
+// NewOptions64 constructs a new instance of ProgressBar, with any options you specify
+func NewOptions64(max int64, options ...Option) *ProgressBar {
+	b := ProgressBar{
+		state: getBlankState(),
+		config: config{
+			writer:           os.Stdout,
+			theme:            defaultTheme,
+			width:            40,
+			max:              max,
+			throttleDuration: 0 * time.Nanosecond,
+		},
+	}
+
+	for _, o := range options {
+		o(&b)
+	}
+
+	if b.config.renderWithBlankState {
+		b.RenderBlank()
+	}
+
+	return &b
+}
+
+func getBlankState() state {
+	now := time.Now()
+	return state{
+		startTime: now,
+		lastShown: now,
+	}
+}
+
+// New returns a new ProgressBar
+// with the specified maximum
+func New(max int) *ProgressBar {
+	return NewOptions(max)
+}
+
+// RenderBlank renders the current bar state, you can use this to render a 0% state
+func (p *ProgressBar) RenderBlank() error {
+	return p.render()
+}
+
+// Reset will reset the clock that is used
+// to calculate current time and the time left.
+func (p *ProgressBar) Reset() {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.state = getBlankState()
+}
+
+// Finish will fill the bar to full
+func (p *ProgressBar) Finish() error {
+	p.lock.Lock()
+	p.state.currentNum = p.config.max
+	p.lock.Unlock()
+	return p.Add(0)
+}
+
+// Add with increase the current count on the progress bar
+func (p *ProgressBar) Set(num int) error {
+	return p.Set64(int64(num))
+}
+
+// Add with increase the current count on the progress bar
+func (p *ProgressBar) Set64(num int64) error {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	if p.config.max == 0 {
+		return errors.New("max must be greater than 0")
+	}
+	p.state.currentNum = num
+	percent := float64(p.state.currentNum) / float64(p.config.max)
+	p.state.currentSaucerSize = int(percent * float64(p.config.width))
+	p.state.currentPercent = int(percent * 100)
+	updateBar := p.state.currentPercent != p.state.lastPercent && p.state.currentPercent > 0
+
+	p.state.currentBytes = float64(percent) * float64(p.config.maxBytes)
+	p.state.lastPercent = p.state.currentPercent
+	if p.state.currentNum > p.config.max {
+		return errors.New("current number exceeds max")
+	}
+
+	// always update if show bytes/second or its/second
+	if updateBar || p.config.showIterationsPerSecond || p.config.maxBytes > 0 {
+		return p.render()
+	}
+
+	return nil
+}
+
+func (p *ProgressBar) Add(num int) error {
+	return p.Add64(int64(num))
+}
+
+func (p *ProgressBar) Add64(num int64) error {
+	return p.Set64(p.state.currentNum + num)
+}
+
+// Clear erases the progress bar from the current line
+func (p *ProgressBar) Clear() error {
+	return clearProgressBar(p.config, p.state)
+}
+
+// Describe will change the description shown before the progress, which
+// can be changed on the fly (as for a slow running process).
+func (p *ProgressBar) Describe(description string) {
+	p.config.description = description
+}
+
+// render renders the progress bar, updating the maximum
+// rendered line width. this function is not thread-safe,
+// so it must be called with an acquired lock.
+func (p *ProgressBar) render() error {
+	// make sure that the rendering is not happening too quickly
+	// but always show if the currentNum reaches the max
+	if time.Since(p.state.lastShown).Nanoseconds() < p.config.throttleDuration.Nanoseconds() &&
+		p.state.currentNum < p.config.max {
+		return nil
+	}
+
+	// first, clear the existing progress bar
+	err := clearProgressBar(p.config, p.state)
+	if err != nil {
+		return err
+	}
+
+	// then, re-render the current progress bar
+	w, err := renderProgressBar(p.config, p.state)
+	if err != nil {
+		return err
+	}
+
+	if w > p.state.maxLineWidth {
+		p.state.maxLineWidth = w
+	}
+
+	p.state.lastShown = time.Now()
+
+	return nil
+}
+
+// State returns the current state
+func (p *ProgressBar) State() State {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	s := State{}
+	s.CurrentPercent = float64(p.state.currentNum) / float64(p.config.max)
+	s.CurrentBytes = p.state.currentBytes
+	s.MaxBytes = p.config.maxBytes
+	s.SecondsSince = time.Since(p.state.startTime).Seconds()
+	if p.state.currentNum > 0 {
+		s.SecondsLeft = s.SecondsSince / float64(p.state.currentNum) * (float64(p.config.max) - float64(p.state.currentNum))
+	}
+	s.KBsPerSecond = float64(p.state.currentBytes) / 1000.0 / s.SecondsSince
+	return s
+}
+
+// regex matching ansi escape codes
+var ansiRegex = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`)
+
+func renderProgressBar(c config, s state) (int, error) {
+	var leftTime float64
+	if s.currentNum > 0 {
+		leftTime = time.Since(s.startTime).Seconds() / float64(s.currentNum) * (float64(c.max) - float64(s.currentNum))
+	}
+
+	var saucer string
+	if s.currentSaucerSize > 0 {
+		saucer = strings.Repeat(c.theme.Saucer, s.currentSaucerSize-1)
+		saucerHead := c.theme.SaucerHead
+		if saucerHead == "" || s.currentSaucerSize == c.width {
+			// use the saucer for the saucer head if it hasn't been set
+			// to preserve backwards compatibility
+			saucerHead = c.theme.Saucer
+		}
+		saucer += saucerHead
+	}
+
+	// add on bytes string if max bytes option was set
+	kbPerSecond := float64(s.currentBytes) / 1000.0 / time.Since(s.startTime).Seconds()
+	bytesString := ""
+	if kbPerSecond > 1000.0 {
+		bytesString = fmt.Sprintf("(%2.1f MB/s)", kbPerSecond/1000.0)
+	} else if kbPerSecond > 0 {
+		bytesString = fmt.Sprintf("(%2.1f kB/s)", kbPerSecond)
+	}
+
+	if c.showIterationsPerSecond && !c.showIterationsCount {
+		// replace bytesString if used
+		bytesString = fmt.Sprintf("(%2.0f it/s)", float64(s.currentNum)/time.Since(s.startTime).Seconds())
+	} else if !c.showIterationsPerSecond && c.showIterationsCount {
+		bytesString = fmt.Sprintf("(%d/%d)", s.currentNum, c.max)
+	} else if c.showIterationsPerSecond && c.showIterationsCount {
+		bytesString = fmt.Sprintf("(%d/%d, %2.0f it/s)", s.currentNum, c.max, float64(s.currentNum)/time.Since(s.startTime).Seconds())
+	}
+
+	str := fmt.Sprintf("\r%s%4d%% %s%s%s%s %s [%s:%s]",
+		c.description,
+		s.currentPercent,
+		c.theme.BarStart,
+		saucer,
+		strings.Repeat(c.theme.SaucerPadding, c.width-s.currentSaucerSize),
+		c.theme.BarEnd,
+		bytesString,
+		(time.Duration(time.Since(s.startTime).Seconds()) * time.Second).String(),
+		(time.Duration(leftTime) * time.Second).String(),
+	)
+
+	if c.colorCodes {
+		// convert any color codes in the progress bar into the respective ANSI codes
+		str = colorstring.Color(str)
+	}
+
+	// the width of the string, if printed to the console
+	// does not include the carriage return character
+	cleanString := strings.Replace(str, "\r", "", -1)
+
+	if c.colorCodes {
+		// the ANSI codes for the colors do not take up space in the console output,
+		// so they do not count towards the output string width
+		cleanString = ansiRegex.ReplaceAllString(cleanString, "")
+	}
+
+	// get the amount of runes in the string instead of the
+	// character count of the string, as some runes span multiple characters.
+	// see https://stackoverflow.com/a/12668840/2733724
+	stringWidth := len([]rune(cleanString))
+
+	return stringWidth, writeString(c, str)
+}
+
+func clearProgressBar(c config, s state) error {
+	// fill the current line with enough spaces
+	// to overwrite the progress bar and jump
+	// back to the beginning of the line
+	str := fmt.Sprintf("\r%s\r", strings.Repeat(" ", s.maxLineWidth))
+	return writeString(c, str)
+}
+
+func writeString(c config, str string) error {
+	if _, err := io.WriteString(c.writer, str); err != nil {
+		return err
+	}
+
+	if f, ok := c.writer.(*os.File); ok {
+		// ignore any errors in Sync(), as stdout
+		// can't be synced on some operating systems
+		// like Debian 9 (Stretch)
+		f.Sync()
+	}
+
+	return nil
+}
+
+// Reader is the progressbar io.Reader struct
+type Reader struct {
+	io.Reader
+	bar *ProgressBar
+}
+
+func (r *Reader) Read(p []byte) (n int, err error) {
+	n, err = r.Reader.Read(p)
+	r.bar.Add(n)
+	return
+}
+
+// Close the reader when it implements io.Closer
+func (r *Reader) Close() (err error) {
+	if closer, ok := r.Reader.(io.Closer); ok {
+		return closer.Close()
+	}
+	return
+}
+
+// Write implement io.Writer
+func (p *ProgressBar) Write(b []byte) (n int, err error) {
+	n = len(b)
+	p.Add(n)
+	return
+}
+
+// Read implement io.Reader
+func (p *ProgressBar) Read(b []byte) (n int, err error) {
+	n = len(b)
+	p.Add(n)
+	return
+}

vendor/modules.txt

@@ -1,9 +1,13 @@
 # github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
 github.com/kirsle/configdir
+# github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
+github.com/mitchellh/colorstring
 # github.com/rs/zerolog v1.13.0
 github.com/rs/zerolog
 github.com/rs/zerolog/internal/cbor
 github.com/rs/zerolog/internal/json
+# github.com/schollz/progressbar/v2 v2.10.0
+github.com/schollz/progressbar/v2
 # gitlab.com/pztrn/flagger v0.0.0-20190122123836-d429d7149cc9
 gitlab.com/pztrn/flagger
 # gitlab.com/pztrn/go-uuid v0.0.0-20190208164458-d6cc46783d2b