From 1fbeb6c0639a12a05a78a41d7b2968762c7387c3 Mon Sep 17 00:00:00 2001 From: Vladimir Hodakov Date: Sat, 30 Mar 2019 07:00:15 +0400 Subject: [PATCH] Add actual file upload to Yandex with progress bar --- README.md | 2 +- domains/yandex/v1/exported.go | 16 +- domains/yandex/v1/requests.go | 106 +++- domains/yandex/v1/structs.go | 39 ++ go.mod | 4 + go.sum | 8 + .../mitchellh/colorstring/.travis.yml | 15 + .../github.com/mitchellh/colorstring/LICENSE | 21 + .../mitchellh/colorstring/README.md | 30 ++ .../mitchellh/colorstring/colorstring.go | 244 ++++++++++ .../github.com/mitchellh/colorstring/go.mod | 1 + .../schollz/progressbar/v2/.travis.yml | 4 + .../github.com/schollz/progressbar/v2/LICENSE | 21 + .../schollz/progressbar/v2/README.md | 122 +++++ .../github.com/schollz/progressbar/v2/go.mod | 3 + .../schollz/progressbar/v2/progressbar.go | 451 ++++++++++++++++++ vendor/modules.txt | 4 + 17 files changed, 1085 insertions(+), 6 deletions(-) create mode 100644 vendor/github.com/mitchellh/colorstring/.travis.yml create mode 100644 vendor/github.com/mitchellh/colorstring/LICENSE create mode 100644 vendor/github.com/mitchellh/colorstring/README.md create mode 100644 vendor/github.com/mitchellh/colorstring/colorstring.go create mode 100644 vendor/github.com/mitchellh/colorstring/go.mod create mode 100644 vendor/github.com/schollz/progressbar/v2/.travis.yml create mode 100644 vendor/github.com/schollz/progressbar/v2/LICENSE create mode 100644 vendor/github.com/schollz/progressbar/v2/README.md create mode 100644 vendor/github.com/schollz/progressbar/v2/go.mod create mode 100644 vendor/github.com/schollz/progressbar/v2/progressbar.go diff --git a/README.md b/README.md index ba7e5c2..466c970 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## English -This app is useful when you want to upload single file to Yandex.Disk but don't want to fiddle with sync official client or WebDAV. It is especially useful for automating backups (e. g. in conjuction with Proxmox's ``vzdump``.) +This app is useful when you want to upload single file to Yandex.Disk but don't want to fiddle with sync official client or WebDAV. It is especially useful for automating backups (e. g. in conjuction with Proxmox's ``vzdump``). ## Russian diff --git a/domains/yandex/v1/exported.go b/domains/yandex/v1/exported.go index 74bad95..3d139ae 100644 --- a/domains/yandex/v1/exported.go +++ b/domains/yandex/v1/exported.go @@ -12,6 +12,7 @@ import ( const YANDEX_APPID = "7d8a0561fdc44c05bb6695b464403f9c" const YANDEX_APPPW = "56e12e4ed0d64738bf441a47f68c7146" const DEVICE_NAME = "yapusher-cli" +const MAX_FILE_SIZE = 10 * 1024 * 1024 * 1024 // 10 gigabytes var ( c *context.Context @@ -34,16 +35,23 @@ func New(cc *context.Context) { Name: "uploadPath", Description: "Path to upload your file on Yandex.Disk. Must exist before uploading.", Type: "string", - DefaultValue: "/", + DefaultValue: "", }) _ = c.Flagger.AddFlag(&flagger.Flag{ Name: "file", - Description: "Path to file that will be uploaded. Max upload size - 50 GB", + Description: "Path to file that will be uploaded. Max upload size - 10 GB", Type: "string", DefaultValue: "", }) + _ = c.Flagger.AddFlag(&flagger.Flag{ + Name: "force", + Description: "Force file to be uploaded even if destination file on Yandex.Disk already exists.", + Type: "bool", + DefaultValue: false, + }) + dlog.Info().Msg("Domain initialized") } @@ -56,7 +64,9 @@ func Process() { filePath, _ := c.Flagger.GetStringValue("file") if filePath != "" { - uploadFile() + uploadPath, _ := c.Flagger.GetStringValue("uploadPath") + forceUpload, _ := c.Flagger.GetBoolValue("force") + uploadFile(uploadPath, filePath, forceUpload) } if !checkAuth() { diff --git a/domains/yandex/v1/requests.go b/domains/yandex/v1/requests.go index dd95f76..45a39c8 100644 --- a/domains/yandex/v1/requests.go +++ b/domains/yandex/v1/requests.go @@ -5,13 +5,18 @@ package yandexv1 import ( "encoding/json" + "github.com/schollz/progressbar/v2" "net/http" "net/url" "os" + "path/filepath" "strconv" "strings" ) +func reportFileProgress(r int64) { +} + func sendCode(code int) { baseURL := "https://oauth.yandex.ru/token" @@ -51,11 +56,108 @@ func sendCode(code int) { dlog.Error().Err(err).Msg("Failed to decode response") } - dlog.Error().Interface("response", errorData).Msg("Got error from Yandex, not authorized. Please retry authorization") + dlog.Error().Str("error", errorData.Error).Str("description", errorData.ErrorDescription).Msg("Got error from Yandex, not authorized. Please retry authorization") authorize() } os.Exit(0) } -func uploadFile() {} +func uploadFile(uploadPath string, filePath string, overwriteFile bool) { + uploadRequestURL := "https://cloud-api.yandex.net/v1/disk/resources/upload" + + // Checking file existence before requesting + normalizedFilePath, _ := filepath.Abs(filePath) + + fileInfo, err := os.Stat(normalizedFilePath) + if err != nil { + if os.IsNotExist(err) { + dlog.Fatal().Err(err).Msg("File for uploading not found") + } else { + dlog.Fatal().Err(err).Msg("Failed to stat uploading file") + } + } + + if fileInfo.Size() > (MAX_FILE_SIZE - 1) { + dlog.Fatal().Msg("Requested file is too big") + } + + if !fileInfo.Mode().IsRegular() { + dlog.Fatal().Msg("Only regular files uploading is supported right now") + } + + client := http.Client{} + uploadInfo := UploadInfo{} + + // The first request will get from Yandex upload URL + req, _ := http.NewRequest("GET", uploadRequestURL, nil) + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", "OAuth "+c.Config.Token.AccessToken) + + query := url.Values{} + query.Add("path", "disk:/"+uploadPath+"/"+fileInfo.Name()) + query.Add("overwrite", strconv.FormatBool(overwriteFile)) + + req.URL.RawQuery = query.Encode() + + resp, err := client.Do(req) + if err != nil { + dlog.Fatal().Err(err).Msg("Failed to send request") + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + err = json.NewDecoder(resp.Body).Decode(&uploadInfo) + if err != nil { + dlog.Error().Err(err).Msg("Failed to decode response") + } + } else { + errorData := UploadError{} + err = json.NewDecoder(resp.Body).Decode(&errorData) + if err != nil { + dlog.Error().Err(err).Msg("Failed to decode response") + } + + dlog.Info().Str("error", errorData.Error).Str("description", errorData.Description).Msg("Failed to upload file") + os.Exit(1) + } + + file, _ := os.Open(normalizedFilePath) + bar := progressbar.NewOptions( + int(fileInfo.Size()), + progressbar.OptionSetBytes(int(fileInfo.Size())), + progressbar.OptionSetRenderBlankState(true), + ) + + progressReader := &progressReader{ + r: file, + progressbar: bar, + } + + if uploadInfo.URL == "" { + dlog.Fatal().Msg("Got empty upload URL. Report a bug at https://github.com/fat0troll/yapusher/issues because this situation is impossible.") + } + + uploadReq, _ := http.NewRequest("PUT", uploadInfo.URL, progressReader) + + uploadResp, err := client.Do(uploadReq) + if err != nil { + dlog.Fatal().Err(err).Msg("Failed to send upload request") + } + defer uploadResp.Body.Close() + + switch uploadResp.StatusCode { + case http.StatusCreated: + dlog.Info().Msg("File uploaded successfully") + case http.StatusAccepted: + dlog.Info().Msg("File uploaded successfully, but it will take time for Yandex.Disk to handle it internally. Be patient and don't try to upload single file many times") + case http.StatusRequestEntityTooLarge: + dlog.Fatal().Msg("File upload is too large. Report a bug at https://github.com/fat0troll/yapusher/issues because this situation should be handled before upload attempt.") + case http.StatusInsufficientStorage: + dlog.Fatal().Msg("There is no space left on your Yandex.Disk.") + default: + dlog.Fatal().Msg("Failed to upload file (error on Yandex's side). Try again later.") + } + + os.Exit(0) +} diff --git a/domains/yandex/v1/structs.go b/domains/yandex/v1/structs.go index 99bdadf..b1d4960 100644 --- a/domains/yandex/v1/structs.go +++ b/domains/yandex/v1/structs.go @@ -3,7 +3,46 @@ package yandexv1 +import ( + "fmt" + "github.com/schollz/progressbar/v2" + "io" +) + +type progressReader struct { + r io.Reader + atEOF bool + progressbar *progressbar.ProgressBar +} + +func (pr *progressReader) Read(p []byte) (int, error) { + n, err := pr.r.Read(p) + if err == io.EOF { + pr.atEOF = true + } + pr.report(int64(n)) + return n, err +} + +func (pr *progressReader) report(n int64) { + _ = pr.progressbar.Add64(n) + if pr.atEOF { + fmt.Print("\n\n") + } +} + type TokenError struct { Error string `json:"error"` ErrorDescription string `json:"error_description"` } + +type UploadError struct { + Error string `json:"error"` + Description string `json:"description"` +} + +type UploadInfo struct { + URL string `json:"href"` + Method string `json:"method"` + URLIsTemplated bool `json:"templated"` +} diff --git a/go.mod b/go.mod index ce6f6bb..c40fe41 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,11 @@ go 1.12 require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/rs/zerolog v1.13.0 + github.com/schollz/progressbar v1.0.0 // indirect + github.com/schollz/progressbar/v2 v2.10.0 + github.com/stretchr/objx v0.1.1 // indirect github.com/stretchr/testify v1.3.0 // indirect gitlab.com/pztrn/flagger v0.0.0-20190122123836-d429d7149cc9 gitlab.com/pztrn/go-uuid v0.0.0-20190208164458-d6cc46783d2b diff --git a/go.sum b/go.sum index c66979c..6451af6 100644 --- a/go.sum +++ b/go.sum @@ -3,11 +3,19 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f h1:dKccXx7xA56UNqOcFIbuqFjAWPVtP688j5QMgmo6OHU= github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f/go.mod h1:4rEELDSfUAlBSyUjPG0JnaNGjf13JySHFeRdD/3dLP0= +github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/zerolog v1.13.0 h1:hSNcYHyxDWycfePW7pUI8swuFkcSMPKh3E63Pokg1Hk= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= +github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= +github.com/schollz/progressbar/v2 v2.10.0 h1:AxYYUjr5fOPlA0Pcqc3R3kDBrqyLFWk2P7LRLdXb3yE= +github.com/schollz/progressbar/v2 v2.10.0/go.mod h1:l6tn6yU6ZdQoF8lwX/VoAUQ3FjhCbrcZDnl9xeWZzYw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= gitlab.com/pztrn/flagger v0.0.0-20190122123836-d429d7149cc9 h1:qLlIZybSEr3MSCaE+cJZpT6O/1q3IsBPV7GcZJc05vs= diff --git a/vendor/github.com/mitchellh/colorstring/.travis.yml b/vendor/github.com/mitchellh/colorstring/.travis.yml new file mode 100644 index 0000000..74e286a --- /dev/null +++ b/vendor/github.com/mitchellh/colorstring/.travis.yml @@ -0,0 +1,15 @@ +language: go + +go: + - 1.0 + - 1.1 + - 1.2 + - 1.3 + - tip + +script: + - go test + +matrix: + allow_failures: + - go: tip diff --git a/vendor/github.com/mitchellh/colorstring/LICENSE b/vendor/github.com/mitchellh/colorstring/LICENSE new file mode 100644 index 0000000..2298515 --- /dev/null +++ b/vendor/github.com/mitchellh/colorstring/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/colorstring/README.md b/vendor/github.com/mitchellh/colorstring/README.md new file mode 100644 index 0000000..0654d45 --- /dev/null +++ b/vendor/github.com/mitchellh/colorstring/README.md @@ -0,0 +1,30 @@ +# colorstring [![Build Status](https://travis-ci.org/mitchellh/colorstring.svg)](https://travis-ci.org/mitchellh/colorstring) + +colorstring is a [Go](http://www.golang.org) library for outputting colored +strings to a console using a simple inline syntax in your string to specify +the color to print as. + +For example, the string `[blue]hello [red]world` would output the text +"hello world" in two colors. The API of colorstring allows for easily disabling +colors, adding aliases, etc. + +## Installation + +Standard `go get`: + +``` +$ go get github.com/mitchellh/colorstring +``` + +## Usage & Example + +For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/colorstring). + +Usage is easy enough: + +```go +colorstring.Println("[blue]Hello [red]World!") +``` + +Additionally, the `Colorize` struct can be used to set options such as +custom colors, color disabling, etc. diff --git a/vendor/github.com/mitchellh/colorstring/colorstring.go b/vendor/github.com/mitchellh/colorstring/colorstring.go new file mode 100644 index 0000000..3de5b24 --- /dev/null +++ b/vendor/github.com/mitchellh/colorstring/colorstring.go @@ -0,0 +1,244 @@ +// colorstring provides functions for colorizing strings for terminal +// output. +package colorstring + +import ( + "bytes" + "fmt" + "io" + "regexp" + "strings" +) + +// Color colorizes your strings using the default settings. +// +// Strings given to Color should use the syntax `[color]` to specify the +// color for text following. For example: `[blue]Hello` will return "Hello" +// in blue. See DefaultColors for all the supported colors and attributes. +// +// If an unrecognized color is given, it is ignored and assumed to be part +// of the string. For example: `[hi]world` will result in "[hi]world". +// +// A color reset is appended to the end of every string. This will reset +// the color of following strings when you output this text to the same +// terminal session. +// +// If you want to customize any of this behavior, use the Colorize struct. +func Color(v string) string { + return def.Color(v) +} + +// ColorPrefix returns the color sequence that prefixes the given text. +// +// This is useful when wrapping text if you want to inherit the color +// of the wrapped text. For example, "[green]foo" will return "[green]". +// If there is no color sequence, then this will return "". +func ColorPrefix(v string) string { + return def.ColorPrefix(v) +} + +// Colorize colorizes your strings, giving you the ability to customize +// some of the colorization process. +// +// The options in Colorize can be set to customize colorization. If you're +// only interested in the defaults, just use the top Color function directly, +// which creates a default Colorize. +type Colorize struct { + // Colors maps a color string to the code for that color. The code + // is a string so that you can use more complex colors to set foreground, + // background, attributes, etc. For example, "boldblue" might be + // "1;34" + Colors map[string]string + + // If true, color attributes will be ignored. This is useful if you're + // outputting to a location that doesn't support colors and you just + // want the strings returned. + Disable bool + + // Reset, if true, will reset the color after each colorization by + // adding a reset code at the end. + Reset bool +} + +// Color colorizes a string according to the settings setup in the struct. +// +// For more details on the syntax, see the top-level Color function. +func (c *Colorize) Color(v string) string { + matches := parseRe.FindAllStringIndex(v, -1) + if len(matches) == 0 { + return v + } + + result := new(bytes.Buffer) + colored := false + m := []int{0, 0} + for _, nm := range matches { + // Write the text in between this match and the last + result.WriteString(v[m[1]:nm[0]]) + m = nm + + var replace string + if code, ok := c.Colors[v[m[0]+1:m[1]-1]]; ok { + colored = true + + if !c.Disable { + replace = fmt.Sprintf("\033[%sm", code) + } + } else { + replace = v[m[0]:m[1]] + } + + result.WriteString(replace) + } + result.WriteString(v[m[1]:]) + + if colored && c.Reset && !c.Disable { + // Write the clear byte at the end + result.WriteString("\033[0m") + } + + return result.String() +} + +// ColorPrefix returns the first color sequence that exists in this string. +// +// For example: "[green]foo" would return "[green]". If no color sequence +// exists, then "" is returned. This is especially useful when wrapping +// colored texts to inherit the color of the wrapped text. +func (c *Colorize) ColorPrefix(v string) string { + return prefixRe.FindString(strings.TrimSpace(v)) +} + +// DefaultColors are the default colors used when colorizing. +// +// If the color is surrounded in underscores, such as "_blue_", then that +// color will be used for the background color. +var DefaultColors map[string]string + +func init() { + DefaultColors = map[string]string{ + // Default foreground/background colors + "default": "39", + "_default_": "49", + + // Foreground colors + "black": "30", + "red": "31", + "green": "32", + "yellow": "33", + "blue": "34", + "magenta": "35", + "cyan": "36", + "light_gray": "37", + "dark_gray": "90", + "light_red": "91", + "light_green": "92", + "light_yellow": "93", + "light_blue": "94", + "light_magenta": "95", + "light_cyan": "96", + "white": "97", + + // Background colors + "_black_": "40", + "_red_": "41", + "_green_": "42", + "_yellow_": "43", + "_blue_": "44", + "_magenta_": "45", + "_cyan_": "46", + "_light_gray_": "47", + "_dark_gray_": "100", + "_light_red_": "101", + "_light_green_": "102", + "_light_yellow_": "103", + "_light_blue_": "104", + "_light_magenta_": "105", + "_light_cyan_": "106", + "_white_": "107", + + // Attributes + "bold": "1", + "dim": "2", + "underline": "4", + "blink_slow": "5", + "blink_fast": "6", + "invert": "7", + "hidden": "8", + + // Reset to reset everything to their defaults + "reset": "0", + "reset_bold": "21", + } + + def = Colorize{ + Colors: DefaultColors, + Reset: true, + } +} + +var def Colorize +var parseReRaw = `\[[a-z0-9_-]+\]` +var parseRe = regexp.MustCompile(`(?i)` + parseReRaw) +var prefixRe = regexp.MustCompile(`^(?i)(` + parseReRaw + `)+`) + +// Print is a convenience wrapper for fmt.Print with support for color codes. +// +// Print formats using the default formats for its operands and writes to +// standard output with support for color codes. Spaces are added between +// operands when neither is a string. It returns the number of bytes written +// and any write error encountered. +func Print(a string) (n int, err error) { + return fmt.Print(Color(a)) +} + +// Println is a convenience wrapper for fmt.Println with support for color +// codes. +// +// Println formats using the default formats for its operands and writes to +// standard output with support for color codes. Spaces are always added +// between operands and a newline is appended. It returns the number of bytes +// written and any write error encountered. +func Println(a string) (n int, err error) { + return fmt.Println(Color(a)) +} + +// Printf is a convenience wrapper for fmt.Printf with support for color codes. +// +// Printf formats according to a format specifier and writes to standard output +// with support for color codes. It returns the number of bytes written and any +// write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Printf(Color(format), a...) +} + +// Fprint is a convenience wrapper for fmt.Fprint with support for color codes. +// +// Fprint formats using the default formats for its operands and writes to w +// with support for color codes. Spaces are added between operands when neither +// is a string. It returns the number of bytes written and any write error +// encountered. +func Fprint(w io.Writer, a string) (n int, err error) { + return fmt.Fprint(w, Color(a)) +} + +// Fprintln is a convenience wrapper for fmt.Fprintln with support for color +// codes. +// +// Fprintln formats using the default formats for its operands and writes to w +// with support for color codes. Spaces are always added between operands and a +// newline is appended. It returns the number of bytes written and any write +// error encountered. +func Fprintln(w io.Writer, a string) (n int, err error) { + return fmt.Fprintln(w, Color(a)) +} + +// Fprintf is a convenience wrapper for fmt.Fprintf with support for color +// codes. +// +// Fprintf formats according to a format specifier and writes to w with support +// for color codes. It returns the number of bytes written and any write error +// encountered. +func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(w, Color(format), a...) +} diff --git a/vendor/github.com/mitchellh/colorstring/go.mod b/vendor/github.com/mitchellh/colorstring/go.mod new file mode 100644 index 0000000..446ff8d --- /dev/null +++ b/vendor/github.com/mitchellh/colorstring/go.mod @@ -0,0 +1 @@ +module github.com/mitchellh/colorstring diff --git a/vendor/github.com/schollz/progressbar/v2/.travis.yml b/vendor/github.com/schollz/progressbar/v2/.travis.yml new file mode 100644 index 0000000..b68d9f4 --- /dev/null +++ b/vendor/github.com/schollz/progressbar/v2/.travis.yml @@ -0,0 +1,4 @@ +language: go + +go: + - tip \ No newline at end of file diff --git a/vendor/github.com/schollz/progressbar/v2/LICENSE b/vendor/github.com/schollz/progressbar/v2/LICENSE new file mode 100644 index 0000000..0ca9765 --- /dev/null +++ b/vendor/github.com/schollz/progressbar/v2/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Zack + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/schollz/progressbar/v2/README.md b/vendor/github.com/schollz/progressbar/v2/README.md new file mode 100644 index 0000000..b59f467 --- /dev/null +++ b/vendor/github.com/schollz/progressbar/v2/README.md @@ -0,0 +1,122 @@ +# progressbar + +[![travis](https://travis-ci.org/schollz/progressbar.svg?branch=master)](https://travis-ci.org/schollz/progressbar) +[![go report card](https://goreportcard.com/badge/github.com/schollz/progressbar)](https://goreportcard.com/report/github.com/schollz/progressbar) +[![coverage](https://img.shields.io/badge/coverage-84%25-brightgreen.svg)](https://gocover.io/github.com/schollz/progressbar) +[![godocs](https://godoc.org/github.com/schollz/progressbar?status.svg)](https://godoc.org/github.com/schollz/progressbar) + +A very simple thread-safe progress bar which should work on every OS without problems. I needed a progressbar for [croc](https://github.com/schollz/croc) and everything I tried had problems, so I made another one. + +![Example of progress bar](https://user-images.githubusercontent.com/6550035/32120326-5f420d42-bb15-11e7-89d4-c502864e78eb.gif) + +## Install + +``` +go get -u github.com/schollz/progressbar +``` + +## Usage + +### Basic usage + +```golang +bar := progressbar.New(100) +for i := 0; i < 100; i++ { + bar.Add(1) + time.Sleep(10 * time.Millisecond) +} +``` + +which looks like: + +```bash + 100% |████████████████████████████████████████| [1s:0s] + ``` + +The times at the end show the elapsed time and the remaining time, respectively. + +### Long running processes + +For long running processes, you might want to render from a 0% state. + +```golang +// Renders the bar right on construction +bar := progressbar.NewOptions(100, progressbar.OptionSetRenderBlankState(true)) +``` + +Alternatively, when you want to delay rendering, but still want to render a 0% state +```golang +bar := progressbar.NewOptions(100) + +// Render the current state, which is 0% in this case +bar.RenderBlank() + +// Emulate work +for i := 0; i < 10; i++ { + time.Sleep(10 * time.Minute) + bar.Add(10) +} +``` + +### Use a custom writer + +The default writer is standard output (os.Stdout), but you can set it to whatever satisfies io.Writer. +```golang +bar := NewOptions( + 10, + OptionSetTheme(Theme{Saucer: "#", SaucerPadding: "-", BarStart: ">", BarEnd: "<"}), + OptionSetWidth(10), + OptionSetWriter(&buf), +) + +bar.Add(5) +result := strings.TrimSpace(buf.String()) + +// Result equals: +// 50% >#####-----< [0s:0s] + +``` + +### Progress for I/O operations + +The `progressbar` implements an `io.Writer` so it can automatically detect the number of bytes written to a stream, so you can use it as a progressbar for an `io.Reader`. + +```golang +urlToGet := "https://github.com/schollz/croc/releases/download/v4.1.4/croc_v4.1.4_Windows-64bit_GUI.zip" +req, _ := http.NewRequest("GET", urlToGet, nil) +resp, _ := http.DefaultClient.Do(req) +defer resp.Body.Close() + +var out io.Writer +f, _ := os.OpenFile("croc_v4.1.4_Windows-64bit_GUI.zip", os.O_CREATE|os.O_WRONLY, 0644) +out = f +defer f.Close() + +bar := progressbar.NewOptions( + int(resp.ContentLength), + progressbar.OptionSetBytes(int(resp.ContentLength)), +) +out = io.MultiWriter(out, bar) +io.Copy(out, resp.Body) +``` + +See the tests for another example. + +## Contributing + +Pull requests are welcome. Feel free to... + +- Revise documentation +- Add new features +- Fix bugs +- Suggest improvements + +## Thanks + +Thanks [@Dynom](https://github.com/dynom) for massive improvements in version 2.0! + +Thanks [@CrushedPixel](https://github.com/CrushedPixel) for adding descriptions and color code support! + +## License + +MIT diff --git a/vendor/github.com/schollz/progressbar/v2/go.mod b/vendor/github.com/schollz/progressbar/v2/go.mod new file mode 100644 index 0000000..0fad234 --- /dev/null +++ b/vendor/github.com/schollz/progressbar/v2/go.mod @@ -0,0 +1,3 @@ +module github.com/schollz/progressbar/v2 + +require github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286 diff --git a/vendor/github.com/schollz/progressbar/v2/progressbar.go b/vendor/github.com/schollz/progressbar/v2/progressbar.go new file mode 100644 index 0000000..b66199b --- /dev/null +++ b/vendor/github.com/schollz/progressbar/v2/progressbar.go @@ -0,0 +1,451 @@ +package progressbar + +import ( + "errors" + "fmt" + "io" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/mitchellh/colorstring" +) + +// ProgressBar is a thread-safe, simple +// progress bar +type ProgressBar struct { + state state + config config + + lock sync.Mutex +} + +// State is the basic properties of the bar +type State struct { + CurrentPercent float64 + CurrentBytes float64 + MaxBytes int64 + SecondsSince float64 + SecondsLeft float64 + KBsPerSecond float64 +} + +type state struct { + currentNum int64 + currentPercent int + lastPercent int + currentSaucerSize int + + lastShown time.Time + startTime time.Time + + maxLineWidth int + currentBytes float64 +} + +type config struct { + max int64 // max number of the counter + width int + writer io.Writer + theme Theme + renderWithBlankState bool + description string + // whether the output is expected to contain color codes + colorCodes bool + maxBytes int64 + // show the iterations per second + showIterationsPerSecond bool + showIterationsCount bool + + // minimum time to wait in between updates + throttleDuration time.Duration +} + +// Theme defines the elements of the bar +type Theme struct { + Saucer string + SaucerHead string + SaucerPadding string + BarStart string + BarEnd string +} + +// Option is the type all options need to adhere to +type Option func(p *ProgressBar) + +// OptionSetWidth sets the width of the bar +func OptionSetWidth(s int) Option { + return func(p *ProgressBar) { + p.config.width = s + } +} + +// OptionSetTheme sets the elements the bar is constructed of +func OptionSetTheme(t Theme) Option { + return func(p *ProgressBar) { + p.config.theme = t + } +} + +// OptionSetWriter sets the output writer (defaults to os.StdOut) +func OptionSetWriter(w io.Writer) Option { + return func(p *ProgressBar) { + p.config.writer = w + } +} + +// OptionSetRenderBlankState sets whether or not to render a 0% bar on construction +func OptionSetRenderBlankState(r bool) Option { + return func(p *ProgressBar) { + p.config.renderWithBlankState = r + } +} + +// OptionSetDescription sets the description of the bar to render in front of it +func OptionSetDescription(description string) Option { + return func(p *ProgressBar) { + p.config.description = description + } +} + +// OptionEnableColorCodes enables or disables support for color codes +// using mitchellh/colorstring +func OptionEnableColorCodes(colorCodes bool) Option { + return func(p *ProgressBar) { + p.config.colorCodes = colorCodes + } +} + +// OptionSetBytes will also print the bytes/second +func OptionSetBytes(maxBytes int) Option { + return OptionSetBytes64(int64(maxBytes)) +} + +// OptionSetBytes64 will also print the bytes/second +func OptionSetBytes64(maxBytes int64) Option { + return func(p *ProgressBar) { + p.config.maxBytes = maxBytes + } +} + +// OptionShowCount will also print current count out of total +func OptionShowCount() Option { + return func(p *ProgressBar) { + p.config.showIterationsCount = true + } +} + +// OptionShowIts will also print the iterations/second +func OptionShowIts() Option { + return func(p *ProgressBar) { + p.config.showIterationsPerSecond = true + } +} + +// OptionThrottle will wait the specified duration before updating again. The default +// duration is 0 seconds. +func OptionThrottle(duration time.Duration) Option { + return func(p *ProgressBar) { + p.config.throttleDuration = duration + } +} + +var defaultTheme = Theme{Saucer: "█", SaucerPadding: " ", BarStart: "|", BarEnd: "|"} + +// NewOptions constructs a new instance of ProgressBar, with any options you specify +func NewOptions(max int, options ...Option) *ProgressBar { + return NewOptions64(int64(max), options...) +} + +// NewOptions64 constructs a new instance of ProgressBar, with any options you specify +func NewOptions64(max int64, options ...Option) *ProgressBar { + b := ProgressBar{ + state: getBlankState(), + config: config{ + writer: os.Stdout, + theme: defaultTheme, + width: 40, + max: max, + throttleDuration: 0 * time.Nanosecond, + }, + } + + for _, o := range options { + o(&b) + } + + if b.config.renderWithBlankState { + b.RenderBlank() + } + + return &b +} + +func getBlankState() state { + now := time.Now() + return state{ + startTime: now, + lastShown: now, + } +} + +// New returns a new ProgressBar +// with the specified maximum +func New(max int) *ProgressBar { + return NewOptions(max) +} + +// RenderBlank renders the current bar state, you can use this to render a 0% state +func (p *ProgressBar) RenderBlank() error { + return p.render() +} + +// Reset will reset the clock that is used +// to calculate current time and the time left. +func (p *ProgressBar) Reset() { + p.lock.Lock() + defer p.lock.Unlock() + + p.state = getBlankState() +} + +// Finish will fill the bar to full +func (p *ProgressBar) Finish() error { + p.lock.Lock() + p.state.currentNum = p.config.max + p.lock.Unlock() + return p.Add(0) +} + +// Add with increase the current count on the progress bar +func (p *ProgressBar) Set(num int) error { + return p.Set64(int64(num)) +} + +// Add with increase the current count on the progress bar +func (p *ProgressBar) Set64(num int64) error { + p.lock.Lock() + defer p.lock.Unlock() + + if p.config.max == 0 { + return errors.New("max must be greater than 0") + } + p.state.currentNum = num + percent := float64(p.state.currentNum) / float64(p.config.max) + p.state.currentSaucerSize = int(percent * float64(p.config.width)) + p.state.currentPercent = int(percent * 100) + updateBar := p.state.currentPercent != p.state.lastPercent && p.state.currentPercent > 0 + + p.state.currentBytes = float64(percent) * float64(p.config.maxBytes) + p.state.lastPercent = p.state.currentPercent + if p.state.currentNum > p.config.max { + return errors.New("current number exceeds max") + } + + // always update if show bytes/second or its/second + if updateBar || p.config.showIterationsPerSecond || p.config.maxBytes > 0 { + return p.render() + } + + return nil +} + +func (p *ProgressBar) Add(num int) error { + return p.Add64(int64(num)) +} + +func (p *ProgressBar) Add64(num int64) error { + return p.Set64(p.state.currentNum + num) +} + +// Clear erases the progress bar from the current line +func (p *ProgressBar) Clear() error { + return clearProgressBar(p.config, p.state) +} + +// Describe will change the description shown before the progress, which +// can be changed on the fly (as for a slow running process). +func (p *ProgressBar) Describe(description string) { + p.config.description = description +} + +// render renders the progress bar, updating the maximum +// rendered line width. this function is not thread-safe, +// so it must be called with an acquired lock. +func (p *ProgressBar) render() error { + // make sure that the rendering is not happening too quickly + // but always show if the currentNum reaches the max + if time.Since(p.state.lastShown).Nanoseconds() < p.config.throttleDuration.Nanoseconds() && + p.state.currentNum < p.config.max { + return nil + } + + // first, clear the existing progress bar + err := clearProgressBar(p.config, p.state) + if err != nil { + return err + } + + // then, re-render the current progress bar + w, err := renderProgressBar(p.config, p.state) + if err != nil { + return err + } + + if w > p.state.maxLineWidth { + p.state.maxLineWidth = w + } + + p.state.lastShown = time.Now() + + return nil +} + +// State returns the current state +func (p *ProgressBar) State() State { + p.lock.Lock() + defer p.lock.Unlock() + s := State{} + s.CurrentPercent = float64(p.state.currentNum) / float64(p.config.max) + s.CurrentBytes = p.state.currentBytes + s.MaxBytes = p.config.maxBytes + s.SecondsSince = time.Since(p.state.startTime).Seconds() + if p.state.currentNum > 0 { + s.SecondsLeft = s.SecondsSince / float64(p.state.currentNum) * (float64(p.config.max) - float64(p.state.currentNum)) + } + s.KBsPerSecond = float64(p.state.currentBytes) / 1000.0 / s.SecondsSince + return s +} + +// regex matching ansi escape codes +var ansiRegex = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) + +func renderProgressBar(c config, s state) (int, error) { + var leftTime float64 + if s.currentNum > 0 { + leftTime = time.Since(s.startTime).Seconds() / float64(s.currentNum) * (float64(c.max) - float64(s.currentNum)) + } + + var saucer string + if s.currentSaucerSize > 0 { + saucer = strings.Repeat(c.theme.Saucer, s.currentSaucerSize-1) + saucerHead := c.theme.SaucerHead + if saucerHead == "" || s.currentSaucerSize == c.width { + // use the saucer for the saucer head if it hasn't been set + // to preserve backwards compatibility + saucerHead = c.theme.Saucer + } + saucer += saucerHead + } + + // add on bytes string if max bytes option was set + kbPerSecond := float64(s.currentBytes) / 1000.0 / time.Since(s.startTime).Seconds() + bytesString := "" + if kbPerSecond > 1000.0 { + bytesString = fmt.Sprintf("(%2.1f MB/s)", kbPerSecond/1000.0) + } else if kbPerSecond > 0 { + bytesString = fmt.Sprintf("(%2.1f kB/s)", kbPerSecond) + } + + if c.showIterationsPerSecond && !c.showIterationsCount { + // replace bytesString if used + bytesString = fmt.Sprintf("(%2.0f it/s)", float64(s.currentNum)/time.Since(s.startTime).Seconds()) + } else if !c.showIterationsPerSecond && c.showIterationsCount { + bytesString = fmt.Sprintf("(%d/%d)", s.currentNum, c.max) + } else if c.showIterationsPerSecond && c.showIterationsCount { + bytesString = fmt.Sprintf("(%d/%d, %2.0f it/s)", s.currentNum, c.max, float64(s.currentNum)/time.Since(s.startTime).Seconds()) + } + + str := fmt.Sprintf("\r%s%4d%% %s%s%s%s %s [%s:%s]", + c.description, + s.currentPercent, + c.theme.BarStart, + saucer, + strings.Repeat(c.theme.SaucerPadding, c.width-s.currentSaucerSize), + c.theme.BarEnd, + bytesString, + (time.Duration(time.Since(s.startTime).Seconds()) * time.Second).String(), + (time.Duration(leftTime) * time.Second).String(), + ) + + if c.colorCodes { + // convert any color codes in the progress bar into the respective ANSI codes + str = colorstring.Color(str) + } + + // the width of the string, if printed to the console + // does not include the carriage return character + cleanString := strings.Replace(str, "\r", "", -1) + + if c.colorCodes { + // the ANSI codes for the colors do not take up space in the console output, + // so they do not count towards the output string width + cleanString = ansiRegex.ReplaceAllString(cleanString, "") + } + + // get the amount of runes in the string instead of the + // character count of the string, as some runes span multiple characters. + // see https://stackoverflow.com/a/12668840/2733724 + stringWidth := len([]rune(cleanString)) + + return stringWidth, writeString(c, str) +} + +func clearProgressBar(c config, s state) error { + // fill the current line with enough spaces + // to overwrite the progress bar and jump + // back to the beginning of the line + str := fmt.Sprintf("\r%s\r", strings.Repeat(" ", s.maxLineWidth)) + return writeString(c, str) +} + +func writeString(c config, str string) error { + if _, err := io.WriteString(c.writer, str); err != nil { + return err + } + + if f, ok := c.writer.(*os.File); ok { + // ignore any errors in Sync(), as stdout + // can't be synced on some operating systems + // like Debian 9 (Stretch) + f.Sync() + } + + return nil +} + +// Reader is the progressbar io.Reader struct +type Reader struct { + io.Reader + bar *ProgressBar +} + +func (r *Reader) Read(p []byte) (n int, err error) { + n, err = r.Reader.Read(p) + r.bar.Add(n) + return +} + +// Close the reader when it implements io.Closer +func (r *Reader) Close() (err error) { + if closer, ok := r.Reader.(io.Closer); ok { + return closer.Close() + } + return +} + +// Write implement io.Writer +func (p *ProgressBar) Write(b []byte) (n int, err error) { + n = len(b) + p.Add(n) + return +} + +// Read implement io.Reader +func (p *ProgressBar) Read(b []byte) (n int, err error) { + n = len(b) + p.Add(n) + return +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0d70415..cc5e846 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,9 +1,13 @@ # github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f github.com/kirsle/configdir +# github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db +github.com/mitchellh/colorstring # github.com/rs/zerolog v1.13.0 github.com/rs/zerolog github.com/rs/zerolog/internal/cbor github.com/rs/zerolog/internal/json +# github.com/schollz/progressbar/v2 v2.10.0 +github.com/schollz/progressbar/v2 # gitlab.com/pztrn/flagger v0.0.0-20190122123836-d429d7149cc9 gitlab.com/pztrn/flagger # gitlab.com/pztrn/go-uuid v0.0.0-20190208164458-d6cc46783d2b