From a599d898b02f52216f65b89860ddae45b249bdce Mon Sep 17 00:00:00 2001
From: Vladimir Hodakov <vladimir@hodakov.me>
Date: Sat, 28 Sep 2019 20:20:23 +0400
Subject: [PATCH] Add basic HTTP errors handling
---
domains/fetcher/v1/fetcher.go | 3 +++
domains/fetcher/v1/request.go | 21 ++++++++++++++++++---
domains/loginer/v1/login.go | 19 ++++++++++++++++---
3 files changed, 37 insertions(+), 6 deletions(-)
diff --git a/domains/fetcher/v1/fetcher.go b/domains/fetcher/v1/fetcher.go
index 44b2ef0..2c289a2 100644
--- a/domains/fetcher/v1/fetcher.go
+++ b/domains/fetcher/v1/fetcher.go
@@ -6,11 +6,14 @@ package fetcherv1
import (
"strconv"
+ "time"
)
func fetch(forumID int) {
startPage := "https://" + c.Config.URL + "/forum/viewforum.php?f=" + strconv.Itoa(forumID)
+ time.Sleep(5 * time.Second)
+
startPageFile, err := dumpForumPage(startPage)
if err != nil {
dlog.Error().Err(err).Msg("Не удалось получить данные с форума")
diff --git a/domains/fetcher/v1/request.go b/domains/fetcher/v1/request.go
index bc5e04f..86a3e22 100644
--- a/domains/fetcher/v1/request.go
+++ b/domains/fetcher/v1/request.go
@@ -9,6 +9,7 @@ import (
"io/ioutil"
"net/http"
"strconv"
+ "time"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/transform"
@@ -36,9 +37,23 @@ func dumpForumPage(url string) (string, error) {
req.AddCookie(c.Cookies[i])
}
- resp, err := dclient.Do(req)
- if err != nil {
- return "", err
+ var resp *http.Response
+ retryCount := 0
+ for {
+ if retryCount < 5 {
+ resp, err = dclient.Do(req)
+ if err != nil {
+ if uberDebug {
+ dlog.Debug().Err(err).Int("попытка", retryCount+1).Msg("Не удалось получить данные, пытаемся ещё раз")
+ }
+ time.Sleep(time.Second)
+ retryCount++
+ } else {
+ break
+ }
+ } else {
+ return "", err
+ }
}
defer resp.Body.Close()
diff --git a/domains/loginer/v1/login.go b/domains/loginer/v1/login.go
index 77d9528..4e9045b 100644
--- a/domains/loginer/v1/login.go
+++ b/domains/loginer/v1/login.go
@@ -10,6 +10,7 @@ import (
"net/url"
"os"
"strings"
+ "time"
)
// login function name is self-descriptive.
@@ -74,9 +75,21 @@ func obtainCookies() {
req, _ := http.NewRequest(http.MethodPost, url, strings.NewReader(formData.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
- resp, err := client.Do(req)
- if err != nil {
- dlog.Fatal().Err(err).Msg("Не удалось отправить запрос на авторизацию в NNM-Club.")
+ retryCount := 0
+ var resp *http.Response
+ var err error
+ for {
+ if retryCount == 10 {
+ dlog.Fatal().Err(err).Msg("Не удалось отправить запрос на авторизацию в NNM-Club.")
+ }
+ resp, err = client.Do(req)
+ if err != nil {
+ dlog.Debug().Err(err).Int("попытка", retryCount).Msg("Не удалось отправить запрос, попробуем ещё раз")
+ retryCount++
+ time.Sleep(2 * time.Second)
+ } else {
+ break
+ }
}
defer resp.Body.Close()