Refactor and copyrights

Move domains to internal/services, remove unnecessary versioning for services. Change copyright year.
2021-12-20 16:53:38 +04:00
parent fd184225f1
commit 32a2262e8c
19 changed files with 60 additions and 78 deletions
--- a/internal/services/fetcher/request.go
+++ b/internal/services/fetcher/request.go
@@ -0,0 +1,90 @@
+// NNM-Club torrent files mass downloader
+// Created for Uploaders group
+// Copyright (c) 2012-2022 Vladimir "fat0troll" Hodakov
+
+package fetcher
+
+import (
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"time"
+
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/transform"
+)
+
+func downloadAdditionalPages() {
+	for i := range forumPagesLinks {
+		forumPage, _ := strconv.Atoi(i)
+		dlog.Info().Int("номер страницы", forumPage).Msg("Скачивается ещё одна страница форума")
+
+		pageFile, err := downloadFile("https://"+c.Config.URL+"/forum/"+forumPagesLinks[i], true)
+		if err != nil {
+			dlog.Fatal().Err(err).Msg("Не удалось загрузить страницу форума")
+		}
+
+		_ = setQuerier(pageFile, "forumPage", forumPage)
+	}
+}
+
+func downloadFile(url string, transformEncoding bool) (string, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	for i := range c.Cookies {
+		req.AddCookie(c.Cookies[i])
+	}
+
+	var resp *http.Response
+	var retryCount int = 0
+
+	for {
+		if retryCount < 5 {
+			resp, err = dclient.Do(req)
+			if err != nil {
+				if uberDebug {
+					dlog.Debug().Err(err).Int("попытка", retryCount+1).Msg("Не удалось получить данные, пытаемся ещё раз")
+				}
+
+				time.Sleep(time.Second)
+				retryCount++
+			} else {
+				break
+			}
+		} else {
+			return "", err
+		}
+	}
+
+	defer resp.Body.Close()
+
+	tempF, err := ioutil.TempFile("", "massdl-*")
+	if err != nil {
+		return "", err
+	}
+	defer tempF.Close()
+
+	if transformEncoding {
+		respInUTF8 := transform.NewReader(resp.Body, charmap.Windows1251.NewDecoder())
+
+		_, err = io.Copy(tempF, respInUTF8)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		_, err = io.Copy(tempF, resp.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	if uberDebug {
+		dlog.Debug().Str("расположение", tempF.Name()).Msg("Загружен файл из сети")
+	}
+
+	return tempF.Name(), nil
+}