Vladimir Hodakov
32a2262e8c
Move domains to internal/services, remove unnecessary versioning for services. Change copyright year.
91 lines
1.9 KiB
Go
91 lines
1.9 KiB
Go
// NNM-Club torrent files mass downloader
|
||
// Created for Uploaders group
|
||
// Copyright (c) 2012-2022 Vladimir "fat0troll" Hodakov
|
||
|
||
package fetcher
|
||
|
||
import (
|
||
"io"
|
||
"io/ioutil"
|
||
"net/http"
|
||
"strconv"
|
||
"time"
|
||
|
||
"golang.org/x/text/encoding/charmap"
|
||
"golang.org/x/text/transform"
|
||
)
|
||
|
||
func downloadAdditionalPages() {
|
||
for i := range forumPagesLinks {
|
||
forumPage, _ := strconv.Atoi(i)
|
||
dlog.Info().Int("номер страницы", forumPage).Msg("Скачивается ещё одна страница форума")
|
||
|
||
pageFile, err := downloadFile("https://"+c.Config.URL+"/forum/"+forumPagesLinks[i], true)
|
||
if err != nil {
|
||
dlog.Fatal().Err(err).Msg("Не удалось загрузить страницу форума")
|
||
}
|
||
|
||
_ = setQuerier(pageFile, "forumPage", forumPage)
|
||
}
|
||
}
|
||
|
||
func downloadFile(url string, transformEncoding bool) (string, error) {
|
||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
for i := range c.Cookies {
|
||
req.AddCookie(c.Cookies[i])
|
||
}
|
||
|
||
var resp *http.Response
|
||
var retryCount int = 0
|
||
|
||
for {
|
||
if retryCount < 5 {
|
||
resp, err = dclient.Do(req)
|
||
if err != nil {
|
||
if uberDebug {
|
||
dlog.Debug().Err(err).Int("попытка", retryCount+1).Msg("Не удалось получить данные, пытаемся ещё раз")
|
||
}
|
||
|
||
time.Sleep(time.Second)
|
||
retryCount++
|
||
} else {
|
||
break
|
||
}
|
||
} else {
|
||
return "", err
|
||
}
|
||
}
|
||
|
||
defer resp.Body.Close()
|
||
|
||
tempF, err := ioutil.TempFile("", "massdl-*")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer tempF.Close()
|
||
|
||
if transformEncoding {
|
||
respInUTF8 := transform.NewReader(resp.Body, charmap.Windows1251.NewDecoder())
|
||
|
||
_, err = io.Copy(tempF, respInUTF8)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
} else {
|
||
_, err = io.Copy(tempF, resp.Body)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
}
|
||
|
||
if uberDebug {
|
||
dlog.Debug().Str("расположение", tempF.Name()).Msg("Загружен файл из сети")
|
||
}
|
||
|
||
return tempF.Name(), nil
|
||
}
|