59 lines
1.3 KiB
Go
59 lines
1.3 KiB
Go
// NNM-Club torrent filess mass downloader
|
||
// Created for Uploaders group
|
||
// Copyright (c) 2012-2019 Vladimir "fat0troll" Hodakov
|
||
|
||
package fetcherv1
|
||
|
||
import (
|
||
"io"
|
||
"io/ioutil"
|
||
"net/http"
|
||
"strconv"
|
||
|
||
"golang.org/x/text/encoding/charmap"
|
||
"golang.org/x/text/transform"
|
||
)
|
||
|
||
func downloadAdditionalPages() {
|
||
for i := range forumPagesLinks {
|
||
forumPage, _ := strconv.Atoi(i)
|
||
dlog.Info().Int("номер страницы", forumPage).Msg("Скачивается ещё одна страница форума")
|
||
pageFile, err := dumpForumPage("https://" + c.Config.URL + "/forum/" + forumPagesLinks[i])
|
||
if err != nil {
|
||
dlog.Fatal().Err(err).Msg("Не удалось загрузить страницу форума")
|
||
}
|
||
|
||
_ = setQuerier(pageFile, "forumPage", forumPage)
|
||
}
|
||
}
|
||
|
||
func dumpForumPage(url string) (string, error) {
|
||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
for i := range c.Cookies {
|
||
req.AddCookie(c.Cookies[i])
|
||
}
|
||
|
||
resp, err := dclient.Do(req)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
tempF, err := ioutil.TempFile("", "massdl-*")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer tempF.Close()
|
||
|
||
respInUTF8 := transform.NewReader(resp.Body, charmap.Windows1251.NewDecoder())
|
||
_, err = io.Copy(tempF, respInUTF8)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
return tempF.Name(), nil
|
||
}
|