Initial commit
This commit is contained in:
commit
4672299afb
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
example/config.yaml
|
||||||
|
vendor/*
|
51
Gopkg.lock
generated
Normal file
51
Gopkg.lock
generated
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||||
|
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/sirupsen/logrus"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "f006c2ac4710855cf0f916dd6b77acf6b048dc6e"
|
||||||
|
version = "v1.0.3"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
branch = "master"
|
||||||
|
name = "golang.org/x/crypto"
|
||||||
|
packages = ["ssh/terminal"]
|
||||||
|
revision = "2509b142fb2b797aa7587dad548f113b2c0f20ce"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
branch = "master"
|
||||||
|
name = "golang.org/x/net"
|
||||||
|
packages = ["html","html/atom"]
|
||||||
|
revision = "7b572d500b0702f1ea30a47083ce2e1e5c61edba"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
branch = "master"
|
||||||
|
name = "golang.org/x/sys"
|
||||||
|
packages = ["unix","windows"]
|
||||||
|
revision = "164713f0dfcec4e80be8b53e1f0811f5f0d84578"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
branch = "v2"
|
||||||
|
name = "gopkg.in/yaml.v2"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
branch = "master"
|
||||||
|
name = "lab.pztrn.name/golibs/flagger"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "b1abee64b9076d189411e666a4c141176bc007f7"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
branch = "master"
|
||||||
|
name = "lab.pztrn.name/golibs/mogrus"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "bdf5f86147cf30852a0e7d03613ca71f6851e747"
|
||||||
|
|
||||||
|
[solve-meta]
|
||||||
|
analyzer-name = "dep"
|
||||||
|
analyzer-version = 1
|
||||||
|
inputs-digest = "0168622020563ff2d9261221f045328fa18a0961e7a613a5a5a4a212c24f2b51"
|
||||||
|
solver-name = "gps-cdcl"
|
||||||
|
solver-version = 1
|
34
Gopkg.toml
Normal file
34
Gopkg.toml
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
|
||||||
|
# Gopkg.toml example
|
||||||
|
#
|
||||||
|
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
|
||||||
|
# for detailed Gopkg.toml documentation.
|
||||||
|
#
|
||||||
|
# required = ["github.com/user/thing/cmd/thing"]
|
||||||
|
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
|
||||||
|
#
|
||||||
|
# [[constraint]]
|
||||||
|
# name = "github.com/user/project"
|
||||||
|
# version = "1.0.0"
|
||||||
|
#
|
||||||
|
# [[constraint]]
|
||||||
|
# name = "github.com/user/project2"
|
||||||
|
# branch = "dev"
|
||||||
|
# source = "github.com/myfork/project2"
|
||||||
|
#
|
||||||
|
# [[override]]
|
||||||
|
# name = "github.com/x/y"
|
||||||
|
# version = "2.4.0"
|
||||||
|
|
||||||
|
|
||||||
|
[[constraint]]
|
||||||
|
branch = "v2"
|
||||||
|
name = "gopkg.in/yaml.v2"
|
||||||
|
|
||||||
|
[[constraint]]
|
||||||
|
branch = "master"
|
||||||
|
name = "lab.pztrn.name/golibs/mogrus"
|
||||||
|
|
||||||
|
[[constraint]]
|
||||||
|
branch = "master"
|
||||||
|
name = "lab.pztrn.name/golibs/flagger"
|
7
MIT-LICENSE
Normal file
7
MIT-LICENSE
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
Copyright 2017 Vladimir "fat0troll" Hodakov.
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
21
README.md
Normal file
21
README.md
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
# 8th Fetcher — Парсер Восьмого ветра
|
||||||
|
|
||||||
|
## Godville game profiles parser
|
||||||
|
|
||||||
|
Dependencies ruled by [dep](https://github.com/golang/dep).
|
||||||
|
|
||||||
|
This parser designed to be companion of [8th Wind](https://wind8.ru) site engine. This program parses profiles and sends them to API endpoint of wind8 site engine.
|
||||||
|
|
||||||
|
You can use this parser for your own purposes, but you will do it at your own risk. I'm not responsible for anyone who violates Godville game rules using this software.
|
||||||
|
|
||||||
|
This software created for demonstation purposes only.
|
||||||
|
|
||||||
|
## Парсер профилей игры Годвилль
|
||||||
|
|
||||||
|
Зависимости управляются при помощи [dep](https://github.com/golang/dep).
|
||||||
|
|
||||||
|
Этот парсер создан для работы с сайтом [Восьмой ветер](https://wind8.ru). Приложение ``wind8_fetcher`` парсит профили Годвилля и отдаёт данные по API движку сайта.
|
||||||
|
|
||||||
|
Вы можете использовать эту программу для своих личных задач, но делайте это на свой страх и риск. Я не несу ответственности за каждого, кто нарушит правиля Годвилля с помощью этой программы.
|
||||||
|
|
||||||
|
Данная программа создана исключительно в ознакомительных целях.
|
23
api/exported.go
Normal file
23
api/exported.go
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
// local
|
||||||
|
"lab.pztrn.name/fat0troll/wind8_fetcher/lib/appcontext"
|
||||||
|
|
||||||
|
// actions
|
||||||
|
"lab.pztrn.name/fat0troll/wind8_fetcher/api/fetch_request"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
c *appcontext.Context
|
||||||
|
)
|
||||||
|
|
||||||
|
// Initialize prepares API endpoints to initialization
|
||||||
|
func Initialize(ac *appcontext.Context) {
|
||||||
|
c = ac
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitializeEndpoints initializes API endpoints
|
||||||
|
func InitializeEndpoints() {
|
||||||
|
fetchrequest.Initialize(c)
|
||||||
|
}
|
18
api/fetch_request/exported.go
Normal file
18
api/fetch_request/exported.go
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
package fetchrequest
|
||||||
|
|
||||||
|
import (
|
||||||
|
// local
|
||||||
|
"lab.pztrn.name/fat0troll/wind8_fetcher/lib/appcontext"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
c *appcontext.Context
|
||||||
|
)
|
||||||
|
|
||||||
|
// Initialize is an initialization function for call request handler
|
||||||
|
func Initialize(ac *appcontext.Context) {
|
||||||
|
c = ac
|
||||||
|
c.Log.Info("Initializing action for /fetch route...")
|
||||||
|
|
||||||
|
c.HTTPServerMux.HandleFunc("/fetch", HandleFetchRequest)
|
||||||
|
}
|
403
api/fetch_request/parser.go
Normal file
403
api/fetch_request/parser.go
Normal file
@ -0,0 +1,403 @@
|
|||||||
|
package fetchrequest
|
||||||
|
|
||||||
|
import (
|
||||||
|
// stdlib
|
||||||
|
"bytes"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
// 3rd-party
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
)
|
||||||
|
|
||||||
|
func getChronicle(profilePageTokenizer *html.Tokenizer) string {
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
chronicle := ""
|
||||||
|
for {
|
||||||
|
tt := profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if tt == html.EndTagToken && txt.Data == "div" {
|
||||||
|
break
|
||||||
|
} else {
|
||||||
|
chronicle += txt.String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return chronicle
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSimpleString(profilePageTokenizer *html.Tokenizer, item string, data map[string]string) map[string]string {
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if data[item] == "" {
|
||||||
|
data[item] = txt.Data
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSkill(profilePageTokenizer *html.Tokenizer, data map[string]string) map[string]string {
|
||||||
|
numbersRegexp := regexp.MustCompile("[0-9]+")
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
possiblySkillName := txt.Data
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt = profilePageTokenizer.Token()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt = profilePageTokenizer.Token()
|
||||||
|
if strings.Contains(txt.Data, "уровня") {
|
||||||
|
label := strconv.Itoa(int(len(data) / 2))
|
||||||
|
labelLvl := strconv.Itoa(int(len(data))/2) + "_level"
|
||||||
|
data[label] = possiblySkillName
|
||||||
|
data[labelLvl] = strings.Join(numbersRegexp.FindAllString(txt.Data, -1), "")
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func getPantheon(profilePageTokenizer *html.Tokenizer, name string, pantheons map[string]string) map[string]string {
|
||||||
|
numbersRegexp := regexp.MustCompile("[0-9]+")
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
pantheons[name] = strings.Join(numbersRegexp.FindAllString(txt.Data, -1), "")
|
||||||
|
return pantheons
|
||||||
|
}
|
||||||
|
|
||||||
|
func getWeapon(profilePageTokenizer *html.Tokenizer, weaponPrefix string, data map[string]string) map[string]string {
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if data[weaponPrefix] == "" {
|
||||||
|
data[weaponPrefix] = txt.Data
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt = profilePageTokenizer.Token()
|
||||||
|
data[weaponPrefix+"_stat"] = txt.Data
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseHTML(data []byte) (map[string]string, map[string]string, map[string]string, bool) {
|
||||||
|
c.Log.Info("Profile parsing started...")
|
||||||
|
profileHTML := bytes.NewReader(data)
|
||||||
|
numbersRegexp := regexp.MustCompile("[0-9]+")
|
||||||
|
|
||||||
|
profilePageTokenizer := html.NewTokenizer(profileHTML)
|
||||||
|
profile := make(map[string]string)
|
||||||
|
skills := make(map[string]string)
|
||||||
|
pantheons := make(map[string]string)
|
||||||
|
for {
|
||||||
|
tt := profilePageTokenizer.Next()
|
||||||
|
switch {
|
||||||
|
case tt == html.ErrorToken:
|
||||||
|
if profile["god_name"] == "Первая полоса" {
|
||||||
|
return profile, skills, pantheons, false
|
||||||
|
} else {
|
||||||
|
return profile, skills, pantheons, true
|
||||||
|
}
|
||||||
|
case tt == html.StartTagToken:
|
||||||
|
t := profilePageTokenizer.Token()
|
||||||
|
switch {
|
||||||
|
case t.Data == "h2":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["god_name"] == "" {
|
||||||
|
profile["god_name"] = strings.Join(strings.Fields(txt.Data)[:], " ")
|
||||||
|
}
|
||||||
|
case t.Data == "h3":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["hero_name"] == "" {
|
||||||
|
profile["hero_name"] = strings.Join(strings.Fields(txt.Data)[:], " ")
|
||||||
|
}
|
||||||
|
case t.Data == "p":
|
||||||
|
for _, attr := range t.Attr {
|
||||||
|
if attr.Key == "class" {
|
||||||
|
switch {
|
||||||
|
case attr.Val == "level":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["level"] == "" {
|
||||||
|
profile["level"] = strings.Split(txt.Data, "-")[0]
|
||||||
|
}
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt = profilePageTokenizer.Token()
|
||||||
|
if strings.Contains(txt.Data, "торговец") {
|
||||||
|
if profile["shop_level"] == "" {
|
||||||
|
profile["shop_level"] = strings.Join(numbersRegexp.FindAllString(txt.Data, -1), "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case attr.Val == "motto":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["motto"] == "" {
|
||||||
|
profile["motto"] = strings.Join(strings.Fields(txt.Data)[:], " ")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case t.Data == "td":
|
||||||
|
for _, attr := range t.Attr {
|
||||||
|
if attr.Key == "class" {
|
||||||
|
switch {
|
||||||
|
case attr.Val == "label":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
lbl := profilePageTokenizer.Token()
|
||||||
|
switch {
|
||||||
|
case lbl.Data == "Возраст":
|
||||||
|
profile = getSimpleString(profilePageTokenizer, "age", profile)
|
||||||
|
case lbl.Data == "Характер":
|
||||||
|
profile = getSimpleString(profilePageTokenizer, "personality", profile)
|
||||||
|
case lbl.Data == "Гильдия":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["guild"] == "" {
|
||||||
|
profile["guild"] = strings.Join(strings.Fields(txt.Data)[:], " ")
|
||||||
|
}
|
||||||
|
case lbl.Data == "Смертей":
|
||||||
|
profile = getSimpleString(profilePageTokenizer, "deaths", profile)
|
||||||
|
case lbl.Data == "Побед / Поражений":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
datas := strings.Join(strings.Fields(txt.Data)[:], " ")
|
||||||
|
if profile["arena_wins"] == "" {
|
||||||
|
profile["arena_wins"] = strings.Split(string(datas), " / ")[0]
|
||||||
|
profile["arena_loss"] = strings.Split(string(datas), " / ")[1]
|
||||||
|
}
|
||||||
|
case lbl.Data == "Храм достроен":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["bricks"] == "" {
|
||||||
|
profile["church_done"] = txt.Data
|
||||||
|
profile["bricks"] = "1000"
|
||||||
|
}
|
||||||
|
case lbl.Data == "Кирпичей для храма":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["bricks"] == "" {
|
||||||
|
profile["church_done"] = ""
|
||||||
|
profile["bricks"] = strings.Join(numbersRegexp.FindAllString(txt.Data, -1), "")
|
||||||
|
}
|
||||||
|
case lbl.Data == "Дерева для ковчега":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["woods"] == "" {
|
||||||
|
profile["boat_done"] = ""
|
||||||
|
profile["woods"] = strings.Join(numbersRegexp.FindAllString(txt.Data, -1), "")
|
||||||
|
}
|
||||||
|
case lbl.Data == "Ковчег достроен":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["woods"] == "" {
|
||||||
|
profile["boat_done"] = strings.Split(txt.Data, " ")[0]
|
||||||
|
profile["woods"] = strings.Join(numbersRegexp.FindAllString(strings.Split(txt.Data, " ")[1], -1), "")
|
||||||
|
}
|
||||||
|
case lbl.Data == "Твари по паре":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["beasts_male"] == "" {
|
||||||
|
profile["beasts_done"] = ""
|
||||||
|
profile["beasts_male"] = strings.Join(numbersRegexp.FindAllString(strings.Fields(txt.Data)[0], -1), "")
|
||||||
|
profile["beasts_female"] = strings.Join(numbersRegexp.FindAllString(strings.Fields(txt.Data)[1], -1), "")
|
||||||
|
profile["beasts_pairs"] = strings.Join(numbersRegexp.FindAllString(strings.Split(strings.Split(txt.Data, " ")[1], "ж")[0], -1), "")
|
||||||
|
}
|
||||||
|
case lbl.Data == "Твари собраны":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["beasts_male"] == "" {
|
||||||
|
profile["beasts_done"] = txt.Data
|
||||||
|
profile["beasts_male"] = "1000"
|
||||||
|
profile["beasts_female"] = "1000"
|
||||||
|
profile["beasts_pairs"] = "1000"
|
||||||
|
}
|
||||||
|
case lbl.Data == "Сбережения":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["pension"] == "" {
|
||||||
|
profile["pension"] = strings.Join(numbersRegexp.FindAllString(strings.Fields(txt.Data)[0], -1), "")
|
||||||
|
}
|
||||||
|
case lbl.Data == "Лавка":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["pension"] == "" {
|
||||||
|
profile["pension"] = "30000"
|
||||||
|
profile["shop"] = txt.Data
|
||||||
|
}
|
||||||
|
case lbl.Data == "Питомец":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["pet_name"] == "" {
|
||||||
|
profile["pet_type"] = txt.Data
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt = profilePageTokenizer.Token()
|
||||||
|
profile["pet_name"] = strings.Fields(txt.Data)[0]
|
||||||
|
if strings.Contains(txt.Data, "уровня") {
|
||||||
|
profile["pet_level"] = strings.Join(numbersRegexp.FindAllString(strings.Fields(txt.Data)[1], -1), "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case lbl.Data == "Оружие":
|
||||||
|
profile = getWeapon(profilePageTokenizer, "weapon", profile)
|
||||||
|
case lbl.Data == "Щит":
|
||||||
|
profile = getWeapon(profilePageTokenizer, "coat", profile)
|
||||||
|
case lbl.Data == "Голова":
|
||||||
|
profile = getWeapon(profilePageTokenizer, "head", profile)
|
||||||
|
case lbl.Data == "Тело":
|
||||||
|
profile = getWeapon(profilePageTokenizer, "body", profile)
|
||||||
|
case lbl.Data == "Руки":
|
||||||
|
profile = getWeapon(profilePageTokenizer, "hands", profile)
|
||||||
|
case lbl.Data == "Ноги":
|
||||||
|
profile = getWeapon(profilePageTokenizer, "legs", profile)
|
||||||
|
case lbl.Data == "Талисман":
|
||||||
|
profile = getWeapon(profilePageTokenizer, "talisman", profile)
|
||||||
|
}
|
||||||
|
case attr.Val == "name":
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
nm := profilePageTokenizer.Token()
|
||||||
|
switch {
|
||||||
|
case nm.Data == "Благодарности":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "gratitude", pantheons)
|
||||||
|
case nm.Data == "Мощи":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "might", pantheons)
|
||||||
|
case nm.Data == "Храмовничества":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "templehood", pantheons)
|
||||||
|
case nm.Data == "Гладиаторства":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "gladiatorship", pantheons)
|
||||||
|
case nm.Data == "Сказаний":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "storytelling", pantheons)
|
||||||
|
case nm.Data == "Поддержки":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "donation", pantheons)
|
||||||
|
case nm.Data == "Мастерства":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "mastery", pantheons)
|
||||||
|
case nm.Data == "Строительства":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "construction", pantheons)
|
||||||
|
case nm.Data == "Звероводства":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "taming", pantheons)
|
||||||
|
case nm.Data == "Живучести":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "survival", pantheons)
|
||||||
|
case nm.Data == "Зажиточности":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "savings", pantheons)
|
||||||
|
case nm.Data == "Величия":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "glory", pantheons)
|
||||||
|
case nm.Data == "Созидания":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "creation", pantheons)
|
||||||
|
case nm.Data == "Разрушения":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "destruction", pantheons)
|
||||||
|
case nm.Data == "Плотничества":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "wood", pantheons)
|
||||||
|
case nm.Data == "Отлова":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "pairs", pantheons)
|
||||||
|
case nm.Data == "Соперничества":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "duelers", pantheons)
|
||||||
|
case nm.Data == "Солидарности":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "unity", pantheons)
|
||||||
|
case nm.Data == "Влиятельности":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "popularity", pantheons)
|
||||||
|
case nm.Data == "Воинственности":
|
||||||
|
pantheons = getPantheon(profilePageTokenizer, "duelery", pantheons)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case t.Data == "div":
|
||||||
|
for _, attr := range t.Attr {
|
||||||
|
switch {
|
||||||
|
case attr.Key == "class":
|
||||||
|
if attr.Val == "guild_status" {
|
||||||
|
profilePageTokenizer.Next()
|
||||||
|
txt := profilePageTokenizer.Token()
|
||||||
|
if profile["guild_status"] == "" {
|
||||||
|
profile["guild_status"] = strings.Split(strings.Split(txt.Data, "(")[1], ")")[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case attr.Key == "id":
|
||||||
|
if attr.Val == "post_content" {
|
||||||
|
profile["chronicle"] = getChronicle(profilePageTokenizer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case t.Data == "li":
|
||||||
|
skills = getSkill(profilePageTokenizer, skills)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func startParsing(godName string) (map[string]string, map[string]string, map[string]string, string) {
|
||||||
|
profile := make(map[string]string)
|
||||||
|
skills := make(map[string]string)
|
||||||
|
pantheons := make(map[string]string)
|
||||||
|
site := "https://godville.net"
|
||||||
|
client := http.Client{}
|
||||||
|
|
||||||
|
resp, err := client.Get(site + "/gods/" + godName)
|
||||||
|
if err != nil {
|
||||||
|
c.Log.Errorln(err)
|
||||||
|
return profile, skills, pantheons, "error"
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := ioutil.ReadAll(resp.Body)
|
||||||
|
resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
c.Log.Errorln(err)
|
||||||
|
return profile, skills, pantheons, "error"
|
||||||
|
}
|
||||||
|
|
||||||
|
profile, skills, pantheons, ok := parseHTML(data)
|
||||||
|
if !ok {
|
||||||
|
return profile, skills, pantheons, "error"
|
||||||
|
}
|
||||||
|
|
||||||
|
return profile, skills, pantheons, "success"
|
||||||
|
}
|
62
api/fetch_request/request.go
Normal file
62
api/fetch_request/request.go
Normal file
@ -0,0 +1,62 @@
|
|||||||
|
package fetchrequest
|
||||||
|
|
||||||
|
import (
|
||||||
|
// stdlib
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
/*
|
||||||
|
HandleFetchRequest accepts and works with request, generated by wind8 site engine
|
||||||
|
*/
|
||||||
|
func HandleFetchRequest(w http.ResponseWriter, r *http.Request) {
|
||||||
|
response := make(map[string]interface{})
|
||||||
|
responseBody := make([]byte, 0)
|
||||||
|
w.Header().Add("Content-Type", "application/json; charset=utf-8")
|
||||||
|
if r.Method == "POST" {
|
||||||
|
requestValid := true
|
||||||
|
requestBody := make(map[string]string)
|
||||||
|
decoder := json.NewDecoder(r.Body)
|
||||||
|
err := decoder.Decode(&requestBody)
|
||||||
|
if err != nil {
|
||||||
|
c.Log.Errorln(err)
|
||||||
|
requestValid = false
|
||||||
|
response["description"] = "Wrong data, must be valid JSON."
|
||||||
|
}
|
||||||
|
|
||||||
|
if requestBody["god_name"] == "" {
|
||||||
|
requestValid = false
|
||||||
|
response["description"] = "Godville god name not passed."
|
||||||
|
}
|
||||||
|
|
||||||
|
if requestValid {
|
||||||
|
profile, skills, pantheons, status := startParsing(requestBody["god_name"])
|
||||||
|
if status != "success" {
|
||||||
|
requestValid = false
|
||||||
|
response["description"] = "Parsing error"
|
||||||
|
} else {
|
||||||
|
response["profile"] = profile
|
||||||
|
response["skills"] = skills
|
||||||
|
response["pantheons"] = pantheons
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if requestValid {
|
||||||
|
response["status"] = "success"
|
||||||
|
} else {
|
||||||
|
response["status"] = "error"
|
||||||
|
response["error"] = "400"
|
||||||
|
w.WriteHeader(400)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
w.WriteHeader(404)
|
||||||
|
response["status"] = "error"
|
||||||
|
response["error_code"] = "404"
|
||||||
|
response["description"] = "We're accepting only POST requests. Sorry."
|
||||||
|
}
|
||||||
|
responseBody, err := json.Marshal(response)
|
||||||
|
if err != nil {
|
||||||
|
c.Log.Errorln(err)
|
||||||
|
}
|
||||||
|
w.Write(responseBody)
|
||||||
|
}
|
27
cmd/wind8_fetcher/wind8_fetcher.go
Normal file
27
cmd/wind8_fetcher/wind8_fetcher.go
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
// local
|
||||||
|
"lab.pztrn.name/fat0troll/wind8_fetcher/api"
|
||||||
|
"lab.pztrn.name/fat0troll/wind8_fetcher/lib/appcontext"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
c := appcontext.New()
|
||||||
|
c.Init()
|
||||||
|
c.InitializeStartupFlags()
|
||||||
|
c.StartupFlags.Parse()
|
||||||
|
|
||||||
|
configPath, err := c.StartupFlags.GetStringValue("config")
|
||||||
|
if err != nil {
|
||||||
|
c.Log.Errorln(err)
|
||||||
|
c.Log.Fatal("Can't get config file parameter from command line. Exiting.")
|
||||||
|
}
|
||||||
|
c.InitializeConfig(configPath)
|
||||||
|
|
||||||
|
c.Log.Info("Starting API endpoints...")
|
||||||
|
api.Initialize(c)
|
||||||
|
api.InitializeEndpoints()
|
||||||
|
|
||||||
|
c.StartHTTPListener()
|
||||||
|
}
|
7
example/config.yaml.example
Normal file
7
example/config.yaml.example
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
sender:
|
||||||
|
host: "localhost"
|
||||||
|
port: "8000"
|
||||||
|
signature: "your-awesome-signature"
|
||||||
|
receiver:
|
||||||
|
host: "localhost"
|
||||||
|
port: "3000"
|
74
lib/appcontext/appcontext.go
Normal file
74
lib/appcontext/appcontext.go
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
package appcontext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
// stdlib
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
// 3rd-party
|
||||||
|
"lab.pztrn.name/golibs/flagger"
|
||||||
|
"lab.pztrn.name/golibs/mogrus"
|
||||||
|
// local
|
||||||
|
"lab.pztrn.name/fat0troll/wind8_fetcher/lib/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Context is an application context struct
|
||||||
|
type Context struct {
|
||||||
|
Cfg *config.Config
|
||||||
|
HTTPServerMux *http.ServeMux
|
||||||
|
Log *mogrus.LoggerHandler
|
||||||
|
StartupFlags *flagger.Flagger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init is an initialization function for context
|
||||||
|
func (c *Context) Init() {
|
||||||
|
l := mogrus.New()
|
||||||
|
l.Initialize()
|
||||||
|
c.Log = l.CreateLogger("stdout")
|
||||||
|
c.Log.CreateOutput("stdout", os.Stdout, true)
|
||||||
|
|
||||||
|
c.Cfg = config.New()
|
||||||
|
|
||||||
|
c.StartupFlags = flagger.New(c.Log)
|
||||||
|
c.StartupFlags.Initialize()
|
||||||
|
c.HTTPServerMux = http.NewServeMux()
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitializeConfig fills config struct with data from given file
|
||||||
|
func (c *Context) InitializeConfig(configPath string) {
|
||||||
|
c.Cfg.Init(c.Log, configPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitializeStartupFlags gives information about available startup flags
|
||||||
|
func (c *Context) InitializeStartupFlags() {
|
||||||
|
configFlag := flagger.Flag{}
|
||||||
|
configFlag.Name = "config"
|
||||||
|
configFlag.Description = "Configuration file path"
|
||||||
|
configFlag.Type = "string"
|
||||||
|
configFlag.DefaultValue = "config.yaml"
|
||||||
|
err := c.StartupFlags.AddFlag(&configFlag)
|
||||||
|
if err != nil {
|
||||||
|
c.Log.Errorln(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StartHTTPListener starts HTTP server on given port
|
||||||
|
func (c *Context) StartHTTPListener() {
|
||||||
|
response := make(map[string]string)
|
||||||
|
responseBody := make([]byte, 0)
|
||||||
|
c.HTTPServerMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Add("Content-Type", "application/json; charset=utf-8")
|
||||||
|
w.WriteHeader(404)
|
||||||
|
response["status"] = "error"
|
||||||
|
response["error_coder"] = "404"
|
||||||
|
response["descirption"] = "Not found."
|
||||||
|
|
||||||
|
responseBody, _ = json.Marshal(response)
|
||||||
|
w.Write(responseBody)
|
||||||
|
})
|
||||||
|
|
||||||
|
c.Log.Info("HTTP server started at http://" + c.Cfg.HTTPListener.Host + ":" + c.Cfg.HTTPListener.Port)
|
||||||
|
err := http.ListenAndServe(c.Cfg.HTTPListener.Host+":"+c.Cfg.HTTPListener.Port, c.HTTPServerMux)
|
||||||
|
log.Fatalln(err)
|
||||||
|
}
|
11
lib/appcontext/exported.go
Normal file
11
lib/appcontext/exported.go
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
package appcontext
|
||||||
|
|
||||||
|
var (
|
||||||
|
a *Context
|
||||||
|
)
|
||||||
|
|
||||||
|
// New is a Context creation function
|
||||||
|
func New() *Context {
|
||||||
|
c := &Context{}
|
||||||
|
return c
|
||||||
|
}
|
44
lib/config/config.go
Normal file
44
lib/config/config.go
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
// stdlib
|
||||||
|
"io/ioutil"
|
||||||
|
"path/filepath"
|
||||||
|
// 3rd-party
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
|
"lab.pztrn.name/golibs/mogrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HTTPServerConfiguration handles HTTP server configuration in config file
|
||||||
|
type HTTPServerConfiguration struct {
|
||||||
|
Host string `yaml:"host"`
|
||||||
|
Port string `yaml:"port"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPClientConfiguration handles HTTP client configuration in config file
|
||||||
|
type HTTPClientConfiguration struct {
|
||||||
|
Host string `yaml:"host"`
|
||||||
|
Port string `yaml:"port"`
|
||||||
|
Signature string `yaml:"signature"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config is a struct which represents config file structure
|
||||||
|
type Config struct {
|
||||||
|
HTTPListener HTTPServerConfiguration `yaml:"receiver"`
|
||||||
|
HTTPSender HTTPClientConfiguration `yaml:"sender"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init is a configuration initializer
|
||||||
|
func (c *Config) Init(log *mogrus.LoggerHandler, configPath string) {
|
||||||
|
log.Info("Config file path: " + configPath)
|
||||||
|
fname, _ := filepath.Abs(configPath)
|
||||||
|
yamlFile, yerr := ioutil.ReadFile(fname)
|
||||||
|
if yerr != nil {
|
||||||
|
log.Fatal("Can't read config file")
|
||||||
|
}
|
||||||
|
|
||||||
|
yperr := yaml.Unmarshal(yamlFile, c)
|
||||||
|
if yperr != nil {
|
||||||
|
log.Fatal("Can't parse config file")
|
||||||
|
}
|
||||||
|
}
|
7
lib/config/exported.go
Normal file
7
lib/config/exported.go
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
// New creates new empty Config object
|
||||||
|
func New() *Config {
|
||||||
|
c := &Config{}
|
||||||
|
return c
|
||||||
|
}
|
Reference in New Issue
Block a user