245 lines
11 KiB
Python
245 lines
11 KiB
Python
|
# *-* encoding: utf-8 *-*
|
|||
|
|
# For NNM-Club Uploaders
|
|||
|
|
# Copyright (c) 2012-2013 Valdos Sine <iam@toofat.ru>
|
|||
|
|
#
|
|||
|
|
# Usage:
|
|||
|
|
#
|
|||
|
|
# python2 ./massdl.py [user] [forum_number]
|
|||
|
|
#
|
|||
|
|
# For getting help, execute "python2 ./massdl.py" without params
|
|||
|
|
|
|||
|
|
import time, tempfile, os, sys, commands
|
|||
|
|
version = 0.61
|
|||
|
|
unixtime = int(time.time())
|
|||
|
|
cookie = tempfile.mkstemp()
|
|||
|
|
curdir = os.getcwd()
|
|||
|
|
log_file = "/%i-massdl.log" % unixtime
|
|||
|
|
# Anti-Zapret
|
|||
|
|
domain = "http://nnm-club.me"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def write_to_log(imessage):
|
|||
|
|
"""
|
|||
|
|
Note, all script messages must be formatted as:
|
|||
|
|
|
|||
|
|
[status] #status_number: message
|
|||
|
|
|
|||
|
|
where:
|
|||
|
|
|
|||
|
|
status: "I" = information, "W" - warning, "E" - error.
|
|||
|
|
status_number - error number:
|
|||
|
|
|
|||
|
|
0xx -- script environment messages
|
|||
|
|
1xx -- authentication messages
|
|||
|
|
2xx -- grabbing and parsing messages
|
|||
|
|
3xx -- torrents downloading messages
|
|||
|
|
4xx -- torrents check messages
|
|||
|
|
|
|||
|
|
Last message in log must be ended with "Stopped" word.
|
|||
|
|
|
|||
|
|
Information starts from x00, success messages from x20, warnings from x40 and errors from x66.
|
|||
|
|
"""
|
|||
|
|
log = open(curdir + log_file, 'a')
|
|||
|
|
log.write(imessage)
|
|||
|
|
log.close()
|
|||
|
|
|
|||
|
|
def cnsl_message(imessage, *args):
|
|||
|
|
if imessage == "invpyver":
|
|||
|
|
print "Ваша версия Python не поддерживается. Используйте Python 2.7!"
|
|||
|
|
write_to_log("[E] #066: версия Python не поддерживается. Завершено.\n")
|
|||
|
|
sys.exit(666)
|
|||
|
|
elif imessage == "nocurl":
|
|||
|
|
print "У вас не установлен PycURL. Обратитесь к справке вашего дистрибутива для подробностей, как его установить."
|
|||
|
|
write_to_log("[E] #067: не установлен cURL. Завершено\n")
|
|||
|
|
sys.exit(666)
|
|||
|
|
elif imessage == "help":
|
|||
|
|
print "massdl.sh — скрипт для пакетной загрузки торрент-файлов из разделов NNM-club.ru.\nИспользование:\n\n\tpython massdl.py [режим] [номер_раздела]\n\nгде [режим] это режим загрузки (от имени пользователя, указанного в конфигурации скрипта — значение user, или же от имени фрилич-пользователя — значение freeleech), а [номер_форума] — номер раздела на NNM-club.ru (например, для форума '*Nix Игры' это 316).\nПримеры:\n\n\tpython massdl.py freeleech 316 — скачивание форума '*Nix Игры' от фрилич-пользователя\n\tpython massdl.py user 332 — скачивание форума 'Русский рок' от собственного, указанного в скрипте, имени\n\nАвтор: Valdos 'fat0troll' Sine.\nСкрипт использует Python версии 2.7. При запуске убедитесь, что используете правильный интерпретатор Python!\nВерсия %s\n" % version
|
|||
|
|
sys.exit(0)
|
|||
|
|
elif imessage == "start":
|
|||
|
|
print "Выполняется скрипт %s.\nВерсия скрипта %s." % (sys.argv[0], version)
|
|||
|
|
print "Параметры командной строки: %s %s" % (sys.argv[1], sys.argv[2])
|
|||
|
|
write_to_log("[I] #020: окружение скрипта в порядке.\n")
|
|||
|
|
elif imessage == "login_ok":
|
|||
|
|
print "Вы залогинились успешно как пользователь %s." % set_username()[0]
|
|||
|
|
write_to_log("[I] #120: выполнен успешный вход на сайт. Cookie лежит по адресу %s.\n" % cookie[1])
|
|||
|
|
elif imessage == "login_failed":
|
|||
|
|
print "Не удалось залогиниться как пользователь %s. Проверьте ваши данные внутри скрипта!" % set_username()[0]
|
|||
|
|
write_to_log("[E] #166: неправильное имя пользователя или пароль. Завершено.\n")
|
|||
|
|
sys.exit(666)
|
|||
|
|
elif imessage == "parsing_started":
|
|||
|
|
print "Начинается загрузка форума..."
|
|||
|
|
elif imessage == "forum":
|
|||
|
|
print "Скачиваем форум " + args[0]
|
|||
|
|
write_to_log('[I] #220: успешно загружена стартовая страница форума "%s".\n' % args[0])
|
|||
|
|
elif imessage == "pages":
|
|||
|
|
print "Скачиваем страницу " + args[0]
|
|||
|
|
write_to_log('[I] #200: загружаем страницу ' + args[0] + '...\n')
|
|||
|
|
elif imessage == "found_topics":
|
|||
|
|
print "Скачивание страниц завершено. Найдено топиков: " + args[0] + "."
|
|||
|
|
write_to_log('[I] #221: успешно загружены все страницы форума. Найдено топиков: ' + args[0] + '.\n')
|
|||
|
|
elif imessage == "download_started":
|
|||
|
|
print "Начинаем загрузку торрентов..."
|
|||
|
|
elif imessage == "downloading_torrent":
|
|||
|
|
print "Скачиваю топик " + args[0] + "/" + args[1] + ": " + args[2] + "."
|
|||
|
|
write_to_log("[I] #300 загружаем торрент " + args[0] + " из " + args[1] + " по ссылке " + args[3] + ".\n")
|
|||
|
|
elif imessage == "done":
|
|||
|
|
print "Скрипт завершен успешно."
|
|||
|
|
write_to_log("[I] #021: работа завершена успешно. Завершено.\n")
|
|||
|
|
sys.exit(0)
|
|||
|
|
|
|||
|
|
def check_requirements():
|
|||
|
|
req_py_version = (2,7)
|
|||
|
|
unsupported_py_version = (3,0)
|
|||
|
|
cur_version = sys.version_info
|
|||
|
|
|
|||
|
|
if cur_version >= req_py_version:
|
|||
|
|
if cur_version < unsupported_py_version:
|
|||
|
|
pass;
|
|||
|
|
else:
|
|||
|
|
cnsl_message("invpyver")
|
|||
|
|
else:
|
|||
|
|
cnsl_message("invpyver")
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
import curl
|
|||
|
|
import pycurl
|
|||
|
|
except ImportError:
|
|||
|
|
cnsl_message("nocurl")
|
|||
|
|
|
|||
|
|
def check_params(params):
|
|||
|
|
"""
|
|||
|
|
Checking script parameters: if there is less or more than 2 params -- show help. If there's 2 params: check it's consistency!
|
|||
|
|
"""
|
|||
|
|
if len(params) != 3:
|
|||
|
|
cnsl_message("help")
|
|||
|
|
else:
|
|||
|
|
if params[1] in ["freeleech", "user"]:
|
|||
|
|
if str(params[2]).isdigit():
|
|||
|
|
pass;
|
|||
|
|
else:
|
|||
|
|
cnsl_message("help")
|
|||
|
|
else:
|
|||
|
|
cnsl_message("help")
|
|||
|
|
|
|||
|
|
def start_log():
|
|||
|
|
log = open(curdir + log_file, 'w')
|
|||
|
|
log.write('================================================================================\n= Скрипт скачки раздела NNM-Club для Linux и Unix-like ОС. =\n= Версия: %s =\n================================================================================\n' % version)
|
|||
|
|
log.close()
|
|||
|
|
|
|||
|
|
def set_username():
|
|||
|
|
if sys.argv[1] == "freeleech":
|
|||
|
|
username = "freeleech_user"
|
|||
|
|
password = "freeleech_password"
|
|||
|
|
else:
|
|||
|
|
username = "username"
|
|||
|
|
password = "password"
|
|||
|
|
return (username, password)
|
|||
|
|
|
|||
|
|
def connect():
|
|||
|
|
cookiefile = open(cookie[1], 'r+w')
|
|||
|
|
write_to_log("[I] #100: пытаемся войти на " + domain + "...\n")
|
|||
|
|
cookie_txt = commands.getoutput('curl -c - -d "username=' + set_username()[0] + '&password=' + set_username()[1] + '&autologin=on&login=%C2%F5%EE%E4&redirect=index.php" "' + domain + '/forum/login.php" 2>>' + curdir + log_file )
|
|||
|
|
if cookie_txt.find("Cookie") != -1:
|
|||
|
|
# If it's -1, then there is no cookie!
|
|||
|
|
cookiefile.write(cookie_txt)
|
|||
|
|
print cookiefile.read()
|
|||
|
|
cnsl_message("login_ok")
|
|||
|
|
else:
|
|||
|
|
cnsl_message("login_failed")
|
|||
|
|
sys.exit(666)
|
|||
|
|
|
|||
|
|
def iteratorium(links_array, offset):
|
|||
|
|
"""
|
|||
|
|
Links array -- array with links to topics containing torrents. Offset -- load parameter (for pages)
|
|||
|
|
"""
|
|||
|
|
cnsl_message("pages", str((offset / 50) + 1))
|
|||
|
|
forum_page = commands.getoutput('curl -b ' + cookie[1] + ' "' + domain + '/forum/viewforum.php?f=' + sys.argv[2] + '&start=' + str(offset) + '" | iconv -f cp1251 -t utf-8')
|
|||
|
|
for line in forum_page.splitlines():
|
|||
|
|
if 'viewtopic.php' in line:
|
|||
|
|
if 'DL:' in line:
|
|||
|
|
for substring in line.split('"'):
|
|||
|
|
if 'viewtopic.php' in substring:
|
|||
|
|
links_array.append(substring.split('&')[0])
|
|||
|
|
# Check for the next page
|
|||
|
|
if forum_page.find("След.") != -1:
|
|||
|
|
iteratorium(links_array, offset + 50)
|
|||
|
|
clean_array = []
|
|||
|
|
for item in links_array:
|
|||
|
|
if item not in clean_array:
|
|||
|
|
clean_array.append(item)
|
|||
|
|
return clean_array
|
|||
|
|
|
|||
|
|
def download():
|
|||
|
|
"""
|
|||
|
|
And now that's main part of the script. We need to:
|
|||
|
|
|
|||
|
|
* iterate over each other page of the forum
|
|||
|
|
* collect all forum topics
|
|||
|
|
* collect all torrents from topics
|
|||
|
|
* check the integrity of downloaded stuff (or why we want to see the log?)
|
|||
|
|
The magic begins...
|
|||
|
|
"""
|
|||
|
|
# Firstly, we need to download forum's startpage
|
|||
|
|
cnsl_message("parsing_started")
|
|||
|
|
startpage = commands.getoutput('curl -b ' + cookie[1] + ' ' + domain + '/forum/viewforum.php?f=' + sys.argv[2] + ' | iconv -f cp1251 -t utf-8')
|
|||
|
|
# Finding forum name :D
|
|||
|
|
for line in startpage.splitlines():
|
|||
|
|
if 'maintitle' in line:
|
|||
|
|
fname_raw = line
|
|||
|
|
forumname = fname_raw.split('>')[3].split('<')[0]
|
|||
|
|
cnsl_message("forum", forumname)
|
|||
|
|
# And now we start our "good iteratorium"
|
|||
|
|
topics = iteratorium([], 0)
|
|||
|
|
cnsl_message("found_topics", str(len(topics)))
|
|||
|
|
# Aaaand... the first real actions is here
|
|||
|
|
torrentsdir = "%s/%s" % (curdir, sys.argv[2])
|
|||
|
|
os.mkdir(torrentsdir)
|
|||
|
|
get_torrent(topics, torrentsdir, forumname)
|
|||
|
|
|
|||
|
|
def parse_topic(topic, forumname, topics, torrents_count, torrentsdir):
|
|||
|
|
topicpage = commands.getoutput('curl -b ' + cookie[1] + ' "' + domain + '/forum/' + topic + '" | iconv -f cp1251 -t utf-8')
|
|||
|
|
for line in topicpage.splitlines():
|
|||
|
|
if 'maintitle' in line:
|
|||
|
|
tname_raw = line
|
|||
|
|
topicname = tname_raw.split('>')[3].split('<')[0]
|
|||
|
|
for line in topicpage.splitlines():
|
|||
|
|
if 'download.php' in line:
|
|||
|
|
for lvl2 in line.split('>'):
|
|||
|
|
if 'download.php' in lvl2:
|
|||
|
|
dl_raw = lvl2
|
|||
|
|
downlink = dl_raw.split('"')[1]
|
|||
|
|
cnsl_message("downloading_torrent", str(torrents_count), str(len(topics)), topicname, (domain + "/" + topic))
|
|||
|
|
os.chdir(torrentsdir)
|
|||
|
|
if not os.path.exists("./%i00" % (torrents_count / 100)):
|
|||
|
|
os.makedirs("./%i00" % (torrents_count / 100))
|
|||
|
|
os.chdir("./%i00" % (torrents_count / 100))
|
|||
|
|
download_torrent(downlink)
|
|||
|
|
|
|||
|
|
def download_torrent(downlink):
|
|||
|
|
os.system('curl -b ' + cookie[1] + ' -O -J -L "' + domain + '/forum/' + downlink + '" 2>>' + curdir + log_file)
|
|||
|
|
|
|||
|
|
def get_torrent(topics, torrentsdir, forumname):
|
|||
|
|
cnsl_message("download_started")
|
|||
|
|
torrents_count = 0
|
|||
|
|
for topic in topics:
|
|||
|
|
torrents_count = torrents_count + 1
|
|||
|
|
parse_topic(topic, forumname, topics, torrents_count, torrentsdir)
|
|||
|
|
|
|||
|
|
def cleanup():
|
|||
|
|
os.remove(cookie[1])
|
|||
|
|
cnsl_message("done")
|
|||
|
|
|
|||
|
|
|
|||
|
|
# That's da fuckin' magic here ;-)
|
|||
|
|
|
|||
|
|
check_params(sys.argv)
|
|||
|
|
start_log()
|
|||
|
|
check_requirements()
|
|||
|
|
|
|||
|
|
import curl, pycurl
|
|||
|
|
|
|||
|
|
cnsl_message("start")
|
|||
|
|
connect()
|
|||
|
|
download()
|
|||
|
|
cleanup()
|