245 lines
11 KiB
Python
245 lines
11 KiB
Python
# *-* encoding: utf-8 *-*
|
||
# For NNM-Club Uploaders
|
||
# Copyright (c) 2012-2013 Valdos Sine <iam@toofat.ru>
|
||
#
|
||
# Usage:
|
||
#
|
||
# python2 ./massdl.py [user] [forum_number]
|
||
#
|
||
# For getting help, execute "python2 ./massdl.py" without params
|
||
|
||
import time, tempfile, os, sys, commands
|
||
version = 0.61
|
||
unixtime = int(time.time())
|
||
cookie = tempfile.mkstemp()
|
||
curdir = os.getcwd()
|
||
log_file = "/%i-massdl.log" % unixtime
|
||
# Anti-Zapret
|
||
domain = "http://nnm-club.me"
|
||
|
||
|
||
def write_to_log(imessage):
|
||
"""
|
||
Note, all script messages must be formatted as:
|
||
|
||
[status] #status_number: message
|
||
|
||
where:
|
||
|
||
status: "I" = information, "W" - warning, "E" - error.
|
||
status_number - error number:
|
||
|
||
0xx -- script environment messages
|
||
1xx -- authentication messages
|
||
2xx -- grabbing and parsing messages
|
||
3xx -- torrents downloading messages
|
||
4xx -- torrents check messages
|
||
|
||
Last message in log must be ended with "Stopped" word.
|
||
|
||
Information starts from x00, success messages from x20, warnings from x40 and errors from x66.
|
||
"""
|
||
log = open(curdir + log_file, 'a')
|
||
log.write(imessage)
|
||
log.close()
|
||
|
||
def cnsl_message(imessage, *args):
|
||
if imessage == "invpyver":
|
||
print "Ваша версия Python не поддерживается. Используйте Python 2.7!"
|
||
write_to_log("[E] #066: версия Python не поддерживается. Завершено.\n")
|
||
sys.exit(666)
|
||
elif imessage == "nocurl":
|
||
print "У вас не установлен PycURL. Обратитесь к справке вашего дистрибутива для подробностей, как его установить."
|
||
write_to_log("[E] #067: не установлен cURL. Завершено\n")
|
||
sys.exit(666)
|
||
elif imessage == "help":
|
||
print "massdl.sh — скрипт для пакетной загрузки торрент-файлов из разделов NNM-club.ru.\nИспользование:\n\n\tpython massdl.py [режим] [номер_раздела]\n\nгде [режим] это режим загрузки (от имени пользователя, указанного в конфигурации скрипта — значение user, или же от имени фрилич-пользователя — значение freeleech), а [номер_форума] — номер раздела на NNM-club.ru (например, для форума '*Nix Игры' это 316).\nПримеры:\n\n\tpython massdl.py freeleech 316 — скачивание форума '*Nix Игры' от фрилич-пользователя\n\tpython massdl.py user 332 — скачивание форума 'Русский рок' от собственного, указанного в скрипте, имени\n\nАвтор: Valdos 'fat0troll' Sine.\nСкрипт использует Python версии 2.7. При запуске убедитесь, что используете правильный интерпретатор Python!\nВерсия %s\n" % version
|
||
sys.exit(0)
|
||
elif imessage == "start":
|
||
print "Выполняется скрипт %s.\nВерсия скрипта %s." % (sys.argv[0], version)
|
||
print "Параметры командной строки: %s %s" % (sys.argv[1], sys.argv[2])
|
||
write_to_log("[I] #020: окружение скрипта в порядке.\n")
|
||
elif imessage == "login_ok":
|
||
print "Вы залогинились успешно как пользователь %s." % set_username()[0]
|
||
write_to_log("[I] #120: выполнен успешный вход на сайт. Cookie лежит по адресу %s.\n" % cookie[1])
|
||
elif imessage == "login_failed":
|
||
print "Не удалось залогиниться как пользователь %s. Проверьте ваши данные внутри скрипта!" % set_username()[0]
|
||
write_to_log("[E] #166: неправильное имя пользователя или пароль. Завершено.\n")
|
||
sys.exit(666)
|
||
elif imessage == "parsing_started":
|
||
print "Начинается загрузка форума..."
|
||
elif imessage == "forum":
|
||
print "Скачиваем форум " + args[0]
|
||
write_to_log('[I] #220: успешно загружена стартовая страница форума "%s".\n' % args[0])
|
||
elif imessage == "pages":
|
||
print "Скачиваем страницу " + args[0]
|
||
write_to_log('[I] #200: загружаем страницу ' + args[0] + '...\n')
|
||
elif imessage == "found_topics":
|
||
print "Скачивание страниц завершено. Найдено топиков: " + args[0] + "."
|
||
write_to_log('[I] #221: успешно загружены все страницы форума. Найдено топиков: ' + args[0] + '.\n')
|
||
elif imessage == "download_started":
|
||
print "Начинаем загрузку торрентов..."
|
||
elif imessage == "downloading_torrent":
|
||
print "Скачиваю топик " + args[0] + "/" + args[1] + ": " + args[2] + "."
|
||
write_to_log("[I] #300 загружаем торрент " + args[0] + " из " + args[1] + " по ссылке " + args[3] + ".\n")
|
||
elif imessage == "done":
|
||
print "Скрипт завершен успешно."
|
||
write_to_log("[I] #021: работа завершена успешно. Завершено.\n")
|
||
sys.exit(0)
|
||
|
||
def check_requirements():
|
||
req_py_version = (2,7)
|
||
unsupported_py_version = (3,0)
|
||
cur_version = sys.version_info
|
||
|
||
if cur_version >= req_py_version:
|
||
if cur_version < unsupported_py_version:
|
||
pass;
|
||
else:
|
||
cnsl_message("invpyver")
|
||
else:
|
||
cnsl_message("invpyver")
|
||
|
||
try:
|
||
import curl
|
||
import pycurl
|
||
except ImportError:
|
||
cnsl_message("nocurl")
|
||
|
||
def check_params(params):
|
||
"""
|
||
Checking script parameters: if there is less or more than 2 params -- show help. If there's 2 params: check it's consistency!
|
||
"""
|
||
if len(params) != 3:
|
||
cnsl_message("help")
|
||
else:
|
||
if params[1] in ["freeleech", "user"]:
|
||
if str(params[2]).isdigit():
|
||
pass;
|
||
else:
|
||
cnsl_message("help")
|
||
else:
|
||
cnsl_message("help")
|
||
|
||
def start_log():
|
||
log = open(curdir + log_file, 'w')
|
||
log.write('================================================================================\n= Скрипт скачки раздела NNM-Club для Linux и Unix-like ОС. =\n= Версия: %s =\n================================================================================\n' % version)
|
||
log.close()
|
||
|
||
def set_username():
|
||
if sys.argv[1] == "freeleech":
|
||
username = "freeleech_user"
|
||
password = "freeleech_password"
|
||
else:
|
||
username = "username"
|
||
password = "password"
|
||
return (username, password)
|
||
|
||
def connect():
|
||
cookiefile = open(cookie[1], 'r+w')
|
||
write_to_log("[I] #100: пытаемся войти на " + domain + "...\n")
|
||
cookie_txt = commands.getoutput('curl -c - -d "username=' + set_username()[0] + '&password=' + set_username()[1] + '&autologin=on&login=%C2%F5%EE%E4&redirect=index.php" "' + domain + '/forum/login.php" 2>>' + curdir + log_file )
|
||
if cookie_txt.find("Cookie") != -1:
|
||
# If it's -1, then there is no cookie!
|
||
cookiefile.write(cookie_txt)
|
||
print cookiefile.read()
|
||
cnsl_message("login_ok")
|
||
else:
|
||
cnsl_message("login_failed")
|
||
sys.exit(666)
|
||
|
||
def iteratorium(links_array, offset):
|
||
"""
|
||
Links array -- array with links to topics containing torrents. Offset -- load parameter (for pages)
|
||
"""
|
||
cnsl_message("pages", str((offset / 50) + 1))
|
||
forum_page = commands.getoutput('curl -b ' + cookie[1] + ' "' + domain + '/forum/viewforum.php?f=' + sys.argv[2] + '&start=' + str(offset) + '" | iconv -f cp1251 -t utf-8')
|
||
for line in forum_page.splitlines():
|
||
if 'viewtopic.php' in line:
|
||
if 'DL:' in line:
|
||
for substring in line.split('"'):
|
||
if 'viewtopic.php' in substring:
|
||
links_array.append(substring.split('&')[0])
|
||
# Check for the next page
|
||
if forum_page.find("След.") != -1:
|
||
iteratorium(links_array, offset + 50)
|
||
clean_array = []
|
||
for item in links_array:
|
||
if item not in clean_array:
|
||
clean_array.append(item)
|
||
return clean_array
|
||
|
||
def download():
|
||
"""
|
||
And now that's main part of the script. We need to:
|
||
|
||
* iterate over each other page of the forum
|
||
* collect all forum topics
|
||
* collect all torrents from topics
|
||
* check the integrity of downloaded stuff (or why we want to see the log?)
|
||
The magic begins...
|
||
"""
|
||
# Firstly, we need to download forum's startpage
|
||
cnsl_message("parsing_started")
|
||
startpage = commands.getoutput('curl -b ' + cookie[1] + ' ' + domain + '/forum/viewforum.php?f=' + sys.argv[2] + ' | iconv -f cp1251 -t utf-8')
|
||
# Finding forum name :D
|
||
for line in startpage.splitlines():
|
||
if 'maintitle' in line:
|
||
fname_raw = line
|
||
forumname = fname_raw.split('>')[3].split('<')[0]
|
||
cnsl_message("forum", forumname)
|
||
# And now we start our "good iteratorium"
|
||
topics = iteratorium([], 0)
|
||
cnsl_message("found_topics", str(len(topics)))
|
||
# Aaaand... the first real actions is here
|
||
torrentsdir = "%s/%s" % (curdir, sys.argv[2])
|
||
os.mkdir(torrentsdir)
|
||
get_torrent(topics, torrentsdir, forumname)
|
||
|
||
def parse_topic(topic, forumname, topics, torrents_count, torrentsdir):
|
||
topicpage = commands.getoutput('curl -b ' + cookie[1] + ' "' + domain + '/forum/' + topic + '" | iconv -f cp1251 -t utf-8')
|
||
for line in topicpage.splitlines():
|
||
if 'maintitle' in line:
|
||
tname_raw = line
|
||
topicname = tname_raw.split('>')[3].split('<')[0]
|
||
for line in topicpage.splitlines():
|
||
if 'download.php' in line:
|
||
for lvl2 in line.split('>'):
|
||
if 'download.php' in lvl2:
|
||
dl_raw = lvl2
|
||
downlink = dl_raw.split('"')[1]
|
||
cnsl_message("downloading_torrent", str(torrents_count), str(len(topics)), topicname, (domain + "/" + topic))
|
||
os.chdir(torrentsdir)
|
||
if not os.path.exists("./%i00" % (torrents_count / 100)):
|
||
os.makedirs("./%i00" % (torrents_count / 100))
|
||
os.chdir("./%i00" % (torrents_count / 100))
|
||
download_torrent(downlink)
|
||
|
||
def download_torrent(downlink):
|
||
os.system('curl -b ' + cookie[1] + ' -O -J -L "' + domain + '/forum/' + downlink + '" 2>>' + curdir + log_file)
|
||
|
||
def get_torrent(topics, torrentsdir, forumname):
|
||
cnsl_message("download_started")
|
||
torrents_count = 0
|
||
for topic in topics:
|
||
torrents_count = torrents_count + 1
|
||
parse_topic(topic, forumname, topics, torrents_count, torrentsdir)
|
||
|
||
def cleanup():
|
||
os.remove(cookie[1])
|
||
cnsl_message("done")
|
||
|
||
|
||
# That's da fuckin' magic here ;-)
|
||
|
||
check_params(sys.argv)
|
||
start_log()
|
||
check_requirements()
|
||
|
||
import curl, pycurl
|
||
|
||
cnsl_message("start")
|
||
connect()
|
||
download()
|
||
cleanup()
|