hdkv/uploader_tools
1
Fork 0
Code Releases Activity

Add subforums support, fix infinite login error

Browse Source
This commit is contained in:
Valdos Sine 2012-04-07 17:42:32 +04:00 committed by Vladimir Hodakov
parent 762b36c0f2
commit e4e95f9c80
1 changed files with 24 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
massdl.sh
View File

@ -12,6 +12,7 @@ UNIXTIME=`date +%s`
TMPDIR=`mktemp -d` TMPDIR=`mktemp -d`
CURDIR=`pwd` CURDIR=`pwd`
LOG_FILE="$UNIXTIME-massdl.log" LOG_FILE="$UNIXTIME-massdl.log"
SUBFORUMSDL=0
wget_func () { wget_func () {
# Wget helper, which makes many wget usages more stable # Wget helper, which makes many wget usages more stable
@ -79,11 +80,12 @@ FORUMNAME=`cat $TMPDIR/page0.html | iconv -f cp1251 -t utf-8 | grep maintitle |
echo "Скачиваем форум \"${FORUMNAME}\"..." echo "Скачиваем форум \"${FORUMNAME}\"..."
ITERATOR=0 ITERATOR=0
THREADS=0 THREADS=0
# Downloading all pages # Downloading all pages
while [ `cat $TMPDIR/page$ITERATOR.html | iconv -f cp1251 -t utf-8 | grep "След."| grep 'start' | wc -l` == "2" ] while [ `cat $TMPDIR/page$ITERATOR.html | iconv -f cp1251 -t utf-8 | grep "След."| grep 'start' | wc -l` == "2" ]
do do
iconv -f cp1251 -t utf-8 $TMPDIR/page$ITERATOR.html > $TMPDIR/page$ITERATOR.utf8.html iconv -f cp1251 -t utf-8 $TMPDIR/page$ITERATOR.html > $TMPDIR/page$ITERATOR.utf8.html
cat $TMPDIR/page$ITERATOR.utf8.html | grep topictitle | sed 's/.*<a href=\([^>]*\).*/\1/' | cut -d '"' -f 2 >> $TMPDIR/alllinkz.txt cat $TMPDIR/page$ITERATOR.utf8.html | grep "DL:" | grep topictitle | sed 's/.*<a href=\([^>]*\).*/\1/' | cut -d '"' -f 2 >> $TMPDIR/alllinkz.txt
THREADS=$(($THREADS + 50)) THREADS=$(($THREADS + 50))
ITERATOR=$(($ITERATOR + 1)) ITERATOR=$(($ITERATOR + 1))
START=$(($ITERATOR * 50)) START=$(($ITERATOR * 50))
@ -92,10 +94,24 @@ do
done done
# Last loop ending :) # Last loop ending :)
iconv -f cp1251 -t utf-8 $TMPDIR/page$ITERATOR.html > $TMPDIR/page$ITERATOR.utf8.html iconv -f cp1251 -t utf-8 $TMPDIR/page$ITERATOR.html > $TMPDIR/page$ITERATOR.utf8.html
cat $TMPDIR/page$ITERATOR.utf8.html | grep topictitle | sed 's/.*<a href=\([^>]*\).*/\1/' | cut -d '"' -f 2 >> $TMPDIR/alllinkz.txt cat $TMPDIR/page$ITERATOR.utf8.html | grep "DL:" | grep topictitle | sed 's/.*<a href=\([^>]*\).*/\1/' | cut -d '"' -f 2 >> $TMPDIR/alllinkz.txt
# Detecting subforums
cat $TMPDIR/page0.utf8.html | grep viewforum | grep forumlink | grep -v "images" | cut -d '"' -f 14 | cut -d '=' -f 2 >> $TMPDIR/subforumz.txt
if [[ `cat $TMPDIR/subforumz.txt | wc -l` -gt 0 ]] ; then
echo "Выявлены подфорумы!"
while read subline
do
echo "Добавляем в очередь закачки форум $subline..."
echo "${0} ${1} $subline" >> $TMPDIR/$UNIXTIME-subforumsdl.sh
done < $TMPDIR/subforumz.txt
chmod +x $TMPDIR/$UNIXTIME-subforumsdl.sh
SUBFORUMSDL=1
fi
# You can use it as statistic :) # You can use it as statistic :)
THREADS=$(($THREADS + `cat $TMPDIR/page$ITERATOR.html | grep topictitle | wc -l`)) THREADS=`cat $TMPDIR/alllinkz.txt | wc -l`
echo -ne "Выявлено $THREADS топиков, содержащих торренты! Скачиваем...\n" echo -ne "Выявлено $THREADS топиков, содержащих торренты! Скачиваем...\n"
# And this is what we need -- getting threads and torrents! # And this is what we need -- getting threads and torrents!
# Torrents will be saved in folders like 100, 200, 300... Every folder contains # Torrents will be saved in folders like 100, 200, 300... Every folder contains
@ -172,5 +188,10 @@ do
popd >> /dev/null popd >> /dev/null
done < $TMPDIR/alllinkz.txt done < $TMPDIR/alllinkz.txt
# Downloading subforums
if [[ "x$SUBFORUMSDL" == "x1" ]] ; then
$TMPDIR/$UNIXTIME-subforumsdl.sh
fi
# Cleaning up... # Cleaning up...
rm -r $TMPDIR rm -r $TMPDIR