hoarder/hoarder.sh

390 lines
10 KiB
Bash
Raw Normal View History

2015-02-06 22:20:06 -05:00
#!/bin/sh
## START CONFIGURATION
# Location where the .torrent files are stored locally
TORRENT_FILE_PATH='/home/tblyler/torrent_files'
# Location to initially download torrent data to from the remote SSH server
TORRENT_TMP_DOWNLOAD='/home/tblyler/torrents_tmp'
# Location to move the completed torrent data to from TORRENT_TMP_DOWNLOAD
TORRENT_DOWNLOAD='/home/tblyler/torrents'
# Amount of rsync processes to have running at one time
RSYNC_PROCESSES=2
# Location on the remote SSH server to copy the .torrent files to
SSH_SERVER_TORRENT_FILE_PATH='watch'
# Location on the remote SSH server where the torrent data is stored
SSH_SERVER_DOWNLOAD_PATH='files'
# Address of the remote SSH server where the torrents are downloaded
SSH_SERVER='remote.rtorrent.com'
# The username to use to login to the SSH server
SSH_USER='sshUserName'
# The XMLRPC basic HTTP authentication username
XML_USER='XMLRPCUserName'
# The XMLRPC basic HTTP authentication password
XML_PASS='XMLRPCPassword'
# The XMLRPC url
XML_URL='https://XMLRPCURL.com/XMLRPC'
## END CONFIGURATION
if ! which curl > /dev/null; then
echo 'curl must be installed'
exit 1
fi
if ! which scp > /dev/null; then
echo 'scp must be installed'
exit 1
fi
if ! which rsync > /dev/null; then
echo 'rsync must be installed'
exit 1
fi
if ! which python > /dev/null; then
if ! which python2 > /dev/null; then
echo 'python must be install'
exit 1
fi
fi
# Hacky method to create the XML for an XMLRPC request to rtorrent
xml() {
local method=$1
local args=$2
echo "<?xml version='1.0'?>
<methodCall>
<methodName>${method}</methodName>
<params>
<param>
<value><string>${args}</string></value>
</param>
</params>
</methodCall>"
}
# Returns the current entity and its content in an XML response
read_dom() {
local IFS=\>
read -d \< ENTITY CONTENT
}
# Sends an XMLRPC request to rtorrent via curl and returns its data
xml_curl() {
local method=$1
local args=$2
local xml_post=`xml "${method}" "${args}"`
local curl_command='curl -s'
if [[ "${XML_USER}" != '' ]]; then
local curl_command="${curl_command} --basic -u '${XML_USER}"
if [[ "${XML_USER}" != '' ]]; then
local curl_command="${curl_command}:${XML_PASS}"
fi
local curl_command="${curl_command}'"
fi
local curl_command="${curl_command} -d \"${xml_post}\" '${XML_URL}'"
local xml_response=$(eval "${curl_command}")
local curl_return=$?
echo "${xml_response}"
return $curl_return
}
# Gets .torrent's name from the remote rtorrent XMLRPC
get_torrent_name() {
local torrent_hash=$1
local xml_response=`xml_curl d.get_name "${torrent_hash}"`
local curl_return=$?
if [[ "${curl_return}" -ne 0 ]]; then
echo "Curl failed to get torrent name with error code ${curl_return}"
return $curl_return
fi
local torrent_name=`echo "${xml_response}" | while read_dom; do
if [[ "${ENTITY}" = "name" ]] && [[ "${CONTENT}" = "faultCode" ]]; then
local error=true
fi
if [[ ! "${error}" ]] && [[ "${ENTITY}" = "string" ]]; then
echo "${CONTENT}"
fi
done`
if [[ "${torrent_name}" = '' ]]; then
echo "${xml_response}"
return 1
else
echo "${torrent_name}"
return 0
fi
}
# Get .torrent's completion status from the remote rtorrent
get_torrent_complete() {
local torrent_hash=$1
local xml_response=`xml_curl d.get_complete "${torrent_hash}"`
local curl_return=$?
if [[ "${curl_return}" -ne 0 ]]; then
echo "Curl failed to get torrent name with error code ${curl_return}"
return ${curl_return}
fi
local torrent_completed=`echo "${xml_response}" | while read_dom; do
if [[ "${ENTITY}" = "name" ]] && [[ "${CONTENT}" = "faultCode" ]]; then
local error=true
fi
if [[ ! "${error}" ]] && [[ "${ENTITY}" = "i8" ]]; then
echo "${CONTENT}"
fi
done`
if [[ "${torrent_completed}" = '' ]]; then
echo "${xml_response}"
return 1
else
echo "${torrent_completed}"
return 0
fi
}
# Check if a .torrent is loaded on the remote rtorrent
get_torrent_added() {
local torrent_hash=$1
local xml_response=`xml_curl d.get_complete "${torrent_hash}"`
local curl_return=$?
if [[ "${curl_return}" -ne 0 ]]; then
echo "Curl failed to get torrent name with error code ${curl_return}"
return ${curl_return}
fi
local torrent_added=`echo "${xml_response}" | while read_dom; do
if [[ "${CONTENT}" = 'Could not find info-hash.' ]]; then
echo "${CONTENT}"
fi
done`
if [[ "${torrent_added}" = '' ]]; then
echo 1
else
echo 0
fi
}
# Get the info hash for a given .torrent file
get_torrent_hash() {
local torrent_file=$1
if [[ ! -f "${torrent_file}" ]]; then
return 1
fi
local python_bin='python2'
if ! which "${python_bin}" 2>&1 > /dev/null; then
local python_bin='python'
if ! which "${python_bin}" 2>&1 > /dev/null; then
return 1
fi
fi
local torrent_hash=`"${python_bin}" - << END
import hashlib
def compute_hash(file_path):
try:
data = open(file_path, 'rb').read()
except:
return False
data_len = len(data)
start = data.find("infod")
if start == -1:
return False
start += 4
current = start + 1
dir_depth = 1
while current < data_len and dir_depth > 0:
if data[current] == 'e':
dir_depth -= 1
current += 1
elif data[current] == 'l' or data[current] == 'd':
dir_depth += 1
current += 1
elif data[current] == 'i':
current += 1
while data[current] != 'e':
current += 1
current += 1
elif data[current].isdigit():
num = data[current]
current += 1
while data[current] != ':':
num += data[current]
current += 1
current += 1 + int(num)
else:
return False
return hashlib.sha1(data[start:current]).hexdigest().upper()
print(compute_hash("${torrent_file}"))
END
`
if [[ ! $? ]] || [[ "${torrent_hash}" = 'False' ]]; then
return 1
fi
echo $torrent_hash
}
# keep track of the .torrent files to be downloaded
declare -A TORRENT_QUEUE
# keep track of the rsyncs to download torrent data
declare -A RUNNING_RSYNCS
# run indefinitely
while true; do
# check to make sure the path of the local .torrent files exists
if [[ ! -d "${TORRENT_FILE_PATH}" ]]; then
echo "${TORRENT_FILE_PATH} Does not exist"
exit 1
fi
OIFS="$IFS"
IFS=$'\n'
2015-02-06 22:20:06 -05:00
# enumerate the .torrent file directory
for file in `find "${TORRENT_FILE_PATH}"`; do
2015-02-06 22:20:06 -05:00
# check if the path is a directory
if [[ -d "${file}" ]]; then
# enumerate the directory
for sub_file in `find "${file}" -type f`; do
2015-02-06 22:20:06 -05:00
# this is the furthest we will descend
if [[ -f "${sub_file}" ]]; then
# get the torrent hash for the .torrent file
torrent_hash=`get_torrent_hash "${sub_file}"`
if [[ ! $? ]]; then
echo "Failed to get the torrent hash of ${sub_file}"
continue
fi
# add the torrent to the queue if it is not already in the queue
if [[ ! ${TORRENT_QUEUE[${torrent_hash}]+_} ]]; then
TORRENT_QUEUE[$torrent_hash]="${sub_file}"
2015-02-06 22:20:06 -05:00
fi
fi
done
# check that the path is a file
elif [[ -f "${file}" ]]; then
# get the torrent hash for the .torrent file
torrent_hash=`get_torrent_hash "${file}"`
if [[ ! $? ]]; then
echo "Failed to get the torrent hash of ${file}"
continue
fi
# add the torrent to the queue if it is not already in the queue
if [[ ! ${TORRENT_QUEUE[${torrent_hash}]+_} ]]; then
TORRENT_QUEUE[$torrent_hash]="${file}"
2015-02-06 22:20:06 -05:00
fi
fi
done
IFS="$OIFS"
2015-02-06 22:20:06 -05:00
# go through the torrent queue
for torrent_hash in "${!TORRENT_QUEUE[@]}"; do
# continue if the torrent is already being downloaded
if [[ ${RUNNING_RSYNCS[$torrent_hash]+_} ]]; then
continue
fi
# check to see if the torrent is on the rtorrent server
torrent_added=`get_torrent_added "${torrent_hash}"`
if [[ ! $? ]]; then
echo "Failed to see if ${TORRENT_QUEUE[$torrent_hash]} exists on the rtorrent server"
continue
fi
# if the torrent is not on the rtorrent server, upload it
if [[ $torrent_added -eq 0 ]]; then
scp "${TORRENT_QUEUE[$torrent_hash]}" "${SSH_USER}@${SSH_SERVER}:${SSH_SERVER_TORRENT_FILE_PATH}"
if [[ ! $? ]]; then
echo "Failed to upload ${TORRENT_QUEUE[$torrent_hash]}"
fi
fi
done
# if the amount of running rsyncs is below the desire amount, run items from the queue
for torrent_hash in "${!TORRENT_QUEUE[@]}"; do
# break out of the loop if we added enough jobs already
if [[ ${#RUNNING_RSYNCS[@]} -ge ${RSYNC_PROCESSES} ]]; then
break
fi
# make sure this torrent is not already being downloaded
if [[ ${RUNNING_RSYNCS[${torrent_hash}]+_} ]]; then
continue
fi
# see if the torrent is finished downloading remotely
torrent_completed=`get_torrent_complete "${torrent_hash}"`
if [[ ! $? ]]; then
echo "Failed to check if ${TORRENT_QUEUE[$torrent_hash]} is completed"
continue
fi
2015-02-06 22:20:06 -05:00
# the torrent is finished downloading remotely
if [[ "${torrent_completed}" -eq 1 ]]; then
torrent_name=`get_torrent_name "${torrent_hash}"`
2015-02-06 22:20:06 -05:00
if [[ ! $? ]]; then
echo "Failed to get torrent name for ${TORRENT_QUEUE[$torrent_hash]}"
2015-02-06 22:20:06 -05:00
continue
fi
# start the download and record the PID
2015-03-01 00:33:53 -05:00
echo "Started download for ${torrent_name} (${TORRENT_QUEUE[$torrent_hash]})"
rsync -hrvP --inplace "${SSH_USER}@${SSH_SERVER}:\"${SSH_SERVER_DOWNLOAD_PATH}/${torrent_name}"\" "${TORRENT_TMP_DOWNLOAD}/" > /dev/null &
RUNNING_RSYNCS[${torrent_hash}]=$!
fi
done
2015-02-06 22:20:06 -05:00
# checkup on the running rsyncs
for torrent_hash in "${!RUNNING_RSYNCS[@]}"; do
pid=${RUNNING_RSYNCS[$torrent_hash]}
# check to see if the given PID is still running
if ! kill -0 "${pid}" 2> /dev/null; then
2015-02-06 22:20:06 -05:00
# get the return code of the PID
wait $pid
return=$?
if [[ $return ]]; then
echo "Successfully downloaded ${TORRENT_QUEUE[$torrent_hash]}"
torrent_name=`get_torrent_name "${torrent_hash}"`
if [[ $? ]]; then
final_location_dir="${TORRENT_DOWNLOAD}"
if [[ `dirname "${TORRENT_QUEUE[$torrent_hash]}"` != "${TORRENT_FILE_PATH}" ]]; then
final_location_dir="${final_location_dir}/$(basename "`dirname "${TORRENT_QUEUE[$torrent_hash]}"`")"
2015-02-06 22:20:06 -05:00
fi
if [[ ! -d "${final_location_dir}" ]]; then
mkdir -p "${final_location_dir}"
fi
mv "${TORRENT_TMP_DOWNLOAD}/${torrent_name}" "${final_location_dir}/"
rm "${TORRENT_QUEUE[$torrent_hash]}"
2015-02-06 22:20:06 -05:00
unset TORRENT_QUEUE[$torrent_hash]
else
echo "Failed to get torrent name for ${TORRENT_QUEUE[$torrent_hash]}"
fi
else
echo "Failed to download ${TORRENT_QUEUE[$torrent_hash]} with rsync return code $return"
fi
unset RUNNING_RSYNCS[$torrent_hash]
fi
done
sleep 5s
done