Shell 脚本 - 简单爬虫

thd_file_name="thunder"temp_file="$root_dir/$thd_file_name.html"curl $url -o $temp_file;

while read linedo    ### parse the sub string from source data by use the grep patten    result=` echo $line | grep -Eo "$patten1"  `    ### the if the line is include the grep patten    if test -n "$result"  ; then 
        # the parse result maybe have multiple values, so traverse the result        for r_item in $result;        do            # parese the torrent value from result            result_sed=` echo $r_item  | sed "s/$patten2/\1/g" `            # There are tow parsing results of sed command, one for successful matching is torrent link value,             #   and one for failed matching is the source string .            # so we need to test if the mathing result is successful.            # if the two strings are not equals, it means the match was successful            if [ "$result_sed" != "$r_item" ]  ; then                 #echo $result_sed; #exit;                # Write the torrent link value into file                echo $result_sed >> $target_file_path;            fi                    done    fi    done < $temp_file

# copy the file to clipboardcat $target_file_path | pbcopy

## the data directory path of thunder, system environment variable used in the sampleTHUNDER_DATA_DIR=$THUNDER_ROOT_DIR## The regular expression when [grep] command parses the substringPATTEN_TO_GREP='name="down_url_list_0" class="down_url" value="[^<>]*" file_name='## The sed expression when [grep] command parses the substringPATTEN_TO_SED='.*value="\(.*\)".*'

#!/bin/bash#derive the url from https://www.meijutt.comecho " <<<<<<<<<<<<<<<<<<< source from https://www.meijutt.com >>>>>>>>>>>>>>>>>>>>>>>>>>>"# derive the root directory pathSHELL_FOLDER=$(dirname "$0")source "$SHELL_FOLDER/.meijutt"source "$SHELL_FOLDER/source_from_url_common.sh"

#!/bin/bash#derive the url from https://www.loldytt.com# set the source file the parse the urls to file# if test -z "$1" ; then#  echo "please input the html source file !!"#  exit 0# fi#html_file=$1
# # test if the file is exists# if ! test -f "$html_file" ; then#  echo "the file [$html_file] is not exits, please input the valid path !!!"#  exit 0# fi
# get file data from url if test -z "$1" ; then echo "please input url !!" exit 0fiurl=$1
# get the file then output into a temp file#root_dir="/Users/jackbai/Downloads"root_dir=$THUNDER_DATA_DIR
# test if the root directory is validif ! test -d "$root_dir" ; then     echo "the data root direcotry path is not valid, please set the variable [ THUNDER_DATA_DIR ] !!!!"    exitfi# echo $root_dir;exit;thd_file_name="thunder"temp_file="$root_dir/$thd_file_name.html"curl $url -o $temp_file; #exit;# echo "temp file: $temp_file"
# read the gb2312 format dataexport LC_CTYPE='C'export LC_COLLATE='C'
#get the file name and use it to name the target filetarget_file_path="$root_dir/$thd_file_name.thdresult"# touch $root_dir/$thd_file_name.thdresultecho "" > $target_file_path# use the regex parse the urlpatten1=$PATTEN_TO_GREPpatten2=$PATTEN_TO_SED# read the file line by linewhile read linedo    ### parse the sub string from source data by use the grep patten    result=` echo $line | grep -Eo "$patten1"  `    ### the if the line is include the grep patten    if test -n "$result"  ; then         # echo "result: "$result;        # the parse result maybe have multiple values, so traverse the result        for r_item in $result;        do            #echo "results: "$r_item            # parese the torrent value from result            result_sed=` echo $r_item  | sed "s/$patten2/\1/g" `            # There are tow parsing results of sed command, one for successful matching is torrent link value,             #   and one for failed matching is the source string .            # so we need to test if the mathing result is successful.            # if the two strings are not equals, it means the match was successful            if [ "$result_sed" != "$r_item" ]  ; then                 #echo $result_sed; #exit;                # Write the torrent link value into file                echo $result_sed >> $target_file_path;            fi                    done        # exit;        # write into file        #  echo ': '$line ; echo ':: '$result; echo $result | sed "s/$patten2/\1/g"; exit        # echo $result >> $target_file_path;        # exit;    fi    done < $temp_file# copy the file to clipboardcat $target_file_path | pbcopy
rm -f $temp_filerm -rf $target_file_pathecho "parse ok !!! ";