diff --git a/week_1/2018202051XYB/README.md b/week_1/2018202051XYB/README.md new file mode 100644 index 0000000..f4f0d8a --- /dev/null +++ b/week_1/2018202051XYB/README.md @@ -0,0 +1,13 @@ +# web_crawler + +Homework1 (2019.3.15) +爬取 http://info.ruc.edu.cn 下的所有页面 + +## Usage +``` +make +./run +``` + +## Known Issues +非常不建议在RUC网络下运行,会蓝屏(我也不知道为什么 diff --git a/week_1/2018202051XYB/src/main.cpp b/week_1/2018202051XYB/src/main.cpp new file mode 100644 index 0000000..328f290 --- /dev/null +++ b/week_1/2018202051XYB/src/main.cpp @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "my_curl.h" +#include "my_string.h" +#include "main.h" + +std::string root_url("http://info.ruc.edu.cn/"); + +std::ofstream fout, ferr; // 成功下载 url 列表 + +std::set visit; // 排重 +std::queue waiting; // bfs 等待队列 +std::vector working; // 本次 multihandle 处理的 url +std::vector results; // 本次 multihandle 结果 html 存放 +std::map fail_time; // 因网络问题重连次数 + + +void push_into_waiting(std::string &x) +{ + if(!visit.count(x)) + { + waiting.push(x); + visit.insert(x); + } +} + +void save_to_file(std::string url, std::string* res) +{ + if(res->empty()) + return; + + url = url.substr(23); + if(url[url.size() - 1] == '/' || url.empty()) + url = url + "index.html"; + + std::size_t found = url.find_last_of("/"); + std::string path = "download/" + url.substr(0, found + 1); + std::string order = "mkdir -p " + path; + + + if(access(path.c_str(), 0) == -1) + system(order.c_str()); + std::ofstream fw; + fw.open(("download/" + url).c_str()); + fw << *res; + fw.close(); +} + + +void bfs() +{ + int page_cnt = 0; //处理网页总数 + + push_into_waiting(root_url); + while(!waiting.empty()) + { + int page_num = 0; // 当前 multihandle 处理成功页面数 + multi(page_num); + int index = working.size(); + for(int i = 0; i < index; i++) + { + // save to file + save_to_file(working[i], results[i]); + + // parse html + parse_html(results[i], working[i]); + + delete results[i]; + } + working.clear(); + results.clear(); + + page_cnt += page_num; + printf("new: %6d total: %6d\n", page_num, page_cnt); + + } + std::cout << "Total : " << page_cnt << std::endl; +} + + +int main() +{ + ferr.open("error.txt"); + fout.open("visit.txt"); + + bfs(); + + ferr.close(); + fout.close(); + +} \ No newline at end of file diff --git a/week_1/2018202051XYB/src/main.h b/week_1/2018202051XYB/src/main.h new file mode 100644 index 0000000..4e00c22 --- /dev/null +++ b/week_1/2018202051XYB/src/main.h @@ -0,0 +1,30 @@ +#ifndef _MAIN_H +#define _MAIN_H + +#include +#include +#include +#include +#include +#include + +extern std::string root_url; + +extern std::ofstream fout, ferr; + +extern std::set visit; // 排重 +extern std::queue waiting; // bfs 等待队列 +extern std::vector working; // 本次 multihandle 处理的 url +extern std::vector results; // 本次 multihandle 结果 html 存放 +extern std::map fail_time; // 因网络问题重连次数 + +extern int page_cnt, //处理网页总数 + page_num; // 当前 multihandle 处理成功页面数 + +void push_into_waiting(std::string &x); + +void save_to_file(std::string url, std::string* res); + +void bfs(); + +#endif \ No newline at end of file diff --git a/week_1/2018202051XYB/src/makefile b/week_1/2018202051XYB/src/makefile new file mode 100644 index 0000000..163d8c3 --- /dev/null +++ b/week_1/2018202051XYB/src/makefile @@ -0,0 +1,12 @@ +.PHONY : clean + +main : main.o my_string.o my_curl.o main.h my_string.h my_curl.h + @g++ -o run main.o my_string.o my_curl.o -lcurl +main.o : main.cpp main.h my_string.h my_curl.h + @g++ -c main.cpp +my_string.o : my_string.cpp main.h my_string.h + @g++ -c my_string.cpp +my_curl.o : my_curl.cpp main.h my_string.h my_curl.h + @g++ -c my_curl.cpp +clean : + @rm -f *.o \ No newline at end of file diff --git a/week_1/2018202051XYB/src/my_curl.cpp b/week_1/2018202051XYB/src/my_curl.cpp new file mode 100644 index 0000000..7333874 --- /dev/null +++ b/week_1/2018202051XYB/src/my_curl.cpp @@ -0,0 +1,121 @@ +#include +#include +#include + +#include +#include + +#include "my_curl.h" +#include "my_string.h" +#include "main.h" + +size_t my_write(void *ptr, size_t size, size_t nmemb, void *stream) +{ + std::string *str = dynamic_cast((std::string*)stream); + if(size == 0 || ptr == NULL) + return -1; + char *p_data = (char*) ptr; + str->append(p_data, nmemb * size); + return nmemb * size; +} + +// 单个链接设置 +void easy_init(CURLM *cm, int x) +{ + CURL *eh = curl_easy_init(); + curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, my_write); + curl_easy_setopt(eh, CURLOPT_WRITEDATA, (void*)results[x]); + curl_easy_setopt(eh, CURLOPT_HEADER, 0L); + curl_easy_setopt(eh, CURLOPT_URL, working[x].c_str()); + curl_easy_setopt(eh, CURLOPT_PRIVATE, working[x].c_str()); + curl_easy_setopt(eh, CURLOPT_VERBOSE, 0L); + curl_easy_setopt(eh, CURLOPT_TIMEOUT_MS, MAX_EASY_TIME); + + curl_multi_add_handle(cm, eh); +} + +// bfs 单次处理 waiting 队列内前 EASY_MAX 个 url +void multi(int &page_num) +{ + // multihandle 初始化 + CURLM *cm = NULL; + CURL *eh = NULL; + CURLMsg *msg = NULL; + CURLcode return_code; + int still_running = 0, msgs_left = 0; + int http_status_code; + const char *sz_url; + page_num = 0; + + curl_global_init(CURL_GLOBAL_ALL); + cm = curl_multi_init(); + + int index = 0; + while(!waiting.empty() && index < EASY_MAX) + { + // 从 waiting 内取出 url 扔进 multihandle 的 working 队列 + std::string x = waiting.front(); + waiting.pop(); + working.push_back(x); + std::string* page = new std::string(); + results.push_back(page); + easy_init(cm, index); + ++index; + } + + // multi_perform() + curl_multi_perform(cm, &still_running); + do + { + curl_multi_perform(cm, &still_running); + } while (still_running); + + // 读取单次 multihandle 结果 + while((msg = curl_multi_info_read(cm, &msgs_left))) + { + if(msg->msg == CURLMSG_DONE) + { + eh = msg->easy_handle; + return_code = msg->data.result; + // error + if(return_code != CURLE_OK) + { + // 28 网络问题链接失败 不记录 + if(msg->data.result != 28) + ferr << "CURL error code: " << msg->data.result << std::endl; + //continue; + } + + // get http status code + http_status_code = 0; + sz_url = NULL; + curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, &http_status_code); + curl_easy_getinfo(eh, CURLINFO_PRIVATE, &sz_url); + + if(http_status_code == 200) + { + // 单个网页爬取成功 + page_num++; + fout << sz_url << std::endl; + } + else + { + // 单个网页爬取失败 + fail_time[sz_url]++; + if(fail_time[sz_url] < MAX_FAIL_TIME) + waiting.push(sz_url); + else + ferr << "GET of " << sz_url + << " returned http status code " + << http_status_code << std::endl; + } + + curl_multi_remove_handle(cm, eh); + curl_easy_cleanup(eh); + } + else + ferr << "error: after curl_multi_info_read(), CURLMsg = " + << msg->msg << std::endl; + } + curl_multi_cleanup(cm); +} \ No newline at end of file diff --git a/week_1/2018202051XYB/src/my_curl.h b/week_1/2018202051XYB/src/my_curl.h new file mode 100644 index 0000000..ed78f01 --- /dev/null +++ b/week_1/2018202051XYB/src/my_curl.h @@ -0,0 +1,23 @@ +#ifndef _MY_CURL_H +#define _MY_CURL_H + +#include +#include + +// MAC_FAIL_TIME : 失败重连次数上限 +#define MAX_FAIL_TIME 5 +// MAC_EASY_TIME : 单次链接时间上限(ms) +#define MAX_EASY_TIME 1500L +// EASY_MAX : 单次 multihandle 处理上限 +#define EASY_MAX 64 + +size_t my_write(void *ptr, size_t size, size_t nmemb, void *stream); + +// 单个链接设置 +void easy_init(CURLM *cm, int x); + +// bfs 单次处理 waiting 队列内前 EASY_MAX 个 url +void multi(int &page_num); + + +#endif \ No newline at end of file diff --git a/week_1/2018202051XYB/src/my_string.cpp b/week_1/2018202051XYB/src/my_string.cpp new file mode 100644 index 0000000..4e36469 --- /dev/null +++ b/week_1/2018202051XYB/src/my_string.cpp @@ -0,0 +1,50 @@ +#include +#include + +#include "my_string.h" +#include "main.h" + +std::string normalize_url(std::string str, std::string &f_url) +{ + std::string empty_str = ""; + if(str[str.size()-2] == '/') + str = str.substr(6, str.size() - 8); + else + str = str.substr(6, str.size() - 7); + + for(auto s : black_sub_list) + { + if(str.find(s) != std::string::npos) + { + return empty_str; + } + } + if(str.substr(0, 23) == "http://info.ruc.edu.cn/") + return str; + if(str.substr(0, 7) == "http://" || + str.substr(0, 8) == "https://") + return empty_str; // empty + + if(str[0] == '/') + return "http://info.ruc.edu.cn" + str; + + + std::size_t found = f_url.find_last_of("/"); + return f_url.substr(0, found + 1) + str; +} + +void parse_html(std::string *content, std::string &f_url) +{ + std::sregex_iterator it(content->begin(), + content->end(), + re_href), + end; + for(; it != end; it++) + { + std::string next_url = normalize_url(it->str(), f_url); + if(!next_url.empty()) + push_into_waiting(next_url); + } + +} + diff --git a/week_1/2018202051XYB/src/my_string.h b/week_1/2018202051XYB/src/my_string.h new file mode 100644 index 0000000..3a6ac59 --- /dev/null +++ b/week_1/2018202051XYB/src/my_string.h @@ -0,0 +1,21 @@ +#ifndef _MY_STRING_H +#define _MY_STRING_H + +#include +#include + +const std::string black_sub_list[] = { + ".css", ".js", ".png", ".jpg", ".jpeg", ".ico", ".mp4", ".mp3", ".flv", + ".xls", ".xlsx", ".doc", ".ppt", ".docx", ".pptx", ".rar", ".zip", ".7z", ".pdf", + "mailto", //"javascript", + //"download.php", // file download +}; + +std::string normalize_url(std::string str, std::string &f_url); + +const std::regex re_href("href=\"(.*?)\""); +void parse_html(std::string *content, std::string &f_url); + + + +#endif \ No newline at end of file