XuYibing's Homework1

week_1/2018202051XYB/README.md

@@ -0,0 +1,13 @@
+# web_crawler
+
+Homework1 (2019.3.15)
+爬取 http://info.ruc.edu.cn 下的所有页面    
+
+## Usage
+```
+make
+./run
+```
+
+## Known Issues 
+非常不建议在RUC网络下运行，会蓝屏（我也不知道为什么

week_1/2018202051XYB/src/main.cpp

@@ -0,0 +1,97 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <queue>
+#include <map>
+#include <set>
+#include <unistd.h>
+
+#include "my_curl.h"
+#include "my_string.h"
+#include "main.h"
+
+std::string root_url("http://info.ruc.edu.cn/");
+
+std::ofstream fout, ferr; // 成功下载 url 列表
+
+std::set<std::string> visit; // 排重
+std::queue<std::string> waiting; // bfs 等待队列
+std::vector<std::string> working; // 本次 multihandle 处理的 url
+std::vector<std::string*> results; // 本次 multihandle 结果 html 存放
+std::map<std::string, int> fail_time; // 因网络问题重连次数
+
+
+void push_into_waiting(std::string &x)
+{
+    if(!visit.count(x))
+    {
+        waiting.push(x);
+        visit.insert(x);
+    }
+}
+
+void save_to_file(std::string url, std::string* res)
+{
+    if(res->empty())
+        return;
+
+    url = url.substr(23);
+    if(url[url.size() - 1] == '/' || url.empty())
+        url = url + "index.html";
+
+    std::size_t found = url.find_last_of("/");
+    std::string path = "download/" + url.substr(0, found + 1);
+    std::string order = "mkdir -p " + path;
+
+
+    if(access(path.c_str(), 0) == -1)
+        system(order.c_str());
+    std::ofstream fw;
+    fw.open(("download/" + url).c_str());
+    fw << *res;
+    fw.close();
+}
+
+
+void bfs()
+{
+    int page_cnt = 0; //处理网页总数
+
+    push_into_waiting(root_url);
+    while(!waiting.empty())
+    {   
+        int page_num = 0; // 当前 multihandle 处理成功页面数
+        multi(page_num);
+        int index = working.size();
+        for(int i = 0; i < index; i++)
+        {
+            // save to file
+            save_to_file(working[i], results[i]);
+
+            // parse html
+            parse_html(results[i], working[i]);
+
+            delete results[i];
+        }
+        working.clear();
+        results.clear();
+
+        page_cnt += page_num;
+        printf("new: %6d   total: %6d\n", page_num, page_cnt);
+
+    }  
+    std::cout << "Total : " << page_cnt << std::endl;
+}
+
+
+int main()
+{
+    ferr.open("error.txt");
+    fout.open("visit.txt");
+
+    bfs();
+
+    ferr.close();
+    fout.close();
+
+}

week_1/2018202051XYB/src/main.h

@@ -0,0 +1,30 @@
+#ifndef _MAIN_H
+#define _MAIN_H
+
+#include <iostream>
+#include <string>
+#include <fstream>
+#include <queue>
+#include <map>
+#include <set>
+
+extern std::string root_url;
+
+extern std::ofstream fout, ferr;
+
+extern std::set<std::string> visit; // 排重
+extern std::queue<std::string> waiting; // bfs 等待队列
+extern std::vector<std::string> working; // 本次 multihandle 处理的 url
+extern std::vector<std::string*> results; // 本次 multihandle 结果 html 存放
+extern std::map<std::string, int> fail_time; // 因网络问题重连次数
+
+extern int page_cnt, //处理网页总数
+           page_num; // 当前 multihandle 处理成功页面数
+
+void push_into_waiting(std::string &x);
+
+void save_to_file(std::string url, std::string* res);
+
+void bfs();
+
+#endif

week_1/2018202051XYB/src/makefile

@@ -0,0 +1,12 @@
+.PHONY : clean
+
+main : main.o my_string.o my_curl.o main.h my_string.h my_curl.h
+	@g++ -o run main.o my_string.o my_curl.o -lcurl
+main.o : main.cpp main.h my_string.h my_curl.h
+	@g++ -c main.cpp
+my_string.o : my_string.cpp main.h my_string.h
+	@g++ -c my_string.cpp
+my_curl.o : my_curl.cpp main.h my_string.h my_curl.h
+	@g++ -c my_curl.cpp
+clean :
+	@rm -f *.o

week_1/2018202051XYB/src/my_curl.cpp

@@ -0,0 +1,121 @@
+#include <string>
+#include <iostream>
+#include <fstream>
+
+#include <curl/curl.h>
+#include <curl/multi.h>
+
+#include "my_curl.h"
+#include "my_string.h"
+#include "main.h"
+
+size_t my_write(void *ptr, size_t size, size_t nmemb, void *stream)
+{
+    std::string *str = dynamic_cast<std::string*>((std::string*)stream);
+    if(size == 0 || ptr == NULL)
+        return -1;
+    char *p_data = (char*) ptr;
+    str->append(p_data, nmemb * size);
+    return nmemb * size;
+}
+
+// 单个链接设置
+void easy_init(CURLM *cm, int x)
+{
+    CURL *eh = curl_easy_init();
+    curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, my_write);
+    curl_easy_setopt(eh, CURLOPT_WRITEDATA, (void*)results[x]);
+    curl_easy_setopt(eh, CURLOPT_HEADER, 0L);
+    curl_easy_setopt(eh, CURLOPT_URL, working[x].c_str());
+    curl_easy_setopt(eh, CURLOPT_PRIVATE, working[x].c_str());
+    curl_easy_setopt(eh, CURLOPT_VERBOSE, 0L);
+    curl_easy_setopt(eh, CURLOPT_TIMEOUT_MS, MAX_EASY_TIME);
+
+    curl_multi_add_handle(cm, eh);
+}
+
+// bfs 单次处理 waiting 队列内前 EASY_MAX 个 url
+void multi(int &page_num)
+{
+    // multihandle 初始化
+    CURLM *cm = NULL;
+    CURL *eh = NULL;
+    CURLMsg *msg = NULL;
+    CURLcode return_code;
+    int still_running = 0, msgs_left = 0;
+    int http_status_code;
+    const char *sz_url;
+    page_num = 0;
+
+    curl_global_init(CURL_GLOBAL_ALL);
+    cm = curl_multi_init();
+
+    int index = 0;
+    while(!waiting.empty() && index < EASY_MAX)
+    {
+        // 从 waiting 内取出 url 扔进 multihandle 的 working 队列
+        std::string x = waiting.front();
+        waiting.pop();
+        working.push_back(x);
+        std::string* page = new std::string();
+        results.push_back(page);
+        easy_init(cm, index);
+        ++index;
+    }
+
+    // multi_perform()
+    curl_multi_perform(cm, &still_running);
+    do
+    {
+        curl_multi_perform(cm, &still_running);
+    } while (still_running);
+
+    // 读取单次 multihandle 结果
+    while((msg = curl_multi_info_read(cm, &msgs_left))) 
+    {
+        if(msg->msg == CURLMSG_DONE)
+        {
+            eh = msg->easy_handle;
+            return_code = msg->data.result;
+            // error
+            if(return_code != CURLE_OK)
+            {
+                // 28 网络问题链接失败 不记录 
+                if(msg->data.result != 28)
+                    ferr << "CURL error code: " << msg->data.result << std::endl;
+                //continue;
+            }
+
+            // get http status code
+            http_status_code = 0;
+            sz_url = NULL;
+            curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, &http_status_code);
+            curl_easy_getinfo(eh, CURLINFO_PRIVATE, &sz_url);
+
+            if(http_status_code == 200)
+            {
+                // 单个网页爬取成功
+                page_num++;
+                fout << sz_url << std::endl;
+            }
+            else
+            {
+                // 单个网页爬取失败
+                fail_time[sz_url]++;
+                if(fail_time[sz_url] < MAX_FAIL_TIME)
+                    waiting.push(sz_url);
+                else
+                    ferr << "GET of " << sz_url 
+                        << " returned http status code "
+                        << http_status_code << std::endl;
+            }
+
+            curl_multi_remove_handle(cm, eh);
+            curl_easy_cleanup(eh);
+        }
+        else 
+            ferr << "error: after curl_multi_info_read(), CURLMsg = "
+                 << msg->msg << std::endl;
+    }
+    curl_multi_cleanup(cm);
+}

week_1/2018202051XYB/src/my_curl.h

@@ -0,0 +1,23 @@
+#ifndef _MY_CURL_H
+#define _MY_CURL_H
+
+#include <curl/curl.h>
+#include <curl/multi.h>
+
+// MAC_FAIL_TIME : 失败重连次数上限
+#define MAX_FAIL_TIME 5
+// MAC_EASY_TIME : 单次链接时间上限（ms）
+#define MAX_EASY_TIME 1500L
+// EASY_MAX : 单次 multihandle 处理上限
+#define EASY_MAX 64
+
+size_t my_write(void *ptr, size_t size, size_t nmemb, void *stream);
+
+// 单个链接设置
+void easy_init(CURLM *cm, int x);
+
+// bfs 单次处理 waiting 队列内前 EASY_MAX 个 url
+void multi(int &page_num);
+
+
+#endif

week_1/2018202051XYB/src/my_string.cpp

@@ -0,0 +1,50 @@
+#include <string>
+#include <regex>
+
+#include "my_string.h"
+#include "main.h"
+
+std::string normalize_url(std::string str, std::string &f_url)
+{
+    std::string empty_str = ""; 
+	if(str[str.size()-2] == '/') 
+		str = str.substr(6, str.size() - 8);
+	else
+		str = str.substr(6, str.size() - 7);
+
+    for(auto s : black_sub_list)
+    {
+        if(str.find(s) != std::string::npos)
+        {
+            return empty_str;
+        }
+    }
+    if(str.substr(0, 23) == "http://info.ruc.edu.cn/") 
+		return str;
+	if(str.substr(0, 7) == "http://" ||
+	   str.substr(0, 8) == "https://")
+		return empty_str; // empty
+
+	if(str[0] == '/') 
+		return "http://info.ruc.edu.cn" + str;
+
+
+    std::size_t found = f_url.find_last_of("/");
+	return f_url.substr(0, found + 1) + str;
+}
+
+void parse_html(std::string *content, std::string &f_url)
+{
+    std::sregex_iterator it(content->begin(),
+                            content->end(),
+                            re_href),
+                         end;
+    for(; it != end; it++)
+    {
+        std::string next_url = normalize_url(it->str(), f_url);
+        if(!next_url.empty())
+            push_into_waiting(next_url);
+    }
+
+}
+

week_1/2018202051XYB/src/my_string.h

@@ -0,0 +1,21 @@
+#ifndef _MY_STRING_H
+#define _MY_STRING_H
+
+#include <string>
+#include <regex>
+
+const std::string black_sub_list[] = {
+    ".css", ".js", ".png", ".jpg", ".jpeg", ".ico", ".mp4", ".mp3", ".flv",
+    ".xls", ".xlsx", ".doc", ".ppt", ".docx", ".pptx", ".rar", ".zip", ".7z", ".pdf",
+    "mailto", //"javascript",
+    //"download.php", // file download
+};
+
+std::string normalize_url(std::string str, std::string &f_url);
+
+const std::regex re_href("href=\"(.*?)\"");
+void parse_html(std::string *content, std::string &f_url);
+
+
+
+#endif