package DownloadPackage;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
/*
 * 根据一个url下载一个html页面
 */
public class HtmlDownloader {
 RequestSet requestset = null;
 public HtmlDownloader(RequestSet requestset){
  this.requestset = requestset;
 }
 public String downloadhtml(String url){
  String html = null;
  //创建一个客户端
  //创建一个读取流从entity读取html
  BufferedReader reader = null;
  CloseableHttpClient httpclient = HttpClients.createDefault();
  HttpResponse response = null;
  try {
   response = httpclient.execute(requestset.getMethod(url));
   HttpEntity entity = response.getEntity();
   reader = new BufferedReader(new InputStreamReader(entity.getContent()));
   StringBuilder sb = new StringBuilder();
   while((html = reader.readLine()) != null){
    sb.append(html);
   }
   html = sb.toString();
   System.out.println("一个html页面获取成功");
  }
  catch (IOException e) {
   System.out.println(url+"连接失败");
  }
  finally{
   if(reader != null){
    try {
     reader.close();
     httpclient.close();
    }
    catch (IOException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
    }
   }
  }
  return html;
 }
}

package DownloadPackage;
import java.util.LinkedList;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class UrlGet {
 public LinkedList<String> geturls(String html){
  LinkedList<String> urls = new LinkedList<String>();
  Document doc = Jsoup.parse(html);
  Elements links = doc.getElementsByTag("a");
  for (Element link:links){
   String url = link.attr("href");
   urls.add(url);
  }
  return urls;
 }
}

package ChoosePackage;
public interface ResourseChooser {
 public Boolean isNeed(String url);
 public Boolean isResourse(String url);
 public String process(String url);
}

package DownloadPackage;
import org.apache.http.client.methods.HttpGet;
/*
 * 一个用于获得Request请求的接口
 * 实现getMethod方法获取Get方法
 */
public interface RequestSet {
 public HttpGet getMethod(String url);
}
Saveutil接口用于自定义保存方式,需要实现save方法
package SaveUtil;
/*
 * 数据储存的工具接口,必须实现保存方法
 */
public interface SaveUtil {
 public void save(String url,String html);
}

package Spider;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import ChoosePackage.MyResourseChooser;
import ChoosePackage.ResourseChooser;
import DownloadPackage.HtmlDownloader;
import DownloadPackage.RequestSet;
import DownloadPackage.UrlGet;
import SaveUtil.MySaveUtil;
import SaveUtil.SaveUtil;
/*
 * 用于爬取资源的类
 */
public class Spider{
 public static void main(String[] args) {
  new Spider("http://www.bilibili.net").spiderstart();
 }
 //种子url
 String seed = null;
 //用于保存数据的类,需要自己实现
 private SaveUtil saveutil = null;
 //html下载类
 private HtmlDownloader downloader = null;
 //url下载类
 private UrlGet urldownloader = null;
 //资源选择工具
 private ResourseChooser resoursechooser = null;
 //用于保存未下载的网页
 LinkedList<String> unvisited = new LinkedList<String>();
 //用于保存已下载的网页
 HashSet<String> visited = new HashSet<String>();
 //自定义储存方式,请求方式,资源筛选方式的构造方法
 public Spider(SaveUtil saveutil,RequestSet request,ResourseChooser resoursechooser,String seed){
  this.saveutil = saveutil;
  this.downloader = new HtmlDownloader(request);
  this.urldownloader = new UrlGet();
  this.resoursechooser = resoursechooser;
  this.seed = seed;
  unvisited.add(seed);
 }
 //自定义储存方式,资源筛选方式的构造方法
 public Spider(SaveUtil saveutil,ResourseChooser resoursechooser,String seed){
  this.resoursechooser = resoursechooser;
  this.downloader = new HtmlDownloader(new getRequest());
  this.saveutil = saveutil;
  this.urldownloader = new UrlGet();
  this.seed = seed;
  unvisited.add(seed);
 }
 //自定义储存方式,请求的构造方法
 public Spider(SaveUtil saveutil,RequestSet requestset,String seed){
  this.saveutil = saveutil;
  this.downloader = new HtmlDownloader(requestset);
  this.resoursechooser = new MyResourseChooser();
  this.urldownloader = new UrlGet();
  this.seed = seed;
  unvisited.add(seed);
 }
 //自定义储存方式的构造方法
 public Spider(SaveUtil saveutil,String seed){
  this.saveutil = saveutil;
  this.downloader = new HtmlDownloader(new getRequest());
  this.resoursechooser = (new MyResourseChooser());
  this.urldownloader = new UrlGet();
  this.seed = seed;
  unvisited.add(seed);
 }
 //默认的爬虫构造方法
 public Spider(String seed){
  this.saveutil = new MySaveUtil();
  this.downloader = new HtmlDownloader(new getRequest());
  this.resoursechooser = (new MyResourseChooser());
  this.urldownloader = new UrlGet();
  this.seed = seed;
  unvisited.add(seed);
 }
 //开始爬取的方法
 private void spiderstart(){
  String html = null;
  while(!unvisited.isEmpty()){
   String url = unvisited.poll();
   System.out.println("开始获取"+url);
   if(resoursechooser.isNeed(url)){
    try{
     html = downloader.downloadhtml(url);
    }
    catch(RuntimeException e){
     System.out.println(url+"连接获取失败");
     continue;
    }
    visited.add(url);
    LinkedList<String> urls = new LinkedList<String>();
    try{
     urls = urldownloader.geturls(html);
    }
    catch(RuntimeException e){
     System.out.println(url+"的html页面为空");
     continue;
    }
    Iterator<String> it = urls.iterator();
    while(it.hasNext()){
     String newurl = it.next();
     if(resoursechooser.isNeed(newurl)&&!visited.contains(newurl)&&!unvisited.contains(newurl)){
      newurl = resoursechooser.process(newurl);
      unvisited.add(newurl);
      System.out.println(newurl+"加入页面");
     }
    }
    System.out.println("获取了"+url+"上的所有url");
    if(resoursechooser.isResourse(url)){
     saveutil.save(url,html);
    }
   }
  }
 }
 //默认资源筛选类
 private class MyResourseChooser implements ResourseChooser{
  @Override
  public Boolean isNeed(String url) {
   // TODO Auto-generated method stub
   if(!url.startsWith("/")&&!url.startsWith("http")){
    return false;
   }
   return true;
  }
  @Override
  public Boolean isResourse(String url) {
   // TODO Auto-generated method stub
   return true;
  }
  @Override
  public String process(String url) {
   // TODO Auto-generated method stub
   if(!url.startsWith("http")){
    url = seed+url;
   }
   return url;
  }
 }
 public class getRequest implements RequestSet{
  public HttpGet getMethod(String url) {
   // TODO Auto-generated method stub
   //创建一个get请求方法
   HttpGet getmethod = new HttpGet(url);
   //HttpHost proxy = new HttpHost("124.88.67.81",80);这里不设置代理IP
   //设置请求超时时间等 
   RequestConfig responseconfig = RequestConfig.custom().setConnectionRequestTimeout(10000).setConnectTimeout(10000).setSocketTimeout(10000).build();
   //设置请求头,主要是user-agent
   getmethod.addHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
   //设置请求参数
   getmethod.setConfig(responseconfig);
   return getmethod;
  }
 }
 //默认的存储类
 public class MySaveUtil implements SaveUtil{
  @Override
  public void save(String url, String html) {
   // TODO Auto-generated method stub
   String filename = getfilename(url);
   BufferedWriter writer = null;
   try{
    writer = new BufferedWriter(new FileWriter(filename));
    writer.write(html);
    writer.flush();
    System.out.println("文件写入成功");
   }
   catch(IOException e){
    System.out.println("文件写入失败");
   }
   finally{
    try {
     if(writer != null)
     writer.close();
    }
    catch (IOException e) {
     // TODO Auto-generated catch block
     System.out.println("流关闭失败");
    }
   }
  }
  private String getfilename(String url){
   String fileparentpath = "f://html";
   File file = new File(fileparentpath);
   if(!file.exists()){
    file.mkdir();
   }
   int last = url.lastIndexOf(".");
   int first = url.indexOf(".");
   url = url.substring(first,last);
   url = url.replaceAll("\\.", "");
   url = url.replaceAll("/", "");
   return fileparentpath+"/"+url+".txt";
  }
 }
}