Java 写的简单网络爬虫

CrawlTheWeb.java

import java.sql.Time;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class CrawlTheWeb extends Thread{
    public static Object lock = new Object();//线程锁 用于使线程进入睡眠，或随机唤醒一个线程
    public void run(){
        while(true){
            if(Thread.currentThread().isInterrupted()){
                System.out.println(Thread.currentThread().getName()+"完成了任务。");
                break;
            }
            if(AllUrl.waitUrl.isEmpty() == false){
                String nextUrl = AllUrl.waitUrl.get(0);
                AllUrl.waitUrl.remove(0);
                AllUrl.workUrl(nextUrl,AllUrl.UrlDepth.get(nextUrl));
                System.out.println(this.getName()+"开始爬取《"+AllUrl.bookName.get(nextUrl)+"》，章节数： "+AllUrl.UrlDepth.get(nextUrl));
            }else{
                synchronized (lock){
                    try {
                        System.out.println("待爬取列表为空，"+this.getName()+"进入等待状态。");
                        lock.wait();
                    }catch (Exception e){
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    public static void main(String[] args) {
        SimpleDateFormat formatter= new SimpleDateFormat("yyyy-MM-dd 'at' HH:mm:ss z");
        Date date = new Date(System.currentTimeMillis());
        System.out.println(formatter.format(date));
        String strUrl = "https://read.qidian.com/chapter/D-1F0Iq1JGPOVUeyz9PqUQ2/DIfEaAmW-9X6ItTi_ILQ7A2/";//爬取的网页
        AllUrl.addUrl(strUrl,1,"模拟器：开局天牢死囚");
        strUrl = "https://read.qidian.com/chapter/W08HMrSPUHj7X4qr8VpWrA2/8W_pmmniqFvM5j8_3RRvhw2/";//爬取的网页
        AllUrl.addUrl(strUrl,1,"我的属性修行人生");
        strUrl = "https://read.qidian.com/chapter/q2B9dFLoeqU3v1oFI-DX8Q2/dsXQ94IHlUZp4rPq4Fd4KQ2/";//爬取的网页
        AllUrl.addUrl(strUrl,1,"这个武圣超有素质");
        for(int i=0;i){
            new CrawlTheWeb().start();
        }

    }
}

大多数人都推荐 实现 Runnable ，但是目前我还用不着，暂时先继承稍微熟悉一点的Thread。