Java|分页爬取表情包图片

//引入的包，略

public  class GetPicture1 implements PageProcessor {

    //定义一个变量用来表示需要下载的总页数

    private static int size = 3;

    //定义一个变量用来表示起始页

    private static int number = 1;

    //声明一个用来存储需要下载的url路径的集合

    private static List<String> listUrl  = new ArrayList<String>();

    //对所要爬取的页面进行相关设置

    private Site site = Site.me()

            .setCharset("utf-8")

            .setSleepTime(1000)//休眠时间

            .setTimeOut(1000);//超时时间

    @Override

    public Site getSite() { return site; }

    //爬取数据的逻辑

    @Override

    public void process(Page page) {

        //获取url

        Selectable url = page.getUrl();

        //url匹配

         if(url.regex("https://www.doutula.com/article/list/\\?page=\\d*").match()){//列表页

            //获取页面

            Html html = page.getHtml();

            //解析页面获取相关信息(获取所有的标题url)

            List<String> urls =  html.xpath("[@class='container_']/div[1]/div[1]/div[2]/a").links().all();

            if(number > size + 1){

                return;

            }

            urls.add(listUrl.get(number -  1));

            //下一页

            number++;

            //将连接放入待爬取序列

            page.addTargetRequests(urls);

        }else{

            //爬取图片   获取页面

            Html html = page.getHtml();

            //获取表情包组的名称

            String title =  html.xpath("[@class='container_']/div[1]/div[1]/div[2]/li/div[1]/h1/a/text()").toString();

            //获取表情包图片的链接//

            List<String> pictureUrl =  html.xpath("[@class='container_']/div[1]/div[1]/div[2]/li/div[2]/div/table/tbody/tr/td[1]/a/img/@src").all();

            //下载到本地

            downPicture(pictureUrl,title);

        }

    }

    //将图片下载到本地

    private void  downPicture(List<String> pictureUrl, String title) {

        for(int  i=0;i<pictureUrl.size();i++){

            //获取每一张图片连接

            String link = pictureUrl.get(i);

            //做一个非空判断

            if(link == null || link ==  ""){

                return;//结束

            }

            try {

                //将连接字符串封装成一个URL对象

                URL url = new URL(link);

                //获取网络连接

                HttpURLConnection conn =  (HttpURLConnection)url.openConnection();

                //获取一个输入流

                InputStream in =  conn.getInputStream();

                //指定图片目录存储的位置

                File file = new  File("D:\\doutula\\" + title);

                //判断目录是否存在

                if(! file.exists()){

                    //创建多级目录

                    file.mkdirs();

                }

                //自定图片的位置

                File file2 = new File("D:\\doutula\\"  + title + "\\" + i + ".jpg");

                //输出流

                FileOutputStream fos = new  FileOutputStream(file2);

                ByteArrayOutputStream  outStream = new ByteArrayOutputStream();

                //定义一个缓冲区

                byte[] buf = new byte[1024];

                //定义一个标记用于判断有没有读完

                int len = 0;

                //循环读取

                while((len = in.read(buf)) !=  -1){

                     outStream.write(buf,0,len);

                }

                System.out.println("下载完毕");

                //写出到本地

                 fos.write(outStream.toByteArray());

                //关闭资源

                in.close();

                outStream.close();

                fos.close();

            } catch (Exception e) {

                e.printStackTrace();

            }

        }

    }

    //生成所有下载页的url列表，通过size控制的页数

    public static List<String>  listUrl(int num){

        List<String> list = new  ArrayList<String>();

        //循环往列表中添加url

        for(int i=1;i<=num;i++){

             list.add("https://www.doutula.com/article/list/?page=" +  (i+1));

        }

        return list;

    }

    //主程序的入口、线程

    public static void main(String[] args) {

        //获取所有需要下载页的url

        listUrl = listUrl(size);

        Spider.create(new  GetPicture1()).thread(1).addUrl("https://www.doutula.com/article/list/?page=1").run();

    }

}