WEB CRAWLER USING JAVA | WEB CRAWLER IN JAVA

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package com.jignesh;

import java.net.*;
import java.io.*;

/**
 *
 * @author jignesh kanjariya
 * mo: +918905388832
 */

public class Crawler {

    public static void main(String[] args) throws Exception {
        String urls[] = new String[1000];
        String url = "http://www.google.co.in";
        int i = 0, j = 0, tmp = 0, total = 0, MAX = 1000;
        int start = 0, end = 0;
        String webpage = Web.getWeb(url);
       
        end = webpage.indexOf("<body");
        for (i = total; i < MAX; i++, total++) {
            start = webpage.indexOf("http://", end);
            if (start == -1) {
                start = 0;
                end = 0;
                try {
                    webpage = Web.getWeb(urls[j++]);
                } catch (Exception e) {
                    System.out.println("******************");
                    System.out.println(urls[j - 1]);
                    System.out.println("Exception caught \n" + e);
                }

                /*logic to fetch urls out of body of webpage only */
                end = webpage.indexOf("<body");
                if (end == -1) {
                    end = start = 0;
                    continue;
                }
            }
            end = webpage.indexOf("\"", start);
            tmp = webpage.indexOf("'", start);
            if (tmp < end && tmp != -1) {
                end = tmp;
            }
            url = webpage.substring(start, end);
            urls[i] = url;
            System.out.println(urls[i]);
        }
        System.out.println("Total URLS Fetched are " + total);
    }
}

class Web {

    public static String getWeb(String address) throws Exception {
        String webpage = "";
        String inputLine = "";
        URL url = new URL(address);
        BufferedReader in = new BufferedReader(
                new InputStreamReader(url.openStream()));
        while ((inputLine = in.readLine()) != null) {
            webpage += inputLine;
        }
        in.close();
        return webpage;
    }
}