Thursday, August 19, 2010

Extract list of urls from a given link

This program will help to extract list of urls from a given link .That is what ever links are there within that link all will be extracted.Example url for images,files,etc.
for these you have to use two api 1)jsoup-1.2.2.jar and 2)cayenne-1.2.1.jar

import org.apache.commons.lang.Validate;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;

import java.net.URL;
import java.io.IOException;
import java.util.Iterator;
import java.util.TreeSet;

/**
* Example program to list links from a URL.
*/
public class ListLinks {
public static void main(String[] args1) throws IOException {

String[] args = {"http://www.mayoclinic.com/health/allergies/DS01118/DSECTION=symptoms"};
Validate.isTrue(args.length == 1, "usage: supply url to fetch");
URL url = new URL(args[0]);
print("Fetching %s...", url.toExternalForm());

Document doc = Jsoup.parse(url, 3*1000);
Elements links = doc.select("a[href]");
Elements media = doc.select("[src]");
Elements imports = doc.select("link[href]");

print("\nMedia: (%d)", media.size());
TreeSet mm = new TreeSet();
for (Element src : media) {
if (src.tagName().equals("img"))
{
print(" * %s: <%s> %sx%s (%s)",
src.tagName(), src.attr("abs:src"), src.attr("width"), src.attr("height"),
trim(src.attr("alt"), 20));
mm.add(src.attr("abs:src"));
}
else
print(" * %s: <%s>", src.tagName(), src.attr("abs:src"));
}

print("\nImports: (%d)", imports.size());
for (Element link : imports)
{
print(" * %s <%s> (%s)", link.tagName(),link.attr("abs:href"), link.attr("rel"));
}

print("\nLinks: (%d)", links.size());
for (Element link : links)
{
print(" * a: <%s> (%s)", link.attr("abs:href"), trim(link.text(), 35));
}

System.out.println("My OutPut");
Iterator ww = mm.iterator();
while(ww.hasNext())
{
System.out.println((String)ww.next());
}

}

private static void print(String msg, Object... args) {
System.out.println(String.format(msg, args));
}

private static String trim(String s, int width) {
if (s.length() > width)
return s.substring(0, width-1) + ".";
else
return s;
}
}

No comments:

Post a Comment