#!/usr/bin/perl
use strict;
use LWP::UserAgent;
use HTTP::Response;
use URI::Heuristic;
use HTML::Entities;
use Text::Unidecode;
sub get_html {
my $raw_url = shift;
my $url = URI::Heuristic::uf_urlstr($raw_url);
my $ua = LWP::UserAgent->new();
# bogus referrer to perplex the log analyzers
my $response = $ua->get($url, Referer => "http://wizard.yellowbrick.oz");
if ($response->is_error()) {
exit;
}
else {
return $response->content();
}
}
sub get_links_in_post {
my $url = shift;
my $html = get_html($url);
my $post;
my $publishTime;
($publishTime) = $html =~ m{tracking\.publishTime = (.*?)\;}s;
printf ("Published Time: %s\n", $publishTime);
($post) = $html =~ m{By\s+Rupa\s+Subramanya(.*?)Rupa\s+Subramanya}s
or die "Couldn't find post";
$post =~ s/insetContent(.*?)div>//s; # remove unnecessary links to past / relevant posts
my $firstTime = 2; # <ul> and </ul> need to be printed only once each
foreach (split /\n/, $post) {
while (m{
<a\ href="
([^\"]+) # link to doc or external source
\">
([^<]+) # link text in the post
</a>
}gx) {
# printf ("\t%s -> %s\n", $2, $1);
print "<ul>\n" if (($firstTime==2) && $firstTime--);
printf ("<li>%s -> <a href=\"%s\">%s</a></li>\n", $2, $1, $1);
}
}
print "</ul>\n" if (($firstTime==1) && $firstTime--);
}
my $base_url = "http://blogs.wsj.com/indiarealtime/tag/Economics-Journal/page/";
my $counter = 0;
my $html;
while (1) {
$counter++;
my $raw_url = sprintf "%s%i/", $base_url, $counter;
$html = get_html($raw_url);
foreach (split /\n/, $html) {
if (/(href)(.*?)Economics Journal:/) {
s/Economics Journal: //; # I don't know why every post has this prefix
m{
<a\ href="
([^\"]+) # link to blog = $1 = everything to next quote
\">
([^<]+) # blog title = $2 = everything up to </a>
</a>}gx;
# print "=============================\n";
print "<hr/>\n";
# printf("%s -> %s ", unidecode(decode_entities($2)), $1);
printf("%s -> <a href=\"%s\">%s</a>\n<br/>\n", unidecode(decode_entities($2)), $1, $1);
get_links_in_post($1);
}
}
}