use LWP::UserAgent;
use autodie qw/ open close /;
use WWW::Mechanize;
use LWP::Simple qw($ua get);
use Encode;
$ua->agent('My agent/1.0');
# create WWW::Mechanize object
# autocheck 1 checks each request to ensure it was successful
my $mech = WWW::Mechanize->new();
$filename = "ilbe.txt";
if(-e $filename)
{
unlink($filename);
}
$mech->get("http://www.ilbe.com/ilbe");
@link = $mech->find_all_links( url_regex => qr/\d{10}/);
for $new_link (@link){
$url = $new_link->url_abs();
$no = $url;
$no =~ m/\d{10}/;
$no = $&;
$mech->get($url);
$cont = $mech->content();
print $no." article is extracted...\n";
@files=();
# paste from url of webbrowser manually later modify
#@temp = split(/og:url/, $cont);
#@temp = split(/science&no=/, $temp[1]);
#@temp = split(/&page/, $temp[1]);
#$no = $temp[0];
$cont=encode("euc-kr", $cont);
@temp = split(/author/, $cont);
@temp = split(/<\/div>/, $temp[1]);
@temp = split(/<\/div>/, $temp[0]);
@temp = split(/margin-right:3px;/, $temp[0]);
$name = $temp[1];
$name =~ s/"//;
$name =~ s/\/>//;
$name =~ s/^\s+//;
@temp = split(/<\/span>/, $name);
$name =~ s/\s+$//;
$name = $temp[0];
#print $name;
@temp = split(/기사 제목/, $cont);
@temp = split(/-->/, $temp[1]);
@temp = split(/<!--/, $temp[1]);
$subject = $temp[0];
print $subject;
@temp = split(/기사 본문/, $cont);
@temp = split(/serverLog/, $temp[1]);
$memo = $temp[0];
$memo =~ s/-->//;
$memo =~ s/^\s+//;
#print $memo;
@img = $mech->find_all_links( text_regex => qr/png/i );
$i = 0;
for my $img_url (@img) {
$i++;
print $img_url->url_abs;
$mech->mirror($img_url->url_abs, $no."-".$i.".png");
push (@files, $no."-".$i.".png");
}
@img = $mech->find_all_links( text_regex => qr/jpg/i );
for my $img_url (@img) {
$i++;
print $img_url->url_abs;
$mech->mirror($img_url->url_abs, $no."-".$i.".jpg");
push (@files, $no."-".$i.".jpg");
#Opens FROM_DIR, ao I can read from it
}
$attached = "file:";
for ($i=0; $i<=$#files; $i++){
$attached = $attached.$files[$i]." ";
}
$name = "name: ".$name;
$subject = "subject: ".$subject;
$memo = "memo: ".$memo;
open MYFILE, ">>$filename" or die ;
print MYFILE "$no";
print MYFILE "------------------------------------------------------------------------------\n";
print MYFILE "$name\n";
print MYFILE "$subject\n";
print MYFILE "$memo\n";
print MYFILE "$attached\n";
}
댓글 없음:
댓글 쓰기