2014년 5월 30일 금요일

perl mechanize example

use LWP::UserAgent;
use autodie qw/ open close /;
use WWW::Mechanize;
use LWP::Simple qw($ua get);
use Encode;


$ua->agent('My agent/1.0');

# create WWW::Mechanize object
# autocheck 1 checks each request to ensure it was successful
my $mech = WWW::Mechanize->new();

$filename = "ilbe.txt";
if(-e $filename)
    {
        unlink($filename);
    }
   
$mech->get("http://www.ilbe.com/ilbe");

@link = $mech->find_all_links( url_regex => qr/\d{10}/);

for $new_link (@link){

$url = $new_link->url_abs();

$no = $url;
$no =~ m/\d{10}/;

$no = $&;

$mech->get($url);
$cont = $mech->content();

print $no." article is extracted...\n";
@files=();

# paste from url of webbrowser manually later modify

#@temp = split(/og:url/, $cont);
#@temp = split(/science&no=/, $temp[1]);
#@temp = split(/&page/, $temp[1]);


#$no = $temp[0];

$cont=encode("euc-kr", $cont);

@temp = split(/author/, $cont);
@temp = split(/<\/div>/, $temp[1]);
@temp = split(/<\/div>/, $temp[0]);
@temp = split(/margin-right:3px;/, $temp[0]);
$name = $temp[1];
$name =~ s/"//;
$name =~ s/\/>//;
$name =~ s/^\s+//;
@temp = split(/<\/span>/, $name);
$name =~ s/\s+$//;
$name = $temp[0];
#print $name;

@temp = split(/기사 제목/, $cont);
@temp = split(/-->/, $temp[1]);
@temp = split(/<!--/, $temp[1]);

$subject = $temp[0];

print $subject;


@temp = split(/기사 본문/, $cont);
@temp = split(/serverLog/, $temp[1]);

$memo = $temp[0];
$memo =~ s/-->//;
$memo =~ s/^\s+//;

#print $memo;



@img = $mech->find_all_links( text_regex => qr/png/i );

$i = 0;
for my $img_url (@img) {
    $i++;
    print $img_url->url_abs;
    $mech->mirror($img_url->url_abs, $no."-".$i.".png");
    push (@files, $no."-".$i.".png");
   
}

@img = $mech->find_all_links( text_regex => qr/jpg/i );
for my $img_url (@img) {
    $i++;
    print $img_url->url_abs;
    $mech->mirror($img_url->url_abs, $no."-".$i.".jpg");
    push (@files, $no."-".$i.".jpg");
   
    #Opens FROM_DIR, ao I can read from it
   
       
}

$attached = "file:";
for ($i=0; $i<=$#files; $i++){
$attached = $attached.$files[$i]." ";

}

$name = "name: ".$name;
$subject = "subject: ".$subject;
$memo = "memo: ".$memo;


open MYFILE, ">>$filename" or die ;
 print MYFILE "$no";
 print MYFILE "------------------------------------------------------------------------------\n";
 print MYFILE "$name\n";
 print MYFILE "$subject\n";
 print MYFILE "$memo\n";
 print MYFILE "$attached\n";
}

댓글 없음:

댓글 쓰기