1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46
|
my $ToTreat = {};
# Retrieve url
my $url = undef;
if ( exists $response->{'_previous'} ) {
$url = $response->{'_previous'}{'_request'}{'_uri'}->as_string;
}
else {
$url = $response->request->url->as_string;
}
$ToTreat->{'_response'} = $response;
$ToTreat->{'_url'} = $url;
$ToTreat->{'_todo'} = $Queue{$url};
$ToTreat->{'_last_modified'} = defined $response->header('last-modified') ? str2time($response->header('last-modified')) : time();
# Check content-type
my $content_type = undef;
if ( ref($response->{'_headers'}{'content-type'}) eq "ARRAY" )
{
my $type = $response->{'_headers'}{'content-type'}[0];
$type = s/(.*);//;
$content_type = $1;
}
else
{
$content_type = $response->{'_headers'}{'content-type'};
}
$content_type = $self->_content_type($content_type);
$ToTreat->{'_content_type'} = defined $content_type ? $content_type : "text-html";
# Generate sha-1 name
$ToTreat->{'_name'} = sha1_hex($url);
# Write data
$self->_write_data($ToTreat->{'_name'},$ToTreat);
# Delete link from data queue
delete $Queue{$url}; |
Partager