Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit c406926

Browse files
Added web crawling example in LWP, Mechanize and Mojo
1 parent 58863f4 commit c406926

File tree

5 files changed

+132
-9
lines changed

5 files changed

+132
-9
lines changed

‎README.md

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,11 @@ Hopefully will work on older version too (as old as 5.10) and across diferent OS
129129
| Problem 7 | Completed |
130130
| Problem 8 | Completed |
131131

132-
| Web Programming | Status |
133-
| ------------------ | --------- |
134-
| Catalyst |
135-
| Dancer2 |
136-
| Mojolicious |
137-
| Web Crawling <ul><li>Stock Price LWP</li></ul><ul><li>Stock Price Mechanize</li></ul> |
138-
| Currency Converter | Completed |
139-
| Weather | Completed |
132+
| Web Programming | Status |
133+
| ----------------------------------------------------------------------------------------------------------------------- | --------- |
134+
| Catalyst |
135+
| Dancer2 |
136+
| Mojolicious |
137+
| Web Crawling <ul><li>Stock Price LWP</li></ul><ul><li>Stock Price Mechanize</li></ul><ul><li>Stock Price Mojo</li></ul> | Completed |
138+
| Currency Converter <ul><li>LWP::UserAgent</li></ul><ul><li>Mojo::UserAgent</li></ul> | Completed |
139+
| Weather <ul><li>LWP::UserAgent</li></ul><ul><li>Mojo::UserAgent</li></ul> | Completed |

‎web_programming/web_crawling.pl

Lines changed: 0 additions & 1 deletion
This file was deleted.
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/usr/bin/env perl
2+
# https://metacpan.org/pod/WWW::Mechanize
3+
# https://metacpan.org/pod/XML::LibXML
4+
5+
# We can use 'HTML::TreeBuilder' or 'XML::Twig' also here for parsing HTML.
6+
# We are using 'XML::LibXML' just as an example as it can also parse HTML
7+
8+
use strict;
9+
use warnings;
10+
use Carp qw( croak );
11+
use LWP::UserAgent ();
12+
use XML::LibXML;
13+
14+
sub get_stock_price {
15+
my ($ua, $symbol) = @_;
16+
my $url = "https://in.finance.yahoo.com/quote/" . $symbol;
17+
my $response = $ua->get($url);
18+
19+
if ($response->is_success) {
20+
my $dom = XML::LibXML->load_html(
21+
string => $response->decoded_content,
22+
recover => 1,
23+
suppress_errors => 1
24+
);
25+
26+
my $xpath = '//span[contains(@class, "Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)")]';
27+
return $dom->findvalue($xpath);
28+
}
29+
else {
30+
croak $response->status_line;
31+
}
32+
}
33+
34+
sub main {
35+
my $ua = LWP::UserAgent->new();
36+
$ua->agent('Mozilla/5.0');
37+
my $symbol = "AMZN";
38+
my $stock_price = get_stock_price($ua, $symbol);
39+
print "\n Current stock price for $symbol is $stock_price";
40+
}
41+
42+
main();
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/usr/bin/env perl
2+
# https://metacpan.org/pod/WWW::Mechanize
3+
# https://metacpan.org/pod/HTML::TreeBuilder
4+
5+
use strict;
6+
use warnings;
7+
use Carp qw( croak );
8+
use WWW::Mechanize;
9+
use HTML::TreeBuilder;
10+
11+
sub get_stock_price {
12+
my ($mech, $symbol) = @_;
13+
my $url = "https://in.finance.yahoo.com/quote/" . $symbol;
14+
my $response = $mech->get($url);
15+
16+
if ($response->is_success) {
17+
if ($mech->is_html()) {
18+
my $tree = HTML::TreeBuilder->new();
19+
$tree->parse($mech->content);
20+
my $current_stock_price = $tree->look_down(
21+
'_tag' => 'span',
22+
'class' => 'Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)'
23+
)->as_text;
24+
return $current_stock_price;
25+
}
26+
else {
27+
print "\n Not a HTML response";
28+
return 0;
29+
}
30+
}
31+
else {
32+
croak $response->status_line;
33+
}
34+
}
35+
36+
sub main {
37+
my $mech = WWW::Mechanize->new();
38+
$mech->agent_alias("Windows Mozilla");
39+
my $symbol = "AMZN";
40+
my $stock_price = get_stock_price($mech, $symbol);
41+
print "\n Current stock price for $symbol is $stock_price";
42+
}
43+
44+
main();
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env perl
2+
# https://metacpan.org/pod/Mojo::Useragent
3+
# https://metacpan.org/pod/Mojo::DOM
4+
5+
use strict;
6+
use warnings;
7+
use Carp qw( croak );
8+
use Mojo::UserAgent;
9+
10+
sub get_stock_price {
11+
my ($ua, $symbol) = @_;
12+
my $url = "https://in.finance.yahoo.com/quote/" . $symbol;
13+
my $response = $ua->get($url)->result;
14+
15+
if ($response->is_success) {
16+
17+
# https://docs.mojolicious.org/Mojo/DOM#find
18+
my $current_stock_price
19+
= $response->dom->find('span.Trsdu(0\.3s).Fw(b).Fz(36px).Mb(-4px).D(ib)')->map('text')
20+
->join("\n");
21+
22+
return $current_stock_price;
23+
}
24+
else {
25+
croak $response->message;
26+
}
27+
}
28+
29+
sub main {
30+
my $ua = Mojo::UserAgent->new;
31+
$ua->transactor->name('Mozilla/5.0');
32+
$ua = $ua->connect_timeout(25);
33+
my $symbol = "AMZN";
34+
my $stock_price = get_stock_price($ua, $symbol);
35+
print "\n Current stock price for $symbol is $stock_price";
36+
}
37+
38+
main();

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /