Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit c5294a1

Browse files
Added the ability to cache responses.
1 parent 461a4bb commit c5294a1

File tree

7 files changed

+135
-56
lines changed

7 files changed

+135
-56
lines changed

‎ChangeLog.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Change Log
22

3+
### 1.3.0
4+
- Added the `cache-lifespan` parameter that determines the cache duration.
5+
- Added the ability to cache responses.
6+
37
### 1.2.1 - 2018年10月23日
48
- Changed to display HTML outputs regardless of the response status if the content exits.
59

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@ http(s)://{app-address}/?url=https%3A%2F%2Fwww.whatismybrowser.com%2Fdetect%2Fwh
9494
#### output-encoding
9595
Sets the encoding used for the output. Default: `utf8`
9696

97+
#### cache-lifespan
98+
All requests are cached for 20 minutes by default. This detemines how long the cache should be retained. If you do not want a cached result or want to renew the cache, pass `0`. Default: `1200`.
99+
97100
#### headers
98101
Sets a custom HTTP headers. Accepts the value as an array.
99102

‎web/include/class/scraper/ScraperHandler.php‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,15 +83,16 @@ private function ___getClientArguments( array $aRequest ) {
8383
*/
8484
private function ___getBaseArguments( array $aRequest ) {
8585
$_aArguments = array(
86-
'url' => urldecode( trim( $aRequest[ 'url' ] ) ),
87-
'user_agent' => $this->___getUserAgent( $aRequest ),
88-
'headers' => isset( $aRequest[ 'headers' ] ) && is_array( $aRequest[ 'headers' ] )
86+
'url' => urldecode( trim( $aRequest[ 'url' ] ) ),
87+
'user_agent' => $this->___getUserAgent( $aRequest ),
88+
'headers' => isset( $aRequest[ 'headers' ] ) && is_array( $aRequest[ 'headers' ] )
8989
? $aRequest[ 'headers' ]
9090
: array(),
91-
'method' => isset( $aRequest[ 'method' ] )
91+
'method' => isset( $aRequest[ 'method' ] )
9292
? strtoupper( $aRequest[ 'method' ] )
9393
: 'GET',
94-
'binary_path' => $this->_sBinaryPath,
94+
'binary_path' => $this->_sBinaryPath,
95+
'cache_lifespan' => isset( $aRequest[ 'cache-lifespan' ] ) ? $aRequest[ 'cache-lifespan' ] : 1200,
9596
);
9697
return $_aArguments;
9798
}

‎web/include/class/scraper/Scraper_Base.php‎

Lines changed: 59 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,24 @@
88

99
class Scraper_Base extends Utility {
1010

11+
/**
12+
* Override this value in an extended class.
13+
* Used to construct a cache directory.
14+
* @var string
15+
*/
16+
protected $_sType = '';
17+
18+
protected $_sCacheDirPath = '';
19+
20+
protected $_sCacheDirPathToday = '';
21+
1122
protected $_aBaseArguments = array(
1223
'url' => '',
1324
'user_agent' => '',
1425
'method' => '',
1526
'headers' => array(),
1627
'binary_path' => '', // PhantomJS binary path
28+
'cache_lifespan' => 1200, // cache life span in seconds. 1200: 20 minutes,
1729
);
1830

1931
protected $_aClientArguments = array(
@@ -24,20 +36,63 @@ class Scraper_Base extends Utility {
2436
);
2537

2638
protected $_aRequestArguments = array(
27-
'method' => '',
28-
'file-type' => 'jpg',
29-
'data' => array(),
39+
'method' => '',
40+
'file-type' => 'jpg',
41+
'data' => array(),
3042
);
3143

3244
public function __construct( array $aBaseArguments, array $aClientArguments, array $aRequestArguments ) {
45+
3346
$this->_aBaseArguments = $aBaseArguments + $this->_aBaseArguments;
3447
$this->_aClientArguments = $aClientArguments + $this->_aClientArguments;
3548
$this->_aRequestArguments = $aRequestArguments + $this->_aRequestArguments;
49+
50+
$this->___setCacheDirectory();
51+
3652
$this->_construct();
3753
}
3854

55+
public function do() {}
56+
3957
protected function _construct() {}
4058

41-
public function do() {}
59+
/**
60+
* Returns a request specific cache name.
61+
* Even if the request URI is the same, if it is a POST request and the query is different, the cached data shouold be different,
62+
* Thus, a different cache name is given by request arguments.
63+
* @return string
64+
*/
65+
protected function _getRequestCacheName() {
66+
$_aBaseArguments = $this->_aRequestArguments;
67+
unset( $_aBaseArguments[ 'cache_lifespan' ] ); // must be removed
68+
return md5(
69+
serialize(
70+
array(
71+
$_aBaseArguments,
72+
$this->_aClientArguments,
73+
$this->_aRequestArguments
74+
)
75+
)
76+
);
77+
}
78+
79+
private function ___setCacheDirectory() {
80+
81+
$this->_sCacheDirPath = Registry::$sTempDirPath . "/{$this->_sType}/";
82+
$_sToday = date("Ymd" );
83+
$this->_sCacheDirPathToday = $this->_sCacheDirPath . $_sToday;
84+
if ( ! file_exists( $this->_sCacheDirPathToday ) ) {
85+
mkdir( $this->_sCacheDirPathToday, 0777, true );
86+
}
87+
88+
// Delete old cache directories.
89+
// Directories other than todays will be deleted. This means the cache lifespan cannot be longer than a day.
90+
$_aSubDirs = $this->getSubDirPaths( $this->_sCacheDirPath, array( $_sToday ) );
91+
foreach( $_aSubDirs as $_sDirPath ) {
92+
$this->deleteDir( $_sDirPath );
93+
}
94+
95+
}
96+
4297

4398
}

‎web/include/class/scraper/Scraper_html.php‎

Lines changed: 38 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,29 +4,48 @@
44
*/
55
class Scraper_html extends Scraper_Base {
66

7+
protected $_sType = 'html';
8+
79
public function do() {
810

9-
$_oBrowser = new Browser(
10-
$this->_aBaseArguments[ 'binary_path' ],
11-
$this->_aBaseArguments[ 'user_agent' ],
12-
$this->_aBaseArguments[ 'headers' ],
13-
$this->_aClientArguments
14-
);
15-
$_oBrowser->setRequestArguments( $this->_aRequestArguments );
16-
17-
$_oResponse = $_oBrowser->get( $this->_aBaseArguments[ 'url' ] );
18-
$_sContent = $_oResponse->getContent();
19-
// $_oResponse->getStatus()
20-
if ( $_sContent ) {
21-
echo $_oResponse->getContent(); // Dump the requested page content
22-
return;
23-
} else {
24-
echo "<h2>Failed to Get Response</h2>";
25-
echo "<pre>";
26-
echo htmlspecialchars( print_r( $_oResponse, true ) );
27-
echo "</pre>";
11+
// Check cache
12+
$_sFileBaseName = $this->_getRequestCacheName() . ".{$this->_sType}";
13+
$_sFilePath = $this->_sCacheDirPathToday . '/' . $_sFileBaseName;
14+
15+
// Use cache
16+
if ( file_exists( $_sFilePath ) ) {
17+
$_iModifiedTime = ( integer ) filemtime( $_sFilePath );
18+
if ( $_iModifiedTime + ( ( integer ) $this->_aBaseArguments[ 'cache_lifespan' ] ) > time() ) {
19+
readfile( $_sFilePath );
20+
return;
21+
}
2822
}
2923

24+
$_sContent = $this->_getContent();
25+
echo $_sContent;
26+
flush();
27+
file_put_contents( $_sFilePath, $_sContent ); // caching
28+
3029
}
30+
protected function _getContent() {
31+
$_oBrowser = new Browser(
32+
$this->_aBaseArguments[ 'binary_path' ],
33+
$this->_aBaseArguments[ 'user_agent' ],
34+
$this->_aBaseArguments[ 'headers' ],
35+
$this->_aClientArguments
36+
);
37+
$_oBrowser->setRequestArguments( $this->_aRequestArguments );
38+
$_oResponse = $_oBrowser->get( $this->_aBaseArguments[ 'url' ] );
39+
$_sContent = $_oResponse->getContent();
40+
if ( $_oResponse->getStatus()&& $_sContent ) {
41+
return $_sContent;
42+
}
43+
// Error
44+
return "<h2>Failed to Get Response</h2>"
45+
. "<pre>"
46+
. htmlspecialchars( print_r( $_oResponse, true ) )
47+
. "</pre>";
48+
49+
}
3150

3251
}

‎web/include/class/scraper/Scraper_json.php‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22
/**
33
* Displays a response as JSON.
44
*/
5-
class Scraper_json extends Scraper_Base {
5+
class Scraper_json extends Scraper_html {
6+
7+
protected $_sType = 'json';
8+
9+
protected function _getContent() {
610

7-
public function do() {
811
$_oBrowser = new Browser(
912
$this->_aBaseArguments[ 'binary_path' ],
1013
$this->_aBaseArguments[ 'user_agent' ],
@@ -13,7 +16,8 @@ public function do() {
1316
);
1417
$_oBrowser->setRequestArguments( $this->_aRequestArguments );
1518
$_oResponse = $_oBrowser->get( $this->_aBaseArguments[ 'url' ] );
16-
echo json_encode( $_oResponse );
19+
return json_encode( $_oResponse );
20+
1721
}
1822

1923
}

‎web/include/class/scraper/Scraper_screenshot.php‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,10 @@
55
*/
66
class Scraper_screenshot extends Scraper_Base {
77

8-
private$___sTodayDirPath;
8+
protected$_sType = 'screenshot';
99

1010
protected function _construct() {
1111

12-
$this->___setImageCacheDirectory();
13-
1412
// Sanitize parameters
1513
$this->_aClientArguments[ 'load-images' ] = isset( $_REQUEST[ 'load-images' ] ) && ! Utility::getBoolean( $_REQUEST[ 'load-images' ] )
1614
? false
@@ -20,6 +18,23 @@ protected function _construct() {
2018

2119
public function do() {
2220

21+
// Check cache
22+
$_aRequestArguments = $this->_aRequestArguments;
23+
$_sFileType = in_array( $_aRequestArguments[ 'file-type' ], array( 'jpg', 'pdf', 'png', 'jpeg', 'bmp', 'ppm' ) )
24+
? $_aRequestArguments[ 'file-type' ]
25+
: 'jpg';
26+
$_sFileBaseName = $this->_getRequestCacheName() . ".{$_sFileType}";
27+
$_sFilePath = $this->_sCacheDirPathToday . '/' . $_sFileBaseName;
28+
29+
// Use cache
30+
if ( file_exists( $_sFilePath ) ) {
31+
$_iModifiedTime = ( integer ) filemtime( $_sFilePath );
32+
if ( $_iModifiedTime + ( ( integer ) $this->_aBaseArguments[ 'cache_lifespan' ] ) > time() ) {
33+
$this->___render( $_sFilePath );
34+
return;
35+
}
36+
}
37+
2338
$_oScreenCapture = new ScreenCapture(
2439
$this->_aBaseArguments[ 'binary_path' ],
2540
$this->_aBaseArguments[ 'user_agent' ],
@@ -28,12 +43,6 @@ public function do() {
2843
);
2944

3045
/// Request Arguments
31-
$_aRequestArguments = $this->_aRequestArguments;
32-
$_sFileType = in_array( $_aRequestArguments[ 'file-type' ], array( 'jpg', 'pdf', 'png', 'jpeg', 'bmp', 'ppm' ) )
33-
? $_aRequestArguments[ 'file-type' ]
34-
: 'jpg';
35-
$_sFileBaseName = md5( $this->_aBaseArguments[ 'url' ] ) . ".{$_sFileType}";
36-
$_sFilePath = $this->___sTodayDirPath . '/' . $_sFileBaseName; // $_sFilePath = Registry::$sDirPath . '/_capture/file.jpg';
3746
$_aRequestArguments[ 'file_path' ] = $_sFilePath;
3847
$_aRequestArguments[ 'file_type' ] = $_sFileType;
3948
$_oScreenCapture->setRequestArguments( $_aRequestArguments );
@@ -45,22 +54,6 @@ public function do() {
4554
$this->___render( $_sFilePath );
4655

4756
}
48-
private function ___setImageCacheDirectory() {
49-
50-
$_sToday = date("Ymd" );
51-
$_sTodayDirPath = Registry::$sTempDirPath . '/capture/' . $_sToday;
52-
if ( ! file_exists( $_sTodayDirPath ) ) {
53-
mkdir( $_sTodayDirPath, 0777, true );
54-
}
55-
$this->___sTodayDirPath = $_sTodayDirPath;
56-
57-
// Delete old cache directories
58-
$_aSubDirs = $this->getSubDirPaths( Registry::$sTempDirPath . '/capture/', array( $_sToday ) );
59-
foreach( $_aSubDirs as $_sDirPath ) {
60-
$this->deleteDir( $_sDirPath );
61-
}
62-
63-
}
6457

6558
private function ___render( $sFilePath ) {
6659
$_aImageInfo = getimagesize( $sFilePath );

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /