PulsarRPA 简介

// Create a pulsar session
val session = PulsarContexts.createSession()
// The main url we are playing with
val url = "https://www.amazon.com/dp/B09V3KXJPB"

// Load a page from local storage, or fetch it from the Internet if it does not exist or has expired
val page = session.load(url, "-expires 10s")

// Submit a url to the URL pool, the submitted url will be processed in a crawl loop
session.submit(url, "-expires 10s")

// Parse the page content into a document
val document = session.parse(page)
// do something with the document
// ...

// Load and parse
val document2 = session.loadDocument(url, "-expires 10s")
// do something with the document
// ...

// Load the portal page and then load all links specified by `-outLink`.
// Option `-outLink` specifies the cssSelector to select links in the portal page to load.
// Option `-topLinks` specifies the maximal number of links selected by `-outLink`.
val pages = session.loadOutPages(url, "-expires 10s -itemExpires 10s -outLink a[href~=/dp/] -topLinks 10")

// Load the portal page and submit the out links specified by `-outLink` to the URL pool.
// Option `-outLink` specifies the cssSelector to select links in the portal page to submit.
// Option `-topLinks` specifies the maximal number of links selected by `-outLink`.
session.submitOutPages(url, "-expires 1d -itemExpires 7d -outLink a[href~=/dp/] -topLinks 10")

// Load, parse and scrape fields
val fields = session.scrape(url, "-expires 10s", "#centerCol",
 listOf("#title", "#acrCustomerReviewText"))

// Load, parse and scrape named fields
val fields2 = session.scrape(url, "-i 10s", "#centerCol",
 mapOf("title" to "#title", "reviews" to "#acrCustomerReviewText"))

// Load, parse and scrape named fields
val fields3 = session.scrapeOutPages(url, "-i 10s -ii 10s -outLink a[href~=/dp/] -topLink 10", "#centerCol",
 mapOf("title" to "#title", "reviews" to "#acrCustomerReviewText"))

// Add `-parse` option to activate the parsing subsystem
val page10 = session.load(url, "-parse -expires 10s")

// Kotlin suspend calls
val page11 = runBlocking { session.loadDeferred(url, "-expires 10s") }

// Java-style async calls
session.loadAsync(url, "-expires 10s").thenApply(session::parse).thenAccept(session::export)

val document = session.loadDocument(url, "-expires 1d")
val price = document.selectFirst('.price').text()

fun main() {
 val context = PulsarContexts.create()

 val parseHandler = { _: WebPage, document: FeaturedDocument ->
 // do something wonderful with the document
 System.out.println(document.getTitle() + "\t|\t" + document.getBaseUri());
 }
 val urls = LinkExtractors.fromResource("seeds.txt")
 .map { ParsableHyperlink("$it -refresh", parseHandler) }
 context.submitAll(urls)
 // feel free to submit millions of urls here
 context.submitAll(urls)
 // ...
 context.await()
}

public class ContinuousCrawler {

 private static void onParse(WebPage page, FeaturedDocument document) {
 // do something wonderful with the document
 System.out.println(document.getTitle() + "\t|\t" + document.getBaseUri());
 }

 public static void main(String[] args) {
 PulsarContext context = PulsarContexts.create();

 List<Hyperlink> urls = LinkExtractors.fromResource("seeds.txt")
 .stream()
 .map(seed -> new ParsableHyperlink(seed, ContinuousCrawler::onParse))
 .collect(Collectors.toList());
 context.submitAll(urls);
 // feel free to submit millions of urls here
 context.submitAll(urls);
 // ...
 context.await();
 }
}

val options = session.options(args)
val event = options.event.browseEvent
event.onBrowserLaunched.addLast { page, driver ->
 // 预热浏览器,以避免被网站阻止,或选择全局设置,例如您的位置
 warnUpBrowser(page, driver)
}
event.onWillFetch.addLast { page, driver ->
 // 必须先访问引荐来源页面,然后才能访问所需页面
 waitForReferrer(page, driver)
 // 网站可能会阻止我们同时打开过多页面,因此我们应该逐一打开链接
 waitForPreviousPage(page, driver)
}
event.onWillCheckDocumentState.addLast { page, driver ->
 // 等待特殊字段出现在页面上
 driver.waitForSelector("body h1[itemprop=name]")
 // 关闭遮罩层,它可能是促销、广告或其他东西
 driver.click(".mask-layer-close-button")
}
// 访问 URL 并触发事件
session.load(url, options)

select
 dom_first_text(dom, '#productTitle') as title,
 dom_first_text(dom, '#bylineInfo') as brand,
 dom_first_text(dom, '#price tr td:matches(^Price) ~ td, #corePrice_desktop tr td:matches(^Price) ~ td') as price,
 dom_first_text(dom, '#acrCustomerReviewText') as ratings,
 str_first_float(dom_first_text(dom, '#reviewsMedley .AverageCustomerReviews span:contains(out of)'), 0.0) as score
 from load_and_select('https://www.amazon.com/dp/B09V3KXJPB -i 1s -njr 3', 'body');

val context = SQLContexts.create()
val rs = context.executeQuery(sql)
println(ResultSetFormatter(rs, withHeader = true))

bin/pulsar

bin/scrape.sh

curl -X POST --location "http://localhost:8182/api/x/e" -H "Content-Type: text/plain" -d "
 select
 dom_base_uri(dom) as url,
 dom_first_text(dom, '#productTitle') as title,
 str_substring_after(dom_first_href(dom, '#wayfinding-breadcrumbs_container ul li:last-child a'), '&node=') as category,
 dom_first_slim_html(dom, '#bylineInfo') as brand,
 cast(dom_all_slim_htmls(dom, '#imageBlock img') as varchar) as gallery,
 dom_first_slim_html(dom, '#landingImage, #imgTagWrapperId img, #imageBlock img:expr(width > 400)') as img,
 dom_first_text(dom, '#price tr td:contains(List Price) ~ td') as listprice,
 dom_first_text(dom, '#price tr td:matches(^Price) ~ td') as price,
 str_first_float(dom_first_text(dom, '#reviewsMedley .AverageCustomerReviews span:contains(out of)'), 0.0) as score
 from load_and_select('https://www.amazon.com/dp/B09V3KXJPB -i 1d -njr 3', 'body');"

{
"uuid":"cc611841-1f2b-4b6b-bcdd-ce822d97a2ad",
"statusCode":200,
"pageStatusCode":200,
"pageContentBytes":1607636,
"resultSet":[
{
"title":"Tara Toys Ariel Necklace Activity Set - Amazon Exclusive (51394)",
"listprice":"19ドル.99",
"price":"12ドル.99",
"categories":"Toys & Games|Arts & Crafts|Craft Kits|Jewelry",
"baseuri":"https://www.amazon.com/dp/B09V3KXJPB"
}
],
"pageStatus":"OK",
"status":"OK"
}