Day25/cnblogs_html2md.py · ForkProject/从零学Python

代码拉取完成,页面将自动刷新

扫描微信二维码支付

取消

支付完成

richgiteeai

Watch

不关注关注所有动态仅关注版本发行动态关注但不提醒动态

1 Star 0 Fork 2.5K

ForkProject/从零学Python

forked from 小柒2012商城/从零学Python

代码 Issues 0 Pull Requests 0 Wiki 统计流水线

服务

加入 Gitee

与超过 1400万开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)

免费加入

已有帐号? 立即登录

文件

master

分支 (2)

管理

master

备份

克隆/下载

HTTPS SSH SVN SVN+SSH 下载ZIP

提示

下载代码请复制以下命令到终端执行

为确保你提交的代码身份被 Gitee 正确识别,请执行以下命令完成配置

git config --global user.name userName 
git config --global user.email userEmail

初次使用 SSH 协议进行代码克隆、推送等操作时,需按下述提示完成 SSH 配置

1 生成 RSA 密钥

2 获取 RSA 公钥内容,并配置到 SSH公钥中

在 Gitee 上使用 SVN,请访问使用指南

使用 HTTPS 协议时,命令行会出现如下账号密码验证步骤。基于安全考虑,Gitee 建议配置并使用私人令牌替代登录密码进行克隆、推送等操作

Username for 'https://gitee.com': userName

Password for 'https://userName@gitee.com': # 私人令牌

分支 2

标签 0

Python

Day25

cnblogs_html2md.py

cnblogs_html2md.py 3.12 KB

# -*- coding: utf-8 -*-
import codecs
import sys
import tomd
import random
import requests
from bs4 import BeautifulSoup
import mysql_DBUtils
from mysql_DBUtils import MyPymysqlPool

blog_headers = [
 "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
 "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
 "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
 'Opera/9.25 (Windows NT 5.1; U; en)',
 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
 'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
 "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
 "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0",
 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36'
]

# 定义DB
mysql = MyPymysqlPool("dbMysql")

def reptile():
 page = 2
 for n in range(1, page):
 url = "https://www.cnblogs.com/yjmyzz/default.html?page=" + str(n)
 headers = {'User-Agent': random.choice(blog_headers)}
 res = requests.get(url, headers=headers)
 soup = BeautifulSoup(res.text, 'html.parser')
 count = soup.find_all(class_='postTitle')
 for c in count:
 try:
 # 获取博客地址
 href = c.find('a').attrs['href']
 headers = {'User-Agent': random.choice(blog_headers)}
 res = requests.get(href, headers=headers)
 soup = BeautifulSoup(res.text, 'html.parser')
 # 获取博客内容
 content = soup.find('div', class_='blogpost-body')
 # 去掉博客外层的DIV
 content = content.decode_contents(formatter="html")
 # 获取博客标题
 title = soup.find('a', id='cb_post_title_url').text
 # 博客HTML转MD
 content = tomd.Tomd(content).markdown
 # 博客写入数据库
 write_db(title, content, href)
 print("已插入:{}".format(href))
 except Exception as e:
 print(e)

# 写入数据库
def write_db(title, content, url):
 sql = "INSERT INTO blog (title, content,url) VALUES(%(title)s, %(content)s, %(url)s);"
 param = {"title": title, "content": content, "url": url}
 mysql.insert(sql, param)

if __name__ == "__main__":
 reptile()

一键复制编辑原始数据按行查看历史

小柒2012商城提交于 2019年08月03日 13:11 +08:00 . :sparkles: 爬取博客文章并入库

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75

# -*- coding: utf-8 -*-
import codecs
import sys
import tomd
import random
import requests
from bs4 import BeautifulSoup
import mysql_DBUtils
from mysql_DBUtils import MyPymysqlPool


blog_headers = [
 "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
 "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
 "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
 'Opera/9.25 (Windows NT 5.1; U; en)',
 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
 'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
 "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
 "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0",
 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36'
]


# 定义DB
mysql = MyPymysqlPool("dbMysql")


def reptile():
 page = 2
 for n in range(1, page):
 url = "https://www.cnblogs.com/yjmyzz/default.html?page=" + str(n)
 headers = {'User-Agent': random.choice(blog_headers)}
 res = requests.get(url, headers=headers)
 soup = BeautifulSoup(res.text, 'html.parser')
 count = soup.find_all(class_='postTitle')
 for c in count:
 try:
 # 获取博客地址
 href = c.find('a').attrs['href']
 headers = {'User-Agent': random.choice(blog_headers)}
 res = requests.get(href, headers=headers)
 soup = BeautifulSoup(res.text, 'html.parser')
 # 获取博客内容
 content = soup.find('div', class_='blogpost-body')
 # 去掉博客外层的DIV
 content = content.decode_contents(formatter="html")
 # 获取博客标题
 title = soup.find('a', id='cb_post_title_url').text
 # 博客HTML转MD
 content = tomd.Tomd(content).markdown
 # 博客写入数据库
 write_db(title, content, href)
 print("已插入:{}".format(href))
 except Exception as e:
 print(e)


# 写入数据库
def write_db(title, content, url):
 sql = "INSERT INTO blog (title, content,url) VALUES(%(title)s, %(content)s, %(url)s);"
 param = {"title": title, "content": content, "url": url}
 mysql.insert(sql, param)


if __name__ == "__main__":
 reptile()