Agricultural Knowledge Graph

项目介绍:

目录结构:

.
├── MyCrawler // scrapy爬虫项目路径(已爬好)
│  └── MyCrawler
│  ├── data
│  └── spiders
├── data\ processing // 数据清洗(已无用)
│  └── data
├── demo // django项目路径
│  ├── Model // 模型层,用于封装Item类,以及neo4j和csv的读取
│  ├── demo // 用于写页面的逻辑(View)
│  ├── label_data // 标注训练集页面的保存路径
│  │  └── handwork
│  ├── static // 静态资源
│  │  ├── css
│  │  ├── js
│  │  └── open-iconic
│  ├── templates // html页面
│  └── toolkit // 工具库,包括预加载,命名实体识别
│  └── KNN_predict 
├── KNN_predict // KNN算法预测标签
├── dfs_tree_crawler // 爬取互动百科农业实体树形结构的爬虫
└── wikidataSpider // 爬取wiki中的关系

可复用资源

项目配置

// 将hudong_pedia.csv 导入
LOAD CSV WITH HEADERS FROM "file:///hudong_pedia.csv" AS line 
CREATE (p:HudongItem{title:line.title,image:line.image,detail:line.detail,url:line.url,openTypeList:line.openTypeList,baseInfoKeyList:line.baseInfoKeyList,baseInfoValueList:line.baseInfoValueList}) 
// 新增了hudong_pedia2.csv
LOAD CSV WITH HEADERS FROM "file:///hudong_pedia2.csv" AS line 
CREATE (p:HudongItem{title:line.title,image:line.image,detail:line.detail,url:line.url,openTypeList:line.openTypeList,baseInfoKeyList:line.baseInfoKeyList,baseInfoValueList:line.baseInfoValueList}) 

// 创建索引
CREATE CONSTRAINT ON (c:HudongItem)
ASSERT c.title IS UNIQUE

// 导入新的节点
LOAD CSV WITH HEADERS FROM "file:///new_node.csv" AS line
CREATE (:NewNode { title: line.title })
//添加索引
CREATE CONSTRAINT ON (c:NewNode)
ASSERT c.title IS UNIQUE
//导入hudongItem和新加入节点之间的关系
LOAD CSV WITH HEADERS FROM "file:///wikidata_relation2.csv" AS line
MATCH (entity1:HudongItem{title:line.HudongItem}) , (entity2:NewNode{title:line.NewNode})
CREATE (entity1)-[:RELATION { type: line.relation }]->(entity2)
LOAD CSV WITH HEADERS FROM "file:///wikidata_relation.csv" AS line
MATCH (entity1:HudongItem{title:line.HudongItem1}) , (entity2:HudongItem{title:line.HudongItem2})
CREATE (entity1)-[:RELATION { type: line.relation }]->(entity2)

LOAD CSV WITH HEADERS FROM "file:///attributes.csv" AS line
MATCH (entity1:HudongItem{title:line.Entity}), (entity2:HudongItem{title:line.Attribute})
CREATE (entity1)-[:RELATION { type: line.AttributeName }]->(entity2);
 
LOAD CSV WITH HEADERS FROM "file:///attributes.csv" AS line
MATCH (entity1:HudongItem{title:line.Entity}), (entity2:NewNode{title:line.Attribute})
CREATE (entity1)-[:RELATION { type: line.AttributeName }]->(entity2);
 
LOAD CSV WITH HEADERS FROM "file:///attributes.csv" AS line
MATCH (entity1:NewNode{title:line.Entity}), (entity2:NewNode{title:line.Attribute})
CREATE (entity1)-[:RELATION { type: line.AttributeName }]->(entity2);
 
LOAD CSV WITH HEADERS FROM "file:///attributes.csv" AS line
MATCH (entity1:NewNode{title:line.Entity}), (entity2:HudongItem{title:line.Attribute})
CREATE (entity1)-[:RELATION { type: line.AttributeName }]->(entity2) 
//我们建索引的时候带了label,因此只有使用label时才会使用索引,这里我们的实体有两个label,所以一共做2*2=4次。当然,可以建立全局索引,即对于不同的label使用同一个索引
 
 
 

//导入节点
LOAD CSV WITH HEADERS FROM "file:///static_weather_list.csv" AS line
MERGE (:Weather { title: line.title })
//添加索引
CREATE CONSTRAINT ON (c:Weather)
ASSERT c.title IS UNIQUE

将wikidataSpider/weatherData/weather_plant.csv放在指定的位置(import文件夹下)
//导入hudongItem和新加入节点之间的关系
LOAD CSV WITH HEADERS FROM "file:///weather_plant.csv" AS line
MATCH (entity1:Weather{title:line.Weather}) , (entity2:HudongItem{title:line.Plant})
CREATE (entity1)-[:Weather2Plant { type: line.relation }]->(entity2)
导入城市的气候
将city_weather.csv放在指定的位置(import 文件夹下)
(这步大约需要15分钟左右)
//导入城市对应的气候
LOAD CSV WITH HEADERS FROM "file:///city_weather.csv" AS line
MATCH (city{title:line.city}) , (weather{title:line.weather})
CREATE (city)-[:CityWeather { type: line.relation }]->(weather)

sudo sh django_server_start.sh

(update 2018年11月11日)

(update 2018年10月26日)

农业实体识别+实体分类

实体查询

关系查询

知识的树形结构

训练集标注

思路

命名实体识别:

实体分类:

特征提取:

分类器:KNN算法

定义两个页面的相似度sim(p1,p2):

Labels:(命名实体的分类)

关系抽取