开放域实体抽取泛用工具

基本环境

前期准备

训练(词语统计)

词语自身属性

语境

协议书(5.180388%)[56]
协议(11.84089%)[128]
合同(58.55689%)[633]
合同书(2.960222%)[32]
买卖合同(3.792784%)[41]
承包合同(12.0259%)[130]
意向书(0.2775208%)[3]
补充协议(1.110083%)[12]
项目(0.2775208%)[3]
书(0.9250694%)[10]
议案(0.2775208%)[3]
)(0.8325624%)[9]

 e.LeadingColonKeyWordList = ContractTraning.ContractNameLeadingDict
 .Where((x) => { return x.Value >= 40; }) //阈值40%以上
 .Select((x) => { return x.Key + ":"; }).ToArray();

表格

 /// <summary>
 /// 增发对象训练
 /// </summary>
 public static void TrainingIncreaseTarget()
 {
 var TargetTool = new TableAnlayzeTool();
 var IncreaseNumberTool = new TableAnlayzeTool();
 IncreaseNumberTool.Transform = NumberUtility.NormalizerStockNumber;
 var IncreaseMoneyTool = new TableAnlayzeTool();
 IncreaseMoneyTool.Transform = MoneyUtility.Format;
 TraningDataset.InitIncreaseStock();
 var PreviewId = String.Empty;
 var PreviewRoot = new HTMLEngine.MyRootHtmlNode();
 foreach (var increase in TraningDataset.IncreaseStockList)
 {
 if (!PreviewId.Equals(increase.id))
 {
 var htmlfile = Program.DocBase + @"\FDDC_announcements_round1_train_20180518\定增\html\" + increase.id + ".html";
 PreviewRoot = new HTMLEngine().Anlayze(htmlfile, "");
 PreviewId = increase.id;
 }
 TargetTool.PutTrainingItem(PreviewRoot, increase.PublishTarget);
 IncreaseNumberTool.PutTrainingItem(PreviewRoot, increase.IncreaseNumber);
 IncreaseMoneyTool.PutTrainingItem(PreviewRoot, increase.IncreaseMoney);
 }
 TargetTool.WriteTop(10);
 }
增发对象
17%	00237	发行对象
16%	00223	发行对象名称
11%	00156	股东名称
09%	00132	认购对象
07%	00096	投资者名称
06%	00085	名称
04%	00061	认购对象名称
04%	00055	获配投资者名称
02%	00035	询价对象名称
02%	00029	配售对象名称
增发数量
30%	00370	获配股数(股)
19%	00234	配售股数(股)
13%	00158	认购股数(股)
10%	00126	持股数量(股)
03%	00045	认购数量(股)
02%	00028	持股总数(股)
02%	00024	配售数量(股)
01%	00019	持股数(股)
01%	00015	获配数量(股)
00%	00011	总股本比例
00%	00011	获配股数(万股)
00%	00011	认购股数(万股)
增发金额
35%	00257	获配金额(元)
21%	00155	认购金额(元)
17%	00125	配售金额(元)
08%	00062	配售金额(元)
02%	00018	认购金额(万元)
02%	00017	认购金额(人民币元)
01%	00014	发行前
01%	00014	申购金额(万元)
01%	00011	获配金额(元)
01%	00008	追加认购金额(元)

 /// <summary>
 /// 增减持训练
 /// </summary>
 /// <param name="TraningCnt">训练条数</param>
 public static void Traning(int TraningCnt = int.MaxValue)
 {
 var ChangeMethodTool = new TableAnlayzeTool();
 var PreviewId = String.Empty;
 var PreviewRoot = new HTMLEngine.MyRootHtmlNode();
 int Cnt = 0;
 foreach (var stockchange in TraningDataset.StockChangeList)
 {
 if (!PreviewId.Equals(stockchange.id))
 {
 var htmlfile = Program.DocBase + @"\FDDC_announcements_round1_train_20180518\增减持\html\" + stockchange.id + ".html";
 PreviewRoot = new HTMLEngine().Anlayze(htmlfile, "");
 PreviewId = stockchange.id;
 Cnt++; if (Cnt == TraningCnt) break;
 }
 ChangeMethodTool.PutValueTrainingItem(PreviewRoot, new string[]{"减持方式","增持方式"}.ToList());
 }
 Program.Training.WriteLine("增减持方式");
 ChangeMethodTool.WriteTop(10);
 }
增减持方式
33%	09277	集中竞价交易
24%	06771	集中竞价
21%	05940	大宗交易
08%	02468	竞价交易
01%	00464	集中竞价减持
01%	00365	减持方式
01%	00303	<null>
01%	00289	二级市场竞价
00%	00258	合计
00%	00196	竞价减持

抽取

表格抽取工具(内容系)

 /// <summary>
 /// 表抽取规则(内容系)
 /// </summary>
 public struct TableSearchContentRule
 {
 /// <summary>
 /// 匹配内容
 /// </summary>
 public List<String> Content;
 /// <summary>
 /// 是否相等模式
 /// </summary>
 public bool IsContentEq;
 }

 var rule = new TableSearchContentRule();
 rule.Content = new string[] { "集中竞价交易", "竞价交易", "大宗交易", "约定式购回" }.ToList();
 rule.IsContentEq = true;
 var result = HTMLTable.GetMultiRowsByContentRule(root,rule);

表格抽取工具(表头规则系)

 /// <summary>
 /// 表抽取规则
 /// </summary>
 public struct TableSearchTitleRule
 {
 public string Name;
 /// <summary>
 /// 父标题
 /// </summary>
 public List<String> SuperTitle;
 /// <summary>
 /// 是否必须一致
 /// </summary>
 public bool IsSuperTitleEq;
 /// <summary>
 /// 标题
 /// </summary>
 public List<String> Title;
 /// <summary>
 /// 是否必须一致
 /// </summary>
 public bool IsTitleEq;
 /// <summary>
 /// 是否必须
 /// </summary>
 public bool IsRequire;
 /// <summary>
 /// 表标题不能包含的文字
 /// </summary>
 public List<String> ExcludeTitle;
 /// <summary>
 /// 抽取内容预处理器
 /// </summary>
 public Func<String, String, String> Normalize;
 }

 var HoldList = new List<struHoldAfter>();
 var StockHolderRule = new TableSearchRule();
 StockHolderRule.Name = "股东全称";
 StockHolderRule.Title = new string[] { "股东名称", "名称", "增持主体", "增持人", "减持主体", "减持人" }.ToList();
 StockHolderRule.IsTitleEq = true;
 StockHolderRule.IsRequire = true;
 var HoldNumberAfterChangeRule = new TableSearchRule();
 HoldNumberAfterChangeRule.Name = "变动后持股数";
 HoldNumberAfterChangeRule.IsRequire = true;
 HoldNumberAfterChangeRule.SuperTitle = new string[] { "减持后", "增持后" }.ToList();
 HoldNumberAfterChangeRule.IsSuperTitleEq = false;
 HoldNumberAfterChangeRule.Title = new string[] {
 "持股股数","持股股数",
 "持股数量","持股数量",
 "持股总数","持股总数","股数"
 }.ToList();
 HoldNumberAfterChangeRule.IsTitleEq = false;
 var HoldPercentAfterChangeRule = new TableSearchRule();
 HoldPercentAfterChangeRule.Name = "变动后持股数比例";
 HoldPercentAfterChangeRule.IsRequire = true;
 HoldPercentAfterChangeRule.SuperTitle = HoldNumberAfterChangeRule.SuperTitle;
 HoldPercentAfterChangeRule.IsSuperTitleEq = false;
 HoldPercentAfterChangeRule.Title = new string[] { "比例" }.ToList();
 HoldPercentAfterChangeRule.IsTitleEq = false;
 var Rules = new List<TableSearchRule>();
 Rules.Add(StockHolderRule);
 Rules.Add(HoldNumberAfterChangeRule);
 Rules.Add(HoldPercentAfterChangeRule);
 var result = HTMLTable.GetMultiInfoByTitleRules(root, Rules, false);

EntityProperty对象

 /// <summary>
 /// 获得合同名
 /// </summary>
 /// <returns></returns>
 string GetContractName()
 {
 var e = new EntityProperty();
 e.PropertyName = "合同名称";
 e.PropertyType = EntityProperty.enmType.Normal;
 e.MaxLength = ContractTraning.MaxContractNameLength;
 e.MinLength = 5;
 e.LeadingColonKeyWordList = new string[] { "合同名称:" };
 e.QuotationTrailingWordList = new string[] { "协议书", "合同书", "确认书", "合同", "协议" };
 e.QuotationTrailingWordList_IsSkipBracket = true; //暂时只能选True
 var KeyList = new List<ExtractPropertyByDP.DPKeyWord>();
 KeyList.Add(new ExtractPropertyByDP.DPKeyWord()
 {
 StartWord = new string[] { "签署", "签订" }, //通过SRL训练获得
 StartDPValue = new string[] { LTPTrainingDP.核心关系, LTPTrainingDP.定中关系, LTPTrainingDP.并列关系 },
 EndWord = new string[] { "补充协议", "合同书", "合同", "协议书", "协议", },
 EndDPValue = new string[] { LTPTrainingDP.核心关系, LTPTrainingDP.定中关系, LTPTrainingDP.并列关系, LTPTrainingDP.动宾关系, LTPTrainingDP.主谓关系 }
 });
 e.DpKeyWordList = KeyList;
 var StartArray = new string[] { "签署了", "签订了" }; //通过语境训练获得
 var EndArray = new string[] { "合同" };
 e.ExternalStartEndStringFeature = Utility.GetStartEndStringArray(StartArray, EndArray);
 e.ExternalStartEndStringFeatureCandidatePreprocess = (x) => { return x + "合同"; };
 e.MaxLengthCheckPreprocess = str =>
 {
 return EntityWordAnlayzeTool.TrimEnglish(str);
 };
 //最高级别的置信度,特殊处理器
 e.LeadingColonKeyWordCandidatePreprocess = str =>
 {
 var c = Normalizer.ClearTrailing(TrimJianCheng(str));
 return c;
 };
 e.CandidatePreprocess = str =>
 {
 var c = Normalizer.ClearTrailing(TrimJianCheng(str));
 var RightQMarkIdx = c.IndexOf(""");
 if (!(RightQMarkIdx != -1 && RightQMarkIdx != c.Length - 1))
 {
 //对于"XXX"合同,有右边引号,但不是最后的时候,不用做
 c = c.TrimStart(""".ToCharArray());
 }
 c = c.TrimStart("《".ToCharArray());
 c = c.TrimEnd("》".ToCharArray()).TrimEnd(""".ToCharArray());
 return c;
 };
 e.ExcludeContainsWordList = new string[] { "日常经营重大合同" };
 //下面这个列表的根据不足
 e.ExcludeEqualsWordList = new string[] { "合同", "重大合同", "项目合同", "终止协议", "经营合同", "特别重大合同", "相关项目合同" };
 e.Extract(this);
 //是否所有的候选词里面包括(测试集无法使用)
 var contractlist = TraningDataset.ContractList.Where((x) => { return x.id == this.Id; });
 if (contractlist.Count() > 0)
 {
 var contract = contractlist.First();
 var contractname = contract.ContractName;
 if (!String.IsNullOrEmpty(contractname))
 {
 e.CheckIsCandidateContainsTarget(contractname);
 }
 }
 //置信度
 e.Confidence = ContractTraning.ContractES.GetStardardCI();
 return e.EvaluateCI();
 }

简单关键字抽取

 /// <summary>
 /// 评估方式
 /// </summary>
 /// <param name="root"></param>
 /// <returns></returns>
 string getEvaluateMethod()
 {
 var p = new EntityProperty();
 foreach (var method in ReOrganizationTraning.EvaluateMethodList)
 {
 p.KeyWordMap.Add(method, method);
 }
 p.Extract(this);
 if (!Program.IsMultiThreadMode) Program.Logger.WriteLine("评估方式:" + string.Join(Utility.SplitChar, p.WordMapResult));
 return string.Join(Utility.SplitChar, p.WordMapResult);
 }

实体位置体系

 /// <summary>
 /// 位置和值
 /// </summary>
 public struct LocAndValue<T>
 {
 /// <summary>
 /// HTML整体位置
 /// </summary>
 public int Loc;
 /// <summary>
 /// 开始位置
 /// </summary>
 public int StartIdx;
 /// <summary>
 /// 值
 /// </summary>
 public T Value;
 /// <summary>
 /// 类型
 /// </summary>
 public string Type;
 }

 /// <summary>
 /// 从释义表中抽取
 /// </summary>
 /// <returns></returns>
 List<(string Target, string Comany)> getTargetListFromReplaceTable()
 {
 var ReplaceCompany = new List<String>();
 foreach (var c in companynamelist)
 {
 if (c.positionId == -1)
 {
 //释义表
 if (!String.IsNullOrEmpty(c.secFullName)) ReplaceCompany.Add(c.secFullName);
 if (!String.IsNullOrEmpty(c.secShortName)) ReplaceCompany.Add(c.secShortName);
 }
 }
 var TargetAndCompanyList = new List<(string Target, string Comany)>();
 var targetRegular = new ExtractProperyBase.struRegularExpressFeature()
 {
 LeadingWordList = ReplaceCompany,
 RegularExpress = RegularTool.PercentExpress,
 TrailingWordList = new string[] { "的股权", "股权", "的权益", "权益" }.ToList()
 };
 var targetLoc = ExtractPropertyByHTML.FindRegularExpressLoc(targetRegular, root);
 targetLoc.AddRange(ExtractPropertyByHTML.FindWordLoc("资产及负债", root));
 targetLoc.AddRange(ExtractPropertyByHTML.FindWordLoc("业务及相关资产负债", root));
 foreach (var item in ReplacementDict)
 {
 var keys = item.Key.Split(Utility.SplitChar);
 var keys2 = item.Key.Split(new char[] { '/', '/' });
 if (keys.Length == 1 && keys2.Length > 1)
 {
 keys = keys2;
 }
 var values = item.Value.Split(Utility.SplitChar);
 var values2 = item.Value.Split(";");
 if (values.Length == 1 && values2.Length > 1)
 {
 values = values2;
 }
 foreach (var key in keys)
 {
 if (key == "交易标的")
 {
 foreach (var value in values)
 {
 var targetAndcompany = value.Trim();
 //将公司名称和交易标的划分开来
 ExtractPropertyByHTML.RegularExFinder(0, value, targetRegular, "|");
 }
 }
 }
 }
 return TargetAndCompanyList;
 }

 //在全文中寻找交易对象出现的地方
 var traderLoc = LocateProperty.LocateCustomerWord(root, trades);
 var targetLoc = LocateProperty.LocateCustomerWord(root, targets.Select(x => x.TargetCompany).ToList());
 var TradeLocMap = new Dictionary<int, List<LocateProperty.LocAndValue<String>>>();
 var TargetLocMap = new Dictionary<int, List<LocateProperty.LocAndValue<String>>>();
 //按照段落为依据,进行分组
 foreach (var trloc in traderLoc)
 {
 if (!TradeLocMap.ContainsKey(trloc.Loc))
 {
 TradeLocMap.Add(trloc.Loc, new List<LocateProperty.LocAndValue<String>>());
 }
 TradeLocMap[trloc.Loc].Add(trloc);
 }
 foreach (var trloc in targetLoc)
 {
 if (!TargetLocMap.ContainsKey(trloc.Loc))
 {
 TargetLocMap.Add(trloc.Loc, new List<LocateProperty.LocAndValue<String>>());
 }
 TargetLocMap[trloc.Loc].Add(trloc);
 }

 public void Anlayze(AnnouceDocument doc)
 {
 var nerlist = new List<LocAndValue<String>>();
 if (doc.Nerlist != null)
 {
 var nh = doc.Nerlist.Where(x => x.Type == enmNerType.Nh).Select(y => y.RawData).Distinct();
 nerlist.AddRange(LocateCustomerWord(doc.root, nh.ToList(), "人名"));
 var ni = doc.Nerlist.Where(x => x.Type == enmNerType.Ni).Select(y => y.RawData).Distinct();
 nerlist.AddRange(LocateCustomerWord(doc.root, ni.ToList(), "机构"));
 var ns = doc.Nerlist.Where(x => x.Type == enmNerType.Ns).Select(y => y.RawData).Distinct();
 nerlist.AddRange(LocateCustomerWord(doc.root, ns.ToList(), "地名"));
 }
 foreach (var paragragh in doc.root.Children)
 {
 foreach (var s in paragragh.Children)
 {
 var p = LocateParagraphInfo(doc, s.PositionId, nerlist);
 if (p.NerList.Count + p.moneylist.Count + p.datelist.Count != 0)
 {
 if (!ParagraghlocateDict.ContainsKey(s.PositionId)) ParagraghlocateDict.Add(s.PositionId, p);
 }
 }
 }
 }

参考文献

鸣谢