code/9-3.py · UsingStuding/NBbook

代码拉取完成,页面将自动刷新

扫描微信二维码支付

取消

支付完成

richgiteeai

Watch

不关注关注所有动态仅关注版本发行动态关注但不提醒动态

1 Star 0 Fork 1

UsingStuding/NBbook

forked from zhang_star/NBbook

代码 Issues 0 Pull Requests 0 Wiki 统计流水线

服务

加入 Gitee

与超过 1400万开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)

免费加入

已有帐号? 立即登录

文件

master

分支 (1)

管理

master

该仓库未声明开源许可证文件(LICENSE),使用请关注具体项目描述及其代码上游依赖。

项目仓库所选许可证以仓库主分支所使用许可证为准

克隆/下载

HTTPS SSH SVN SVN+SSH 下载ZIP

提示

下载代码请复制以下命令到终端执行

为确保你提交的代码身份被 Gitee 正确识别,请执行以下命令完成配置

git config --global user.name userName 
git config --global user.email userEmail

初次使用 SSH 协议进行代码克隆、推送等操作时,需按下述提示完成 SSH 配置

1 生成 RSA 密钥

2 获取 RSA 公钥内容,并配置到 SSH公钥中

在 Gitee 上使用 SVN,请访问使用指南

使用 HTTPS 协议时,命令行会出现如下账号密码验证步骤。基于安全考虑,Gitee 建议配置并使用私人令牌替代登录密码进行克隆、推送等操作

Username for 'https://gitee.com': userName

Password for 'https://userName@gitee.com': # 私人令牌

分支 1

标签 0

NBbook

code

9-3.py

9-3.py 6.06 KB

# -*- coding:utf-8 -*-

import sys
import urllib
import urlparse
import re
from hmmlearn import hmm
import numpy as np
from sklearn.externals import joblib
import HTMLParser
import nltk
import csv
import matplotlib.pyplot as plt
import os

#处理域名的最小长度
MIN_LEN=10

#状态个数
N=8
#最大似然概率阈值
T=-50

#模型文件名
FILE_MODEL="9-2.m"

def load_alexa(filename):
 domain_list=[]
 csv_reader = csv.reader(open(filename))
 for row in csv_reader:
 domain=row[1]
 if len(domain) >= MIN_LEN:
 domain_list.append(domain)
 return domain_list

def domain2ver(domain):
 ver=[]
 for i in range(0,len(domain)):
 ver.append([ord(domain[i])])
 return ver

def train_hmm(domain_list):
 X = [[0]]
 X_lens = [1]
 for domain in domain_list:
 ver=domain2ver(domain)
 np_ver = np.array(ver)
 X=np.concatenate([X,np_ver])
 X_lens.append(len(np_ver))

remodel = hmm.GaussianHMM(n_components=N, covariance_type="full", n_iter=100)
 remodel.fit(X,X_lens)
 joblib.dump(remodel, FILE_MODEL)

return remodel

def load_dga(filename):
 domain_list=[]
 #xsxqeadsbgvpdke.co.uk,Domain used by Cryptolocker - Flashback DGA for 13 Apr 2017,2017年04月13日,
 # http://osint.bambenekconsulting.com/manual/cl.txt
 with open(filename) as f:
 for line in f:
 domain=line.split(",")[0]
 if len(domain) >= MIN_LEN:
 domain_list.append(domain)
 return domain_list

def test_dga(remodel,filename):
 x=[]
 y=[]
 dga_cryptolocke_list = load_dga(filename)
 for domain in dga_cryptolocke_list:
 domain_ver=domain2ver(domain)
 np_ver = np.array(domain_ver)
 pro = remodel.score(np_ver)
 #print "SCORE:(%d) DOMAIN:(%s) " % (pro, domain)
 x.append(len(domain))
 y.append(pro)
 return x,y

def test_alexa(remodel,filename):
 x=[]
 y=[]
 alexa_list = load_alexa(filename)
 for domain in alexa_list:
 domain_ver=domain2ver(domain)
 np_ver = np.array(domain_ver)
 pro = remodel.score(np_ver)
 #print "SCORE:(%d) DOMAIN:(%s) " % (pro, domain)
 x.append(len(domain))
 y.append(pro)
 return x, y

def show_hmm():
 domain_list = load_alexa("../data/top-1000.csv")
 if not os.path.exists(FILE_MODEL):
 remodel=train_hmm(domain_list)
 remodel=joblib.load(FILE_MODEL)
 x_3,y_3=test_dga(remodel, "../data/dga-post-tovar-goz-1000.txt")
 x_2,y_2=test_dga(remodel,"../data/dga-cryptolocke-1000.txt")
 x_1,y_1=test_alexa(remodel, "../data/test-top-1000.csv")
 fig,ax=plt.subplots()
 ax.set_xlabel('Domain Length')
 ax.set_ylabel('HMM Score')
 ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz",marker='o')
 ax.scatter(x_2, y_2, color='g', label="dga_cryptolock",marker='v')
 ax.scatter(x_1, y_1, color='r', label="alexa",marker='*')
 ax.legend(loc='best')
 plt.show()

def get_aeiou(domain_list):
 x=[]
 y=[]
 for domain in domain_list:
 x.append(len(domain))
 count=len(re.findall(r'[aeiou]',domain.lower()))
 count=(0.0+count)/len(domain)
 y.append(count)
 return x,y

def show_aeiou():
 x1_domain_list = load_alexa("../data/top-1000.csv")
 x_1,y_1=get_aeiou(x1_domain_list)
 x2_domain_list = load_dga("../data/dga-cryptolocke-1000.txt")
 x_2,y_2=get_aeiou(x2_domain_list)
 x3_domain_list = load_dga("../data/dga-post-tovar-goz-1000.txt")
 x_3,y_3=get_aeiou(x3_domain_list)

fig,ax=plt.subplots()
 ax.set_xlabel('Domain Length')
 ax.set_ylabel('AEIOU Score')
 ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz",marker='o')
 ax.scatter(x_2, y_2, color='g', label="dga_cryptolock",marker='v')
 ax.scatter(x_1, y_1, color='r', label="alexa",marker='*')
 ax.legend(loc='best')
 plt.show()

def get_uniq_char_num(domain_list):
 x=[]
 y=[]
 for domain in domain_list:
 x.append(len(domain))
 count=len(set(domain))
 count=(0.0+count)/len(domain)
 y.append(count)
 return x,y

def show_uniq_char_num():
 x1_domain_list = load_alexa("../data/top-1000.csv")
 x_1,y_1=get_uniq_char_num(x1_domain_list)
 x2_domain_list = load_dga("../data/dga-cryptolocke-1000.txt")
 x_2,y_2=get_uniq_char_num(x2_domain_list)
 x3_domain_list = load_dga("../data/dga-post-tovar-goz-1000.txt")
 x_3,y_3=get_uniq_char_num(x3_domain_list)

fig,ax=plt.subplots()
 ax.set_xlabel('Domain Length')
 ax.set_ylabel('UNIQ CHAR NUMBER')
 ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz",marker='o')
 ax.scatter(x_2, y_2, color='g', label="dga_cryptolock",marker='v')
 ax.scatter(x_1, y_1, color='r', label="alexa",marker='*')
 ax.legend(loc='best')
 plt.show()

def count2string_jarccard_index(a,b):
 x=set(' '+a[0])
 y=set(' '+b[0])
 for i in range(0,len(a)-1):
 x.add(a[i]+a[i+1])
 x.add(a[len(a)-1]+' ')

for i in range(0,len(b)-1):
 y.add(b[i]+b[i+1])
 y.add(b[len(b)-1]+' ')

return (0.0+len(x-y))/len(x|y)

def get_jarccard_index(a_list,b_list):
 x=[]
 y=[]
 for a in a_list:
 j=0.0
 for b in b_list:
 j+=count2string_jarccard_index(a,b)
 x.append(len(a))
 y.append(j/len(b_list))

return x,y

def show_jarccard_index():
 x1_domain_list = load_alexa("../data/top-1000.csv")
 x_1,y_1=get_jarccard_index(x1_domain_list,x1_domain_list)
 x2_domain_list = load_dga("../data/dga-cryptolocke-1000.txt")
 x_2,y_2=get_jarccard_index(x2_domain_list,x1_domain_list)
 x3_domain_list = load_dga("../data/dga-post-tovar-goz-1000.txt")
 x_3,y_3=get_jarccard_index(x3_domain_list,x1_domain_list)

fig,ax=plt.subplots()
 ax.set_xlabel('Domain Length')
 ax.set_ylabel('JARCCARD INDEX')
 ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz",marker='o')
 ax.scatter(x_2, y_2, color='g', label="dga_cryptolock",marker='v')
 ax.scatter(x_1, y_1, color='r', label="alexa",marker='*')
 ax.legend(loc='lower right')
 plt.show()

if __name__ == '__main__':
 #show_hmm()
 #show_aeiou()
 #show_uniq_char_num()
 show_jarccard_index()

一键复制编辑原始数据按行查看历史

duoergun0729 提交于 2017年10月25日 13:04 +08:00 . 增加len函数

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219

# -*- coding:utf-8 -*-

import sys
import urllib
import urlparse
import re
from hmmlearn import hmm
import numpy as np
from sklearn.externals import joblib
import HTMLParser
import nltk
import csv
import matplotlib.pyplot as plt
import os


#处理域名的最小长度
MIN_LEN=10

#状态个数
N=8
#最大似然概率阈值
T=-50

#模型文件名
FILE_MODEL="9-2.m"

def load_alexa(filename):
 domain_list=[]
 csv_reader = csv.reader(open(filename))
 for row in csv_reader:
 domain=row[1]
 if len(domain) >= MIN_LEN:
 domain_list.append(domain)
 return domain_list

def domain2ver(domain):
 ver=[]
 for i in range(0,len(domain)):
 ver.append([ord(domain[i])])
 return ver

def train_hmm(domain_list):
 X = [[0]]
 X_lens = [1]
 for domain in domain_list:
 ver=domain2ver(domain)
 np_ver = np.array(ver)
 X=np.concatenate([X,np_ver])
 X_lens.append(len(np_ver))

 remodel = hmm.GaussianHMM(n_components=N, covariance_type="full", n_iter=100)
 remodel.fit(X,X_lens)
 joblib.dump(remodel, FILE_MODEL)

 return remodel

def load_dga(filename):
 domain_list=[]
 #xsxqeadsbgvpdke.co.uk,Domain used by Cryptolocker - Flashback DGA for 13 Apr 2017,2017年04月13日,
 # http://osint.bambenekconsulting.com/manual/cl.txt
 with open(filename) as f:
 for line in f:
 domain=line.split(",")[0]
 if len(domain) >= MIN_LEN:
 domain_list.append(domain)
 return domain_list

def test_dga(remodel,filename):
 x=[]
 y=[]
 dga_cryptolocke_list = load_dga(filename)
 for domain in dga_cryptolocke_list:
 domain_ver=domain2ver(domain)
 np_ver = np.array(domain_ver)
 pro = remodel.score(np_ver)
 #print "SCORE:(%d) DOMAIN:(%s) " % (pro, domain)
 x.append(len(domain))
 y.append(pro)
 return x,y

def test_alexa(remodel,filename):
 x=[]
 y=[]
 alexa_list = load_alexa(filename)
 for domain in alexa_list:
 domain_ver=domain2ver(domain)
 np_ver = np.array(domain_ver)
 pro = remodel.score(np_ver)
 #print "SCORE:(%d) DOMAIN:(%s) " % (pro, domain)
 x.append(len(domain))
 y.append(pro)
 return x, y

def show_hmm():
 domain_list = load_alexa("../data/top-1000.csv")
 if not os.path.exists(FILE_MODEL):
 remodel=train_hmm(domain_list)
 remodel=joblib.load(FILE_MODEL)
 x_3,y_3=test_dga(remodel, "../data/dga-post-tovar-goz-1000.txt")
 x_2,y_2=test_dga(remodel,"../data/dga-cryptolocke-1000.txt")
 x_1,y_1=test_alexa(remodel, "../data/test-top-1000.csv")
 fig,ax=plt.subplots()
 ax.set_xlabel('Domain Length')
 ax.set_ylabel('HMM Score')
 ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz",marker='o')
 ax.scatter(x_2, y_2, color='g', label="dga_cryptolock",marker='v')
 ax.scatter(x_1, y_1, color='r', label="alexa",marker='*')
 ax.legend(loc='best')
 plt.show()


def get_aeiou(domain_list):
 x=[]
 y=[]
 for domain in domain_list:
 x.append(len(domain))
 count=len(re.findall(r'[aeiou]',domain.lower()))
 count=(0.0+count)/len(domain)
 y.append(count)
 return x,y

def show_aeiou():
 x1_domain_list = load_alexa("../data/top-1000.csv")
 x_1,y_1=get_aeiou(x1_domain_list)
 x2_domain_list = load_dga("../data/dga-cryptolocke-1000.txt")
 x_2,y_2=get_aeiou(x2_domain_list)
 x3_domain_list = load_dga("../data/dga-post-tovar-goz-1000.txt")
 x_3,y_3=get_aeiou(x3_domain_list)

 fig,ax=plt.subplots()
 ax.set_xlabel('Domain Length')
 ax.set_ylabel('AEIOU Score')
 ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz",marker='o')
 ax.scatter(x_2, y_2, color='g', label="dga_cryptolock",marker='v')
 ax.scatter(x_1, y_1, color='r', label="alexa",marker='*')
 ax.legend(loc='best')
 plt.show()

def get_uniq_char_num(domain_list):
 x=[]
 y=[]
 for domain in domain_list:
 x.append(len(domain))
 count=len(set(domain))
 count=(0.0+count)/len(domain)
 y.append(count)
 return x,y

def show_uniq_char_num():
 x1_domain_list = load_alexa("../data/top-1000.csv")
 x_1,y_1=get_uniq_char_num(x1_domain_list)
 x2_domain_list = load_dga("../data/dga-cryptolocke-1000.txt")
 x_2,y_2=get_uniq_char_num(x2_domain_list)
 x3_domain_list = load_dga("../data/dga-post-tovar-goz-1000.txt")
 x_3,y_3=get_uniq_char_num(x3_domain_list)

 fig,ax=plt.subplots()
 ax.set_xlabel('Domain Length')
 ax.set_ylabel('UNIQ CHAR NUMBER')
 ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz",marker='o')
 ax.scatter(x_2, y_2, color='g', label="dga_cryptolock",marker='v')
 ax.scatter(x_1, y_1, color='r', label="alexa",marker='*')
 ax.legend(loc='best')
 plt.show()


def count2string_jarccard_index(a,b):
 x=set(' '+a[0])
 y=set(' '+b[0])
 for i in range(0,len(a)-1):
 x.add(a[i]+a[i+1])
 x.add(a[len(a)-1]+' ')

 for i in range(0,len(b)-1):
 y.add(b[i]+b[i+1])
 y.add(b[len(b)-1]+' ')

 return (0.0+len(x-y))/len(x|y)


def get_jarccard_index(a_list,b_list):
 x=[]
 y=[]
 for a in a_list:
 j=0.0
 for b in b_list:
 j+=count2string_jarccard_index(a,b)
 x.append(len(a))
 y.append(j/len(b_list))

 return x,y


def show_jarccard_index():
 x1_domain_list = load_alexa("../data/top-1000.csv")
 x_1,y_1=get_jarccard_index(x1_domain_list,x1_domain_list)
 x2_domain_list = load_dga("../data/dga-cryptolocke-1000.txt")
 x_2,y_2=get_jarccard_index(x2_domain_list,x1_domain_list)
 x3_domain_list = load_dga("../data/dga-post-tovar-goz-1000.txt")
 x_3,y_3=get_jarccard_index(x3_domain_list,x1_domain_list)

 fig,ax=plt.subplots()
 ax.set_xlabel('Domain Length')
 ax.set_ylabel('JARCCARD INDEX')
 ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz",marker='o')
 ax.scatter(x_2, y_2, color='g', label="dga_cryptolock",marker='v')
 ax.scatter(x_1, y_1, color='r', label="alexa",marker='*')
 ax.legend(loc='lower right')
 plt.show()

if __name__ == '__main__':
 #show_hmm()
 #show_aeiou()
 #show_uniq_char_num()
 show_jarccard_index()