From a9bb5c0038e6809b973ebd934afe53e830635f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月21日 12:05:30 +0800 Subject: [PATCH 001/121] =?UTF-8?q?=E7=AE=A1=E7=90=86=E7=95=8C=E9=9D=A2?= =?UTF-8?q?=E5=85=81=E8=AE=B8=E8=AE=BF=E9=97=AE=E7=9A=84IP=E5=92=8C?= =?UTF-8?q?=E7=A6=81=E6=AD=A2=E8=AE=BF=E9=97=AE=E7=9A=84IP=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0IP=E8=8C=83=E5=9B=B4=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/ip_range.go | 48 ++++++++++- teaconfigs/ip_range_test.go | 62 ++++++++++++- teaweb/configs/admin.go | 40 ++++++--- teaweb/configs/admin_security.go | 68 ++++++++++++++- teaweb/configs/admin_security_test.go | 120 ++++++++++++++++++++++++++ 5 files changed, 319 insertions(+), 19 deletions(-) create mode 100644 teaweb/configs/admin_security_test.go diff --git a/teaconfigs/ip_range.go b/teaconfigs/ip_range.go index d3d5eec..7f107ac 100644 --- a/teaconfigs/ip_range.go +++ b/teaconfigs/ip_range.go @@ -5,14 +5,16 @@ import ( "errors" "github.com/iwind/TeaGo/utils/string" "net" + "strings" ) // IP Range类型 type IPRangeType = int const ( - IPRangeTypeRange = IPRangeType(1) - IPRangeTypeCIDR = IPRangeType(2) + IPRangeTypeRange IPRangeType = 1 + IPRangeTypeCIDR IPRangeType = 2 + IPRangeTypeAll IPRangeType = 3 ) // IP Range @@ -38,6 +40,45 @@ func NewIPRangeConfig() *IPRangeConfig { } } +// 从字符串中分析 +func ParseIPRange(s string) (*IPRangeConfig, error) { + if len(s) == 0 { + return nil, errors.New("invalid ip range") + } + + ipRange := &IPRangeConfig{} + + if s == "all" || s == "ALL" || s == "0.0.0.0" { + ipRange.Type = IPRangeTypeAll + return ipRange, nil + } + + if strings.Contains(s, "/") { + ipRange.Type = IPRangeTypeCIDR + ipRange.CIDR = strings.Replace(s, " ", "", -1) + } else if strings.Contains(s, "-") { + ipRange.Type = IPRangeTypeRange + pieces := strings.SplitN(s, "-", 2) + ipRange.IPFrom = strings.TrimSpace(pieces[0]) + ipRange.IPTo = strings.TrimSpace(pieces[1]) + } else if strings.Contains(s, ",") { + ipRange.Type = IPRangeTypeRange + pieces := strings.SplitN(s, ",", 2) + ipRange.IPFrom = strings.TrimSpace(pieces[0]) + ipRange.IPTo = strings.TrimSpace(pieces[1]) + } else { + ipRange.Type = IPRangeTypeRange + ipRange.IPFrom = s + ipRange.IPTo = s + } + + err := ipRange.Validate() + if err != nil { + return nil, err + } + return ipRange, nil +} + // 校验 func (this *IPRangeConfig) Validate() error { if this.Type == IPRangeTypeCIDR { @@ -86,5 +127,8 @@ func (this *IPRangeConfig) Contains(ipString string) bool { } return bytes.Compare(ip, this.ipFrom)>= 0 && bytes.Compare(ip, this.ipTo) <= 0 } + if this.Type == IPRangeTypeAll { + return true + } return false } diff --git a/teaconfigs/ip_range_test.go b/teaconfigs/ip_range_test.go index ed61f51..383e519 100644 --- a/teaconfigs/ip_range_test.go +++ b/teaconfigs/ip_range_test.go @@ -13,7 +13,7 @@ func TestGeoConfig_Contains(t *testing.T) { geo.Type = IPRangeTypeRange geo.IPFrom = "192.168.1.100" geo.IPTo = "192.168.1.110" - geo.Validate() + a.IsNil(geo.Validate()) a.IsTrue(geo.Contains("192.168.1.100")) a.IsTrue(geo.Contains("192.168.1.101")) a.IsTrue(geo.Contains("192.168.1.110")) @@ -24,7 +24,7 @@ func TestGeoConfig_Contains(t *testing.T) { geo := NewIPRangeConfig() geo.Type = IPRangeTypeCIDR geo.CIDR = "192.168.1.1/24" - geo.Validate() + a.IsNil(geo.Validate()) a.IsTrue(geo.Contains("192.168.1.100")) a.IsFalse(geo.Contains("192.168.2.100")) } @@ -33,7 +33,63 @@ func TestGeoConfig_Contains(t *testing.T) { geo := NewIPRangeConfig() geo.Type = IPRangeTypeCIDR geo.CIDR = "192.168.1.1/16" - geo.Validate() + a.IsNil(geo.Validate()) a.IsTrue(geo.Contains("192.168.2.100")) } } + +func TestParseIPRange(t *testing.T) { + a := assert.NewAssertion(t) + + { + _, err := ParseIPRange("") + a.IsNotNil(err) + } + + { + r, err := ParseIPRange("192.168.1.100") + a.IsNil(err) + a.IsTrue(r.IPFrom == r.IPTo) + a.IsTrue(r.IPFrom == "192.168.1.100") + a.IsTrue(r.Contains("192.168.1.100")) + a.IsFalse(r.Contains("192.168.1.99")) + } + + { + r, err := ParseIPRange("192.168.1.100/24") + a.IsNil(err) + a.IsTrue(r.CIDR == "192.168.1.100/24") + a.IsTrue(r.Contains("192.168.1.100")) + a.IsTrue(r.Contains("192.168.1.99")) + a.IsFalse(r.Contains("192.168.2.100")) + } + + { + r, err := ParseIPRange("192.168.1.100, 192.168.1.200") + a.IsNil(err) + a.IsTrue(r.IPFrom == "192.168.1.100") + a.IsTrue(r.IPTo == "192.168.1.200") + a.IsTrue(r.Contains("192.168.1.100")) + a.IsTrue(r.Contains("192.168.1.150")) + a.IsFalse(r.Contains("192.168.2.100")) + } + + { + r, err := ParseIPRange("192.168.1.100-192.168.1.200") + a.IsNil(err) + a.IsTrue(r.IPFrom == "192.168.1.100") + a.IsTrue(r.IPTo == "192.168.1.200") + a.IsTrue(r.Contains("192.168.1.100")) + a.IsTrue(r.Contains("192.168.1.150")) + a.IsFalse(r.Contains("192.168.2.100")) + } + + { + r, err := ParseIPRange("all") + a.IsNil(err) + a.IsTrue(r.Type == IPRangeTypeAll) + a.IsTrue(r.Contains("192.168.1.100")) + a.IsTrue(r.Contains("192.168.1.150")) + a.IsTrue(r.Contains("192.168.2.100")) + } +} diff --git a/teaweb/configs/admin.go b/teaweb/configs/admin.go index d775bc6..fbaab1c 100644 --- a/teaweb/configs/admin.go +++ b/teaweb/configs/admin.go @@ -5,7 +5,6 @@ import ( "github.com/go-yaml/yaml" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" - "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" "io/ioutil" "sync" @@ -53,20 +52,45 @@ func SharedAdminConfig() *AdminConfig { return adminConfig } + err = adminConfig.Validate() + if err != nil { + logs.Error(err) + } + return adminConfig } +// 校验 +func (this *AdminConfig) Validate() error { + if this.Security != nil { + err := this.Security.Validate() + if err != nil { + return err + } + } + return nil +} + // 写回配置文件 func (this *AdminConfig) Save() error { adminConfigLocker.Lock() defer adminConfigLocker.Unlock() + // 校验 + err := this.Validate() + if err != nil { + logs.Error(err) + } + writer, err := files.NewWriter(Tea.ConfigFile("admin.conf")) if err != nil { return err } - defer writer.Close() + defer func() { + _ = writer.Close() + }() _, err = writer.WriteYAML(this) + return err } @@ -205,7 +229,7 @@ func (this *AdminConfig) FindAllActiveGrants() []*AdminGrant { NewAdminGrant("测试小Q", AdminGrantQ), NewAdminGrant("API", AdminGrantApi), NewAdminGrant("团队", AdminGrantTeam), - } ...) + }...) } return grants @@ -218,17 +242,11 @@ func (this *AdminConfig) AddGrant(grant *AdminGrant) { // 检查是否允许IP func (this *AdminConfig) AllowIP(ip string) bool { - // deny - if len(this.Security.Deny)> 0 && lists.ContainsString(this.Security.Deny, ip) { - return false - } - - // allow - if lists.ContainsString(this.Security.Allow, "all") || lists.ContainsString(this.Security.Allow, "0.0.0.0") || lists.ContainsString(this.Security.Allow, ip) { + if this.Security == nil { return true } - return false + return this.Security.AllowIP(ip) } // 重置状态 diff --git a/teaweb/configs/admin_security.go b/teaweb/configs/admin_security.go index 7289625..e23f0b0 100644 --- a/teaweb/configs/admin_security.go +++ b/teaweb/configs/admin_security.go @@ -1,9 +1,71 @@ package configs +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/iwind/TeaGo/logs" + "net" +) + // 安全设置定义 type AdminSecurity struct { - Allow []string `yaml:"allow" json:"allow"` - Deny []string `yaml:"deny" json:"deny"` - Secret string `yaml:"secret" json:"secret"` + Allow []string `yaml:"allow" json:"allow"` //支持的IP + Deny []string `yaml:"deny" json:"deny"` // 拒绝的IP + Secret string `yaml:"secret" json:"secret"` // 密钥 IsDisabled bool `yaml:"isDisabled" json:"isDisabled"` // 是否禁用 + + allowIPRanges []*teaconfigs.IPRangeConfig + denyIPRanges []*teaconfigs.IPRangeConfig +} + +func (this *AdminSecurity) Validate() error { + this.allowIPRanges = []*teaconfigs.IPRangeConfig{} + for _, s := range this.Allow { + r, err := teaconfigs.ParseIPRange(s) + if err != nil { + logs.Error(err) + } else { + this.allowIPRanges = append(this.allowIPRanges, r) + } + } + + this.denyIPRanges = []*teaconfigs.IPRangeConfig{} + for _, s := range this.Deny { + r, err := teaconfigs.ParseIPRange(s) + if err != nil { + logs.Error(err) + } else { + this.denyIPRanges = append(this.denyIPRanges, r) + } + } + + return nil +} + +// 判断某个IP是否允许访问 +func (this *AdminSecurity) AllowIP(ip string) bool { + netIP := net.ParseIP(ip) + if netIP == nil { + return true + } + + // deny + if len(this.denyIPRanges)> 0 { + for _, r := range this.denyIPRanges { + if r.Contains(ip) { + return false + } + } + } + + // allow + if len(this.Allow)> 0 { + for _, r := range this.allowIPRanges { + if r.Contains(ip) { + return true + } + } + return false + } + + return true } diff --git a/teaweb/configs/admin_security_test.go b/teaweb/configs/admin_security_test.go new file mode 100644 index 0000000..ba89efa --- /dev/null +++ b/teaweb/configs/admin_security_test.go @@ -0,0 +1,120 @@ +package configs + +import ( + "github.com/iwind/TeaGo/assert" + "testing" +) + +func TestAdminSecurity_AllowIP(t *testing.T) { + a := assert.NewAssertion(t) + + { + security := new(AdminSecurity) + a.IsNil(security.Validate()) + a.IsTrue(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{"192.168.2.40"} + a.IsNil(security.Validate()) + a.IsFalse(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{"192.168.2.1/24"} + a.IsNil(security.Validate()) + a.IsFalse(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{"192.168.3.1 / 24"} + a.IsNil(security.Validate()) + a.IsTrue(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{"192.168.2.40,192.168.2.42"} + a.IsNil(security.Validate()) + a.IsFalse(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{"192.168.2.41,192.168.2.42", "all"} + a.IsNil(security.Validate()) + a.IsFalse(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{"0.0.0.0"} + a.IsNil(security.Validate()) + a.IsFalse(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{} + security.Allow = []string{"192.168.2.1"} + a.IsNil(security.Validate()) + a.IsFalse(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{} + security.Allow = []string{"192.168.2.1/24"} + a.IsNil(security.Validate()) + a.IsTrue(security.AllowIP("192.168.2.40")) + a.IsFalse(security.AllowIP("192.168.1.100")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{} + security.Allow = []string{"192.168.2.1,192.168.2.100"} + a.IsNil(security.Validate()) + a.IsTrue(security.AllowIP("192.168.2.40")) + a.IsFalse(security.AllowIP("192.168.1.100")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{} + security.Allow = []string{"all"} + a.IsNil(security.Validate()) + a.IsTrue(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{} + security.Allow = []string{"0.0.0.0"} + a.IsNil(security.Validate()) + a.IsTrue(security.AllowIP("192.168.2.40")) + } + + { + security := new(AdminSecurity) + security.Deny = []string{} + security.Allow = []string{} + a.IsNil(security.Validate()) + a.IsTrue(security.AllowIP("192.168.2.40")) + } +} + +func BenchmarkAdminSecurity_AllowIP(b *testing.B) { + security := new(AdminSecurity) + security.Deny = []string{} + security.Allow = []string{"192.168.2.1/24"} + _ = security.Validate() + + for i := 0; i < b.N; i++ { + _ = security.AllowIP("192.168.2.40") + _ = security.AllowIP("192.168.1.100") + } +} From 6c2adc9e85a3e2917a44a5d54247ac2e7f1e4784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月21日 15:50:24 +0800 Subject: [PATCH 002/121] =?UTF-8?q?[security]=E5=AE=89=E5=85=A8=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E4=B8=AD=E5=A2=9E=E5=8A=A0=E2=80=9C=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E8=A1=A5=E5=85=A8=E2=80=9D=E9=80=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/proxy/add.go | 9 +++++++ teaweb/actions/default/proxy/locations/add.go | 9 +++++++ .../actions/default/proxy/locations/update.go | 9 +++++++ teaweb/actions/default/proxy/update.go | 9 +++++++ .../default/settings/server/security.go | 18 ++++++++++--- teaweb/configs/admin_security.go | 27 ++++++++++++++++--- 6 files changed, 74 insertions(+), 7 deletions(-) diff --git a/teaweb/actions/default/proxy/add.go b/teaweb/actions/default/proxy/add.go index cb73d62..93e8ed0 100644 --- a/teaweb/actions/default/proxy/add.go +++ b/teaweb/actions/default/proxy/add.go @@ -5,6 +5,7 @@ import ( "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" + "github.com/TeaWeb/code/teaweb/configs" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/logs" "strings" @@ -15,6 +16,14 @@ type AddAction actions.Action func (this *AddAction) Run(params struct { }) { + // 目录补全 + security := configs.SharedAdminConfig().Security + if security != nil { + this.Data["dirAutoComplete"] = security.DirAutoComplete + } else { + this.Data["dirAutoComplete"] = false + } + this.Show() } diff --git a/teaweb/actions/default/proxy/locations/add.go b/teaweb/actions/default/proxy/locations/add.go index f5c535b..08d8f50 100644 --- a/teaweb/actions/default/proxy/locations/add.go +++ b/teaweb/actions/default/proxy/locations/add.go @@ -5,6 +5,7 @@ import ( "github.com/TeaWeb/code/teaconfigs/shared" "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" + "github.com/TeaWeb/code/teaweb/configs" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/types" @@ -51,6 +52,14 @@ func (this *AddAction) Run(params struct { MimeTypes: teaconfigs.DefaultGzipMimeTypes, } + // 目录补全 + security := configs.SharedAdminConfig().Security + if security != nil { + this.Data["dirAutoComplete"] = security.DirAutoComplete + } else { + this.Data["dirAutoComplete"] = false + } + this.Show() } diff --git a/teaweb/actions/default/proxy/locations/update.go b/teaweb/actions/default/proxy/locations/update.go index 07280ba..62b3b82 100644 --- a/teaweb/actions/default/proxy/locations/update.go +++ b/teaweb/actions/default/proxy/locations/update.go @@ -6,6 +6,7 @@ import ( "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/locations/locationutils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" + "github.com/TeaWeb/code/teaweb/configs" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/maps" @@ -89,6 +90,14 @@ func (this *UpdateAction) Run(params struct { // 变量 this.Data["variables"] = proxyutils.DefaultRequestVariables() + // 目录补全 + security := configs.SharedAdminConfig().Security + if security != nil { + this.Data["dirAutoComplete"] = security.DirAutoComplete + } else { + this.Data["dirAutoComplete"] = false + } + this.Show() } diff --git a/teaweb/actions/default/proxy/update.go b/teaweb/actions/default/proxy/update.go index 53d0530..92fc61d 100644 --- a/teaweb/actions/default/proxy/update.go +++ b/teaweb/actions/default/proxy/update.go @@ -4,6 +4,7 @@ import ( "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" + "github.com/TeaWeb/code/teaweb/configs" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/types" "strconv" @@ -41,6 +42,14 @@ func (this *UpdateAction) Run(params struct { server.SetupNoticeItems() this.Data["noticeItems"] = server.NoticeItems + // 目录补全 + security := configs.SharedAdminConfig().Security + if security != nil { + this.Data["dirAutoComplete"] = security.DirAutoComplete + } else { + this.Data["dirAutoComplete"] = false + } + this.Show() } diff --git a/teaweb/actions/default/settings/server/security.go b/teaweb/actions/default/settings/server/security.go index 418569a..a9a6021 100644 --- a/teaweb/actions/default/settings/server/security.go +++ b/teaweb/actions/default/settings/server/security.go @@ -11,6 +11,11 @@ type SecurityAction actions.Action // 安全设置 func (this *SecurityAction) Run(params struct{}) { admin := configs.SharedAdminConfig() + + if admin.Security == nil { + admin.Security = configs.NewAdminSecurity() + } + this.Data["security"] = admin.Security this.Data["allowAll"] = lists.ContainsString(admin.Security.Allow, "all") this.Data["userIP"] = this.RequestRemoteIP() @@ -19,11 +24,16 @@ func (this *SecurityAction) Run(params struct{}) { } func (this *SecurityAction) RunPost(params struct { - AllowIPs []string - DenyIPs []string - AllowAll bool + AllowIPs []string + DenyIPs []string + AllowAll bool + DirAutoComplete bool }) { admin := configs.SharedAdminConfig() + if admin.Security == nil { + admin.Security = configs.NewAdminSecurity() + } + if params.AllowAll { admin.Security.Allow = []string{"all"} } else { @@ -51,6 +61,8 @@ func (this *SecurityAction) RunPost(params struct { admin.Security.Deny = ips } + admin.Security.DirAutoComplete = params.DirAutoComplete + err := admin.Save() if err != nil { this.Fail("保存失败:" + err.Error()) diff --git a/teaweb/configs/admin_security.go b/teaweb/configs/admin_security.go index e23f0b0..462a0ad 100644 --- a/teaweb/configs/admin_security.go +++ b/teaweb/configs/admin_security.go @@ -2,22 +2,41 @@ package configs import ( "github.com/TeaWeb/code/teaconfigs" + "github.com/TeaWeb/code/teaconst" "github.com/iwind/TeaGo/logs" "net" ) // 安全设置定义 type AdminSecurity struct { - Allow []string `yaml:"allow" json:"allow"` //支持的IP - Deny []string `yaml:"deny" json:"deny"` // 拒绝的IP - Secret string `yaml:"secret" json:"secret"` // 密钥 - IsDisabled bool `yaml:"isDisabled" json:"isDisabled"` // 是否禁用 + TeaVersion string `yaml:"teaVersion" json:"teaVersion"` + + Allow []string `yaml:"allow" json:"allow"` //支持的IP + Deny []string `yaml:"deny" json:"deny"` // 拒绝的IP + Secret string `yaml:"secret" json:"secret"` // 密钥 + IsDisabled bool `yaml:"isDisabled" json:"isDisabled"` // 是否禁用 + DirAutoComplete bool `yaml:"dirAutoComplete" json:"dirAutoComplete"` // 是否支持目录自动补全 allowIPRanges []*teaconfigs.IPRangeConfig denyIPRanges []*teaconfigs.IPRangeConfig } +// 获取新对象 +func NewAdminSecurity() *AdminSecurity { + return &AdminSecurity{ + Allow: []string{}, + Deny: []string{}, + } +} + +// 校验 func (this *AdminSecurity) Validate() error { + // 兼容性 + if len(this.TeaVersion) == 0 { + this.TeaVersion = teaconst.TeaVersion + this.DirAutoComplete = true + } + this.allowIPRanges = []*teaconfigs.IPRangeConfig{} for _, s := range this.Allow { r, err := teaconfigs.ParseIPRange(s) From ae6c990e6a18fb634d0119c0d95def0d8d3b170f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月21日 16:39:18 +0800 Subject: [PATCH 003/121] =?UTF-8?q?[security]=E5=AE=89=E5=85=A8=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E4=B8=AD=E5=A2=9E=E5=8A=A0=E2=80=9C=E7=99=BB=E5=BD=95?= =?UTF-8?q?=E7=95=8C=E9=9D=A2URL=E2=80=9D=E9=80=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/login/init.go | 14 ++++++++++++-- teaweb/actions/default/settings/server/security.go | 12 ++++++++++++ teaweb/configs/admin_security.go | 13 +++++++++++++ teaweb/helpers/user_must_auth.go | 7 ++++++- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/teaweb/actions/default/login/init.go b/teaweb/actions/default/login/init.go index 59f0c1f..da74ecf 100644 --- a/teaweb/actions/default/login/init.go +++ b/teaweb/actions/default/login/init.go @@ -1,11 +1,21 @@ package login -import "github.com/iwind/TeaGo" +import ( + "github.com/TeaWeb/code/teaweb/configs" + "github.com/iwind/TeaGo" +) func init() { TeaGo.BeforeStart(func(server *TeaGo.Server) { + // 自定义登录URL + prefix := "/login" + security := configs.SharedAdminConfig().Security + if security != nil { + prefix = security.NewLoginURL() + } + server. - Prefix("/login"). + Prefix(prefix). GetPost("", new(IndexAction)). EndAll() }) diff --git a/teaweb/actions/default/settings/server/security.go b/teaweb/actions/default/settings/server/security.go index a9a6021..c130683 100644 --- a/teaweb/actions/default/settings/server/security.go +++ b/teaweb/actions/default/settings/server/security.go @@ -1,6 +1,7 @@ package server import ( + "github.com/TeaWeb/code/teaweb/actions/default/settings" "github.com/TeaWeb/code/teaweb/configs" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" @@ -28,6 +29,7 @@ func (this *SecurityAction) RunPost(params struct { DenyIPs []string AllowAll bool DirAutoComplete bool + LoginURL string }) { admin := configs.SharedAdminConfig() if admin.Security == nil { @@ -63,11 +65,21 @@ func (this *SecurityAction) RunPost(params struct { admin.Security.DirAutoComplete = params.DirAutoComplete + isServerChanged := false + if admin.Security.LoginURL != params.LoginURL { + isServerChanged = true + } + admin.Security.LoginURL = params.LoginURL + err := admin.Save() if err != nil { this.Fail("保存失败:" + err.Error()) } + if isServerChanged { + settings.NotifyServerChange() + } + this.Next("/settings", map[string]interface{}{}) this.Success("保存成功") } diff --git a/teaweb/configs/admin_security.go b/teaweb/configs/admin_security.go index 462a0ad..002c853 100644 --- a/teaweb/configs/admin_security.go +++ b/teaweb/configs/admin_security.go @@ -16,6 +16,7 @@ type AdminSecurity struct { Secret string `yaml:"secret" json:"secret"` // 密钥 IsDisabled bool `yaml:"isDisabled" json:"isDisabled"` // 是否禁用 DirAutoComplete bool `yaml:"dirAutoComplete" json:"dirAutoComplete"` // 是否支持目录自动补全 + LoginURL string `yaml:"loginURL" json:"loginURL"` // 登录页面的URL allowIPRanges []*teaconfigs.IPRangeConfig denyIPRanges []*teaconfigs.IPRangeConfig @@ -88,3 +89,15 @@ func (this *AdminSecurity) AllowIP(ip string) bool { return true } + +// 获取登录URL +func (this *AdminSecurity) NewLoginURL() string { + url := "/login" + if len(this.LoginURL)> 0 { + url = this.LoginURL + } + if url[0] != '/' { + url = "/" + url + } + return url +} diff --git a/teaweb/helpers/user_must_auth.go b/teaweb/helpers/user_must_auth.go index fcd9d57..76d8a4d 100644 --- a/teaweb/helpers/user_must_auth.go +++ b/teaweb/helpers/user_must_auth.go @@ -153,5 +153,10 @@ func (this *UserMustAuth) BeforeAction(actionPtr actions.ActionWrapper, paramNam } func (this *UserMustAuth) login(action *actions.ActionObject) { - action.RedirectURL("/login") + security := configs.SharedAdminConfig().Security + if security != nil { + action.RedirectURL(security.NewLoginURL()) + } else { + action.RedirectURL("/login") + } } From d8f3a8e1058e5f746937ed46f26a403fbff68fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月21日 17:30:11 +0800 Subject: [PATCH 004/121] =?UTF-8?q?[security]=E5=AE=89=E5=85=A8=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E4=B8=AD=E5=A2=9E=E5=8A=A0=E7=99=BB=E5=BD=95=E5=AF=86?= =?UTF-8?q?=E7=A0=81=E5=8A=A0=E5=AF=86=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/login/index.go | 2 +- teaweb/actions/default/settings/index.go | 4 ++ .../actions/default/settings/login/index.go | 6 ++- .../actions/default/settings/login/update.go | 4 +- .../default/settings/server/security.go | 13 +++--- teaweb/configs/admin.go | 27 ++++++++++++ teaweb/configs/admin_security.go | 41 +++++++++++++++---- teaweb/configs/admin_test.go | 26 +++++++++++- teaweb/configs/admin_user.go | 2 +- 9 files changed, 107 insertions(+), 18 deletions(-) diff --git a/teaweb/actions/default/login/index.go b/teaweb/actions/default/login/index.go index e2a4b6f..741172e 100644 --- a/teaweb/actions/default/login/index.go +++ b/teaweb/actions/default/login/index.go @@ -79,7 +79,7 @@ func (this *IndexAction) RunPost(params struct { } // 密码错误 - if user.Password != params.Password { + if !adminConfig.ComparePassword(params.Password, user.Password) { user.IncreaseLoginTries() this.Fail("登录失败,请检查用户名密码") } diff --git a/teaweb/actions/default/settings/index.go b/teaweb/actions/default/settings/index.go index 8982170..bb8af67 100644 --- a/teaweb/actions/default/settings/index.go +++ b/teaweb/actions/default/settings/index.go @@ -22,7 +22,11 @@ func (this *IndexAction) Run(params struct{}) { // admin admin := configs.SharedAdminConfig() + if admin.Security == nil { + admin.Security = configs.NewAdminSecurity() + } this.Data["security"] = admin.Security + this.Data["passwordEncryptTypeText"] = admin.Security.PasswordEncryptTypeText() this.Show() } diff --git a/teaweb/actions/default/settings/login/index.go b/teaweb/actions/default/settings/login/index.go index 1135fa6..d9dad29 100644 --- a/teaweb/actions/default/settings/login/index.go +++ b/teaweb/actions/default/settings/login/index.go @@ -21,7 +21,11 @@ func (this *IndexAction) Run(params struct{}) { return } - this.Data["passwordMask"] = strings.Repeat("*", len(user.Password)) + encryptedType := "" + if strings.HasPrefix(user.Password, "md5:") { + encryptedType = "md5:" + } + this.Data["passwordMask"] = encryptedType + strings.Repeat("*", len(user.Password)) this.Data["key"] = user.Key this.Show() diff --git a/teaweb/actions/default/settings/login/update.go b/teaweb/actions/default/settings/login/update.go index 8bbbbd2..5cfd4ae 100644 --- a/teaweb/actions/default/settings/login/update.go +++ b/teaweb/actions/default/settings/login/update.go @@ -1,8 +1,8 @@ package login import ( - "github.com/iwind/TeaGo/actions" "github.com/TeaWeb/code/teaweb/configs" + "github.com/iwind/TeaGo/actions" ) type UpdateAction actions.Action @@ -45,7 +45,7 @@ func (this *UpdateAction) RunPost(params struct { user.Username = params.Username if len(params.Password)> 0 { - user.Password = params.Password + user.Password = config.EncryptPassword(params.Password) } found = true diff --git a/teaweb/actions/default/settings/server/security.go b/teaweb/actions/default/settings/server/security.go index c130683..bc5bc62 100644 --- a/teaweb/actions/default/settings/server/security.go +++ b/teaweb/actions/default/settings/server/security.go @@ -20,16 +20,18 @@ func (this *SecurityAction) Run(params struct{}) { this.Data["security"] = admin.Security this.Data["allowAll"] = lists.ContainsString(admin.Security.Allow, "all") this.Data["userIP"] = this.RequestRemoteIP() + this.Data["types"] = configs.PasswordEncryptTypes() this.Show() } func (this *SecurityAction) RunPost(params struct { - AllowIPs []string - DenyIPs []string - AllowAll bool - DirAutoComplete bool - LoginURL string + AllowIPs []string + DenyIPs []string + AllowAll bool + DirAutoComplete bool + LoginURL string + PasswordEncryptType string }) { admin := configs.SharedAdminConfig() if admin.Security == nil { @@ -70,6 +72,7 @@ func (this *SecurityAction) RunPost(params struct { isServerChanged = true } admin.Security.LoginURL = params.LoginURL + admin.Security.PasswordEncryptType = params.PasswordEncryptType err := admin.Save() if err != nil { diff --git a/teaweb/configs/admin.go b/teaweb/configs/admin.go index fbaab1c..fc618a1 100644 --- a/teaweb/configs/admin.go +++ b/teaweb/configs/admin.go @@ -6,7 +6,9 @@ import ( "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" "github.com/iwind/TeaGo/logs" + stringutil "github.com/iwind/TeaGo/utils/string" "io/ioutil" + "strings" "sync" ) @@ -94,6 +96,31 @@ func (this *AdminConfig) Save() error { return err } +// 加密密码 +func (this *AdminConfig) EncryptPassword(password string) string { + if this.Security == nil { + return password + } + switch this.Security.PasswordEncryptType { + case "clear": + return "clear:" + password + case "md5": + return "md5:" + stringutil.Md5(password) + } + return "clear:" + password +} + +// 对比密码 +func (this *AdminConfig) ComparePassword(rawPassword, encryptedPassword string) bool { + if strings.HasPrefix(encryptedPassword, "clear:") { + return "clear:"+rawPassword == encryptedPassword + } + if strings.HasPrefix(encryptedPassword, "md5:") { + return "md5:"+stringutil.Md5(rawPassword) == encryptedPassword + } + return rawPassword == encryptedPassword +} + // 是否包含某个激活的用户名 func (this *AdminConfig) ContainsActiveUser(username string) bool { adminConfigLocker.Lock() diff --git a/teaweb/configs/admin_security.go b/teaweb/configs/admin_security.go index 002c853..5539ce5 100644 --- a/teaweb/configs/admin_security.go +++ b/teaweb/configs/admin_security.go @@ -4,6 +4,7 @@ import ( "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teaconst" "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" "net" ) @@ -11,12 +12,13 @@ import ( type AdminSecurity struct { TeaVersion string `yaml:"teaVersion" json:"teaVersion"` - Allow []string `yaml:"allow" json:"allow"` //支持的IP - Deny []string `yaml:"deny" json:"deny"` // 拒绝的IP - Secret string `yaml:"secret" json:"secret"` // 密钥 - IsDisabled bool `yaml:"isDisabled" json:"isDisabled"` // 是否禁用 - DirAutoComplete bool `yaml:"dirAutoComplete" json:"dirAutoComplete"` // 是否支持目录自动补全 - LoginURL string `yaml:"loginURL" json:"loginURL"` // 登录页面的URL + Allow []string `yaml:"allow" json:"allow"` //支持的IP + Deny []string `yaml:"deny" json:"deny"` // 拒绝的IP + Secret string `yaml:"secret" json:"secret"` // 密钥 + IsDisabled bool `yaml:"isDisabled" json:"isDisabled"` // 是否禁用 + DirAutoComplete bool `yaml:"dirAutoComplete" json:"dirAutoComplete"` // 是否支持目录自动补全 + LoginURL string `yaml:"loginURL" json:"loginURL"` // 登录页面的URL + PasswordEncryptType string `yaml:"passwordEncryptType" json:"passwordEncryptType"` // 密码加密方式 allowIPRanges []*teaconfigs.IPRangeConfig denyIPRanges []*teaconfigs.IPRangeConfig @@ -30,12 +32,27 @@ func NewAdminSecurity() *AdminSecurity { } } +// 密码加密方式列表 +func PasswordEncryptTypes() []maps.Map { + return []maps.Map{ + { + "name": "明文", + "code": "clear", + }, + { + "name": "MD5", + "code": "md5", + }, + } +} + // 校验 func (this *AdminSecurity) Validate() error { - // 兼容性 + // 版本兼容性 if len(this.TeaVersion) == 0 { this.TeaVersion = teaconst.TeaVersion this.DirAutoComplete = true + this.PasswordEncryptType = "clear" } this.allowIPRanges = []*teaconfigs.IPRangeConfig{} @@ -101,3 +118,13 @@ func (this *AdminSecurity) NewLoginURL() string { } return url } + +// 获取加密方式文字说明 +func (this *AdminSecurity) PasswordEncryptTypeText() string { + for _, m := range PasswordEncryptTypes() { + if m.GetString("code") == this.PasswordEncryptType { + return m.GetString("name") + } + } + return "明文" +} diff --git a/teaweb/configs/admin_test.go b/teaweb/configs/admin_test.go index a6a63dc..01ec4d5 100644 --- a/teaweb/configs/admin_test.go +++ b/teaweb/configs/admin_test.go @@ -1,8 +1,32 @@ package configs -import "testing" +import ( + "github.com/iwind/TeaGo/assert" + stringutil "github.com/iwind/TeaGo/utils/string" + "testing" +) func TestSharedAdminConfig(t *testing.T) { adminConfig := SharedAdminConfig() t.Logf("%#v", adminConfig) } + +func TestAdminConfig_ComparePassword(t *testing.T) { + a := assert.NewAssertion(t) + + { + config := AdminConfig{} + a.IsTrue(config.ComparePassword("123456", "123456")) + } + + { + config := AdminConfig{} + a.IsTrue(config.ComparePassword("123456", "clear:123456")) + } + + { + config := AdminConfig{} + a.IsTrue(config.ComparePassword("123456", "md5:"+stringutil.Md5("123456"))) + a.IsFalse(config.ComparePassword("123456789", "md5:"+stringutil.Md5("123456"))) + } +} diff --git a/teaweb/configs/admin_user.go b/teaweb/configs/admin_user.go index 7ad232a..3f99f14 100644 --- a/teaweb/configs/admin_user.go +++ b/teaweb/configs/admin_user.go @@ -64,7 +64,7 @@ func (this *AdminUser) Granted(grant string) bool { func (this *AdminUser) IncreaseLoginTries() { this.locker.Lock() defer this.locker.Unlock() - this.countLoginTries ++ + this.countLoginTries++ } func (this *AdminUser) CountLoginTries() uint { From 516d1a847b0c657e6e3b03d724e9c4f8a95dd95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月22日 23:09:32 +0800 Subject: [PATCH 005/121] =?UTF-8?q?[proxy]=E6=94=AF=E6=8C=81FTP=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=E6=9C=8D=E5=8A=A1=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/backend.go | 10 +- teaconfigs/backend_ftp.go | 8 ++ teaproxy/ftp_client.go | 119 ++++++++++++++++++ teaproxy/ftp_client_pool.go | 76 +++++++++++ teaproxy/ftp_client_test.go | 91 ++++++++++++++ teaproxy/ftp_connection_pool.go | 114 +++++++++++++++++ teaproxy/ftp_connection_pool_test.go | 98 +++++++++++++++ teaproxy/ftp_response_body.go | 18 +++ teaproxy/{client.go => http_client_pool.go} | 14 +-- ...lient_test.go => http_client_pool_test.go} | 0 teaproxy/request_backend.go | 20 ++- teaweb/actions/default/proxy/backend/add.go | 13 ++ .../actions/default/proxy/backend/update.go | 18 +++ 13 files changed, 586 insertions(+), 13 deletions(-) create mode 100644 teaconfigs/backend_ftp.go create mode 100644 teaproxy/ftp_client.go create mode 100644 teaproxy/ftp_client_pool.go create mode 100644 teaproxy/ftp_client_test.go create mode 100644 teaproxy/ftp_connection_pool.go create mode 100644 teaproxy/ftp_connection_pool_test.go create mode 100644 teaproxy/ftp_response_body.go rename teaproxy/{client.go => http_client_pool.go} (89%) rename teaproxy/{client_test.go => http_client_pool_test.go} (100%) diff --git a/teaconfigs/backend.go b/teaconfigs/backend.go index 09fb813..572f718 100644 --- a/teaconfigs/backend.go +++ b/teaconfigs/backend.go @@ -28,7 +28,7 @@ type BackendConfig struct { Version int `yaml:"version" json:"version"` // 版本号 Code string `yaml:"code" json:"code"` // 代号 Address string `yaml:"address" json:"address"` // 地址 - Scheme string `yaml:"scheme" json:"scheme"` // 协议,http、https、tcp、tcp+tls + Scheme string `yaml:"scheme" json:"scheme"` // 协议,http、https、tcp、tcp+tls、ftp Weight uint `yaml:"weight" json:"weight"` // 权重 IsBackup bool `yaml:"backup" json:"isBackup"` // 是否为备份 FailTimeout string `yaml:"failTimeout" json:"failTimeout"` // 连接失败超时 @@ -57,6 +57,9 @@ type BackendConfig struct { Cert *SSLCertConfig `yaml:"cert" json:"cert"` // 请求源服务器用的证书 + // ftp + FTP *FTPBackendConfig `yaml:"ftp" json:"ftp"` + failTimeoutDuration time.Duration readTimeoutDuration time.Duration idleTimeoutDuration time.Duration @@ -460,3 +463,8 @@ func (this *BackendConfig) IsHTTP() bool { func (this *BackendConfig) IsTCP() bool { return this.Scheme == "tcp" || this.Scheme == "tcp+tls" } + +// 是否为FTP +func (this *BackendConfig) IsFTP() bool { + return this.Scheme == "ftp" +} diff --git a/teaconfigs/backend_ftp.go b/teaconfigs/backend_ftp.go new file mode 100644 index 0000000..0c19bcc --- /dev/null +++ b/teaconfigs/backend_ftp.go @@ -0,0 +1,8 @@ +package teaconfigs + +// backend +type FTPBackendConfig struct { + Username string `yaml:"username" json:"username"` // 用户名 + Password string `yaml:"password" json:"password"` // 密码 + Dir string `yaml:"dir" json:"dir"` // 目录 +} diff --git a/teaproxy/ftp_client.go b/teaproxy/ftp_client.go new file mode 100644 index 0000000..191c4df --- /dev/null +++ b/teaproxy/ftp_client.go @@ -0,0 +1,119 @@ +package teaproxy + +import ( + "bytes" + "github.com/TeaWeb/code/teaconfigs" + "github.com/jlaffaye/ftp" + "io/ioutil" + "mime" + "net/http" + "net/textproto" + "path/filepath" + "strings" + "time" +) + +// FTP客户端 +type FTPClient struct { + backend *teaconfigs.BackendConfig + pool *FTPConnectionPool +} + +// 执行请求 +func (this *FTPClient) Do(req *http.Request) (*http.Response, error) { + return this.doRetries(req, 0) +} + +// 执行请求 +func (this *FTPClient) doRetries(req *http.Request, retries int) (*http.Response, error) { + conn, err := this.pool.Get() + if err != nil { + // retry + if err == ErrFTPTooManyConnections && retries == 0 { + time.Sleep(1 * time.Second) + return this.doRetries(req, retries+1) + } + return nil, err + } + + // file size + path := strings.TrimLeft(req.URL.Path, "/") + size, err := conn.FileSize(path) + if err != nil { + isDisconnected := false + textErr, ok := err.(*textproto.Error) + if ok { + if textErr.Code != ftp.StatusFileUnavailable && + textErr.Code != ftp.StatusBadFileName { + isDisconnected = true + } else { + // not found + this.pool.Put(conn) + message := "404 page not found" + return &http.Response{ + Status: http.StatusText(http.StatusNotFound), + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(message))), + ContentLength: int64(len(message)), + Close: false, + }, nil + } + } else { + isDisconnected = true + } + + // retry + if isDisconnected { + this.pool.Decrease() + _ = this.pool.CloseAll() + + if retries == 0 { + return this.doRetries(req, retries+1) + } + } else { + this.pool.Put(conn) + } + return nil, err + } + + // read file + response, err := conn.Retr(path) + if err != nil { + if response != nil { + _ = response.Close() + } + return nil, err + } + + ext := filepath.Ext(path) + headers := map[string][]string{} + if len(ext)> 0 { + mimeType := mime.TypeByExtension(ext) + if len(mimeType)> 0 { + headers["Content-Type"] = []string{mimeType} + } + } + + resp := &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Header: headers, + Body: &FTPResponseBody{ + ReadCloser: response, + callback: func() { + this.pool.Put(conn) + }, + }, + ContentLength: size, + Close: false, + } + return resp, nil +} + +// 关闭客户端 +func (this *FTPClient) Close() error { + if this.pool == nil { + return nil + } + return this.pool.CloseAll() +} diff --git a/teaproxy/ftp_client_pool.go b/teaproxy/ftp_client_pool.go new file mode 100644 index 0000000..8a1919b --- /dev/null +++ b/teaproxy/ftp_client_pool.go @@ -0,0 +1,76 @@ +package teaproxy + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/jlaffaye/ftp" + "runtime" + "strings" + "sync" +) + +// FTP客户端池单例 +var SharedFTPClientPool = NewFTPClientPool() + +// FTP客户端Pool +type FTPClientPool struct { + clientMap map[string]*FTPClient // key => client + locker sync.Mutex +} + +// 获取新的客户端Pool +func NewFTPClientPool() *FTPClientPool { + return &FTPClientPool{ + clientMap: map[string]*FTPClient{}, + } +} + +// 通过Backend配置FTP客户端 +func (this *FTPClientPool) client(backend *teaconfigs.BackendConfig) *FTPClient { + key := backend.UniqueKey() + + this.locker.Lock() + defer this.locker.Unlock() + client, ok := this.clientMap[key] + if ok { + return client + } + + // 关闭以前的连接 + this.closeOldClients(key) + + if backend.FTP == nil { + backend.FTP = &teaconfigs.FTPBackendConfig{} + } + if backend.IdleConns <= 0 { + backend.IdleConns = int32(runtime.NumCPU()) + } + client = &FTPClient{ + pool: &FTPConnectionPool{ + addr: backend.Address, + username: backend.FTP.Username, + password: backend.FTP.Password, + dir: backend.FTP.Dir, + timeout: backend.FailTimeoutDuration(), + c: make(chan *ftp.ServerConn, backend.IdleConns), + maxConnections: int64(backend.MaxConns), + }, + } + this.clientMap[key] = client + + return client +} + +// 关闭老的client +func (this *FTPClientPool) closeOldClients(key string) { + backendId := strings.Split(key, "@")[0] + for key2, client := range this.clientMap { + backendId2 := strings.Split(key2, "@")[0] + if backendId == backendId2 && key != key2 { + go func() { + _ = client.Close() + }() + delete(this.clientMap, key2) + break + } + } +} diff --git a/teaproxy/ftp_client_test.go b/teaproxy/ftp_client_test.go new file mode 100644 index 0000000..4294aa4 --- /dev/null +++ b/teaproxy/ftp_client_test.go @@ -0,0 +1,91 @@ +package teaproxy + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/TeaWeb/code/teatesting" + "io/ioutil" + "net/http" + "testing" +) + +func TestFTPClient_Do(t *testing.T) { + if teatesting.IsGlobal() { + return + } + + backend := &teaconfigs.BackendConfig{ + Address: "192.168.2.30:21", + FTP: &teaconfigs.FTPBackendConfig{ + Username: "www", + Password: "123456", + Dir: "", + }, + } + client := SharedFTPClientPool.client(backend) + + for _, file := range []string{"/index.html", "index.a", "/dir1/dir2/hello.txt"} { + func() { + req, err := http.NewRequest(http.MethodGet, file, nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) + if err != nil { + t.Log(file+":", err.Error()) + return + } + defer func() { + _ = resp.Body.Close() + }() + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Log(file+":", err.Error()) + return + } + + t.Log(file+":", string(data)) + }() + } +} + +func TestFTPClient_Do_ChangeDir(t *testing.T) { + if teatesting.IsGlobal() { + return + } + + backend := &teaconfigs.BackendConfig{ + Address: "192.168.2.30:21", + FTP: &teaconfigs.FTPBackendConfig{ + Username: "www", + Password: "123456", + Dir: "/dir1/dir2", + }, + } + client := SharedFTPClientPool.client(backend) + + for _, file := range []string{"hello.txt", "hello1.txt", "hello2.txt"} { + func() { + req, err := http.NewRequest(http.MethodGet, file, nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) + if err != nil { + t.Log(file+":", err.Error()) + return + } + defer func() { + _ = resp.Body.Close() + }() + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Log(file+":", err.Error()) + return + } + + t.Log(file+":", string(data)) + }() + } +} diff --git a/teaproxy/ftp_connection_pool.go b/teaproxy/ftp_connection_pool.go new file mode 100644 index 0000000..3c32ffc --- /dev/null +++ b/teaproxy/ftp_connection_pool.go @@ -0,0 +1,114 @@ +package teaproxy + +import ( + "errors" + "github.com/jlaffaye/ftp" + "runtime" + "strings" + "sync/atomic" + "time" +) + +var ErrFTPTooManyConnections = errors.New("ftp: too many connections") + +// FTP连接池 +type FTPConnectionPool struct { + addr string + username string + password string + timeout time.Duration + dir string + + currentConnections int64 + maxConnections int64 + + c chan *ftp.ServerConn +} + +// 获取新的连接 +func (this *FTPConnectionPool) Get() (*ftp.ServerConn, error) { + if this.timeout <= 0 { + this.timeout = 5 * time.Second + } + if this.maxConnections <= 0 { + this.maxConnections = int64(runtime.NumCPU()) + } + + select { + case client := <-this.c: + return client, nil + default: + if atomic.LoadInt64(&this.currentConnections)>= this.maxConnections { + return nil, ErrFTPTooManyConnections + } + + this.Increase() + + // create a new connection + client, err := ftp.DialTimeout(this.addr, this.timeout) + if err != nil { + this.Decrease() + return nil, err + } + if len(this.username)> 0 { + err = client.Login(this.username, this.password) + if err != nil { + this.Decrease() + _ = client.Quit() + return nil, err + } + } + if len(this.dir)> 0 { + err = client.ChangeDir(strings.TrimLeft(this.dir, "/")) + if err != nil { + this.Decrease() + _ = client.Quit() + return nil, err + } + } + + return client, err + } +} + +// 复用连接 +func (this *FTPConnectionPool) Put(client *ftp.ServerConn) { + select { + case this.c <- client: + default: + this.Decrease() + _ = client.Quit() + } +} + +// 关闭连接 +func (this *FTPConnectionPool) Close(client *ftp.ServerConn) error { + this.Decrease() + err := client.Quit() + return err +} + +// 关闭所有连接 +func (this *FTPConnectionPool) CloseAll() error { +FOR: + for { + select { + case conn := <-this.c: + this.Decrease() + _ = conn.Quit() + default: + break FOR + } + } + return nil +} + +// 增加连接数量 +func (this *FTPConnectionPool) Increase() { + atomic.AddInt64(&this.currentConnections, 1) +} + +// 减少连接数量 +func (this *FTPConnectionPool) Decrease() { + atomic.AddInt64(&this.currentConnections, -1) +} diff --git a/teaproxy/ftp_connection_pool_test.go b/teaproxy/ftp_connection_pool_test.go new file mode 100644 index 0000000..ec9d0e2 --- /dev/null +++ b/teaproxy/ftp_connection_pool_test.go @@ -0,0 +1,98 @@ +package teaproxy + +import ( + "github.com/TeaWeb/code/teatesting" + "github.com/jlaffaye/ftp" + "math/rand" + "sync" + "testing" + "time" +) + +func TestFTPConnectionPool_Get(t *testing.T) { + if teatesting.IsGlobal() { + return + } + pool := &FTPConnectionPool{ + addr: "192.168.2.30:21", + timeout: 5 * time.Second, + dir: "", + c: make(chan *ftp.ServerConn, 5), + } + wg := sync.WaitGroup{} + count := 100 + wg.Add(count) + for i := 0; i < count; i++ { + go func() { + defer wg.Done() + time.Sleep(time.Duration(rand.Int()%10) * time.Second) + client, err := pool.Get() + if err != nil { + t.Fatal(err) + } + pool.Put(client) + }() + } + wg.Wait() + t.Log(len(pool.c)) +} + +func TestFTPConnectionPool_Get2(t *testing.T) { + if teatesting.IsGlobal() { + return + } + pool := &FTPConnectionPool{ + addr: "192.168.2.31:21", + timeout: 5 * time.Second, + dir: "", + c: make(chan *ftp.ServerConn, 5), + } + wg := sync.WaitGroup{} + count := 100 + wg.Add(count) + for i := 0; i < count; i++ { + go func() { + defer wg.Done() + time.Sleep(time.Duration(rand.Int()%10) * time.Second) + client, err := pool.Get() + if err != nil { + t.Fatal(err) + } + pool.Put(client) + }() + } + wg.Wait() + t.Log(len(pool.c)) +} + +func TestChanFull_Write(t *testing.T) { + c := make(chan int, 5) + for i := 0; i < 5; i++ { + c <- i + } + t.Log(len(c)) + select { + case c <- 6: + t.Log("write 6") + default: + t.Log("write failed") + } +} + +func TestChanFull_Read(t *testing.T) { + c := make(chan int, 5) + for i := 0; i < 5; i++ { + c <- i + } + +FOR: + for { + select { + case x := <-c: + t.Log("read", x) + default: + t.Log("read failed") + break FOR + } + } +} diff --git a/teaproxy/ftp_response_body.go b/teaproxy/ftp_response_body.go new file mode 100644 index 0000000..4ba03a0 --- /dev/null +++ b/teaproxy/ftp_response_body.go @@ -0,0 +1,18 @@ +package teaproxy + +import "io" + +// FTP响应内容 +type FTPResponseBody struct { + io.ReadCloser + callback func() // 关闭时回调 +} + +// 关闭 +func (this *FTPResponseBody) Close() error { + err := this.ReadCloser.Close() + if this.callback != nil { + this.callback() + } + return err +} diff --git a/teaproxy/client.go b/teaproxy/http_client_pool.go similarity index 89% rename from teaproxy/client.go rename to teaproxy/http_client_pool.go index 68a6075..0a17bc4 100644 --- a/teaproxy/client.go +++ b/teaproxy/http_client_pool.go @@ -13,24 +13,24 @@ import ( "time" ) -// 客户端池单例 -var SharedClientPool = NewClientPool() +// HTTP客户端池单例 +var SharedHTTPClientPool = NewHTTPClientPool() // 客户端池 -type ClientPool struct { +type HTTPClientPool struct { clientsMap map[string]*http.Client // backend key => client locker sync.RWMutex } // 获取新对象 -func NewClientPool() *ClientPool { - return &ClientPool{ +func NewHTTPClientPool() *HTTPClientPool { + return &HTTPClientPool{ clientsMap: map[string]*http.Client{}, } } // 根据地址获取客户端 -func (this *ClientPool) client(backend *teaconfigs.BackendConfig) *http.Client { +func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig) *http.Client { key := backend.UniqueKey() this.locker.RLock() @@ -122,7 +122,7 @@ func (this *ClientPool) client(backend *teaconfigs.BackendConfig) *http.Client { } // 关闭老的client -func (this *ClientPool) closeOldClient(key string) { +func (this *HTTPClientPool) closeOldClient(key string) { backendId := strings.Split(key, "@")[0] for key2, client := range this.clientsMap { backendId2 := strings.Split(key2, "@")[0] diff --git a/teaproxy/client_test.go b/teaproxy/http_client_pool_test.go similarity index 100% rename from teaproxy/client_test.go rename to teaproxy/http_client_pool_test.go diff --git a/teaproxy/request_backend.go b/teaproxy/request_backend.go index 9ce0eb0..27c27d1 100644 --- a/teaproxy/request_backend.go +++ b/teaproxy/request_backend.go @@ -7,6 +7,7 @@ import ( "github.com/TeaWeb/code/teaevents" "github.com/iwind/TeaGo/logs" "io" + "net/http" "net/url" "strings" "time" @@ -51,7 +52,7 @@ func (this *Request) callBackend(writer *ResponseWriter) error { this.raw.URL.Host = this.host } - if len(this.backend.Scheme)> 0 && this.backend.Scheme != "http" { + if len(this.backend.Scheme)> 0 && this.backend.Scheme != "http" && this.backend.Scheme != "ftp" { this.raw.URL.Scheme = this.backend.Scheme } else { this.raw.URL.Scheme = this.scheme @@ -108,11 +109,17 @@ func (this *Request) callBackend(writer *ResponseWriter) error { } } - client := SharedClientPool.client(this.backend) - this.raw.RequestURI = "" - resp, err := client.Do(this.raw) + var resp *http.Response = nil + var err error = nil + if this.backend.IsFTP() { + client := SharedFTPClientPool.client(this.backend) + resp, err = client.Do(this.raw) + } else { + client := SharedHTTPClientPool.client(this.backend) + resp, err = client.Do(this.raw) + } if err != nil { // 客户端取消请求,则不提示 httpErr, ok := err.(*url.Error) @@ -142,12 +149,15 @@ func (this *Request) callBackend(writer *ResponseWriter) error { this.serverError(writer) - logs.Println("[proxy]" + err.Error()) + logs.Println("[proxy]'" + this.raw.URL.String() + "': " + err.Error()) this.addError(err) } else { this.serverError(writer) this.addError(err) } + if resp != nil && resp.Body != nil { + _ = resp.Body.Close() + } return nil } diff --git a/teaweb/actions/default/proxy/backend/add.go b/teaweb/actions/default/proxy/backend/add.go index efd822b..1e68fc4 100644 --- a/teaweb/actions/default/proxy/backend/add.go +++ b/teaweb/actions/default/proxy/backend/add.go @@ -60,6 +60,10 @@ func (this *AddAction) RunPost(params struct { CertId string CertServerName string + FtpDir string + FtpUsername string + FtpPassword string + Weight uint On bool Code string @@ -179,6 +183,15 @@ func (this *AddAction) RunPost(params struct { backend.Host = params.Host + // ftp + if params.Scheme == "ftp" { + backend.FTP = &teaconfigs.FTPBackendConfig{ + Username: params.FtpUsername, + Password: params.FtpPassword, + Dir: params.FtpDir, + } + } + backendList, err := server.FindBackendList(params.LocationId, params.Websocket) if err != nil { this.Fail(err.Error()) diff --git a/teaweb/actions/default/proxy/backend/update.go b/teaweb/actions/default/proxy/backend/update.go index 3bbde10..848b2a2 100644 --- a/teaweb/actions/default/proxy/backend/update.go +++ b/teaweb/actions/default/proxy/backend/update.go @@ -66,6 +66,10 @@ func (this *UpdateAction) Run(params struct { backend.RequestURI = "${requestURI}" } + if backend.FTP == nil { + backend.FTP = &teaconfigs.FTPBackendConfig{} + } + this.Data["backend"] = maps.Map{ "id": backend.Id, "address": backend.Address, @@ -91,6 +95,7 @@ func (this *UpdateAction) Run(params struct { "responseHeaders": backend.ResponseHeaders, "host": backend.Host, "cert": backend.Cert, + "ftp": backend.FTP, } // 公共可以使用的证书 @@ -130,6 +135,10 @@ func (this *UpdateAction) RunPost(params struct { CheckInterval int CheckTimeout string + FtpUsername string + FtpPassword string + FtpDir string + RequestHeaderNames []string RequestHeaderValues []string @@ -246,6 +255,15 @@ func (this *UpdateAction) RunPost(params struct { backend.Host = params.Host + // ftp + if params.Scheme == "ftp" { + backend.FTP = &teaconfigs.FTPBackendConfig{ + Username: params.FtpUsername, + Password: params.FtpPassword, + Dir: params.FtpDir, + } + } + err = server.Save() if err != nil { this.Fail("保存失败:" + err.Error()) From cf030e816a0172db45d9a67bc0654a1e7588ebfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月23日 10:53:30 +0800 Subject: [PATCH 006/121] =?UTF-8?q?[proxy]=E5=B0=86=E5=90=8E=E7=AB=AF?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=99=A8=E5=BD=93=E5=89=8D=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E6=95=B0=E6=94=B9=E6=88=90=E5=BD=93=E5=89=8D=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/proxy/backend/index.go | 2 ++ teaweb/actions/default/proxy/locations/backends/index.go | 4 +++- teaweb/actions/default/proxy/locations/websocket/index.go | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/teaweb/actions/default/proxy/backend/index.go b/teaweb/actions/default/proxy/backend/index.go index a1f93fb..02c5371 100644 --- a/teaweb/actions/default/proxy/backend/index.go +++ b/teaweb/actions/default/proxy/backend/index.go @@ -20,6 +20,8 @@ func (this *IndexAction) Run(params struct { this.Data["selectedTab"] = "backend" this.Data["server"] = server this.Data["location"] = nil + this.Data["isTCP"] = server.IsTCP() + this.Data["isHTTP"] = server.IsHTTP() this.Data["queryParams"] = maps.Map{ "serverId": params.ServerId, diff --git a/teaweb/actions/default/proxy/locations/backends/index.go b/teaweb/actions/default/proxy/locations/backends/index.go index 1bc3226..676239f 100644 --- a/teaweb/actions/default/proxy/locations/backends/index.go +++ b/teaweb/actions/default/proxy/locations/backends/index.go @@ -18,7 +18,9 @@ func (this *IndexAction) Run(params struct { "locationId": params.LocationId, } - locationutils.SetCommonInfo(this, params.ServerId, params.LocationId, "backends") + server, _ := locationutils.SetCommonInfo(this, params.ServerId, params.LocationId, "backends") + this.Data["isTCP"] = server.IsTCP() + this.Data["isHTTP"] = server.IsHTTP() this.Show() } diff --git a/teaweb/actions/default/proxy/locations/websocket/index.go b/teaweb/actions/default/proxy/locations/websocket/index.go index 1b248b9..fdbfcea 100644 --- a/teaweb/actions/default/proxy/locations/websocket/index.go +++ b/teaweb/actions/default/proxy/locations/websocket/index.go @@ -19,7 +19,7 @@ func (this *IndexAction) Run(params struct { "websocket": 1, } - _, location := locationutils.SetCommonInfo(this, params.ServerId, params.LocationId, "websocket") + server, location := locationutils.SetCommonInfo(this, params.ServerId, params.LocationId, "websocket") if location.Websocket == nil { this.Data["websocket"] = nil @@ -33,5 +33,8 @@ func (this *IndexAction) Run(params struct { } } + this.Data["isTCP"] = server.IsTCP() + this.Data["isHTTP"] = server.IsHTTP() + this.Show() } From 852dc783738f2957851d5f55335014522cf020e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月23日 11:14:08 +0800 Subject: [PATCH 007/121] =?UTF-8?q?[proxy]=E4=BB=A3=E7=90=86=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E5=88=97=E8=A1=A8=E6=98=BE=E7=A4=BA=E7=AB=AF=E5=8F=A3?= =?UTF-8?q?=E5=92=8C=E6=9C=AA=E5=90=AF=E7=94=A8=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../actions/default/proxy/proxyutils/menu.go | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/teaweb/actions/default/proxy/proxyutils/menu.go b/teaweb/actions/default/proxy/proxyutils/menu.go index 16899a9..bca3ade 100644 --- a/teaweb/actions/default/proxy/proxyutils/menu.go +++ b/teaweb/actions/default/proxy/proxyutils/menu.go @@ -6,6 +6,7 @@ import ( "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" + "strings" ) // 添加服务器菜单 @@ -53,6 +54,38 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { item.SupName = "tcp" } + // port + ports := []string{} + if server.Http { + for _, listen := range server.Listen { + index := strings.LastIndex(listen, ":") + if index> -1 { + ports = append(ports, listen[index+1:]) + } + } + } + if server.SSL != nil && server.SSL.On { + for _, listen := range server.SSL.Listen { + index := strings.LastIndex(listen, ":") + if index> -1 { + ports = append(ports, listen[index+1:]) + } + } + } + if len(ports)> 0 { + if len(ports)> 1 { + item.SubName = "Port: " + ports[0] + "等 " + } else { + item.SubName = "Port: " + ports[0] + } + } + + // on | off + if (server.IsHTTP() && !server.Http && (server.SSL == nil || !server.SSL.On)) || (server.IsTCP() && (server.TCP == nil || !server.TCP.TCPOn)) { + item.SubName = "未启用" + item.SubColor = "red" + } + if server.Id == serverId { isTCP = server.IsTCP() } From c37a502a88c55a5fd57ebd76b04afbdad77d7787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月23日 11:33:34 +0800 Subject: [PATCH 008/121] =?UTF-8?q?=E5=A4=87=E4=BB=BD=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=B8=85=E9=99=A430=E5=A4=A9=E4=BB=A5?= =?UTF-8?q?=E5=A4=96=E7=9A=84=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../actions/default/settings/backup/clean.go | 40 +++++++++++++++++++ .../actions/default/settings/backup/init.go | 9 ++++- .../default/settings/backup/init_test.go | 12 ++++-- 3 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 teaweb/actions/default/settings/backup/clean.go diff --git a/teaweb/actions/default/settings/backup/clean.go b/teaweb/actions/default/settings/backup/clean.go new file mode 100644 index 0000000..d58788d --- /dev/null +++ b/teaweb/actions/default/settings/backup/clean.go @@ -0,0 +1,40 @@ +package backup + +import ( + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/files" + timeutil "github.com/iwind/TeaGo/utils/time" + "regexp" + "time" +) + +type CleanAction actions.Action + +// 清除N天以前的备份文件 +func (this *CleanAction) RunPost(params struct{}) { + oldDay := timeutil.Format("Ymd", time.Now().AddDate(0, 0, -30)) + ".zip" + + reg := regexp.MustCompile(`^\d{8}\.zip$`) + dir := files.NewFile(Tea.Root + "/backups/") + var lastErr error = nil + if dir.Exists() { + for _, f := range dir.List() { + if !reg.MatchString(f.Name()) { + continue + } + if f.Name() < oldDay { + err := f.Delete() + if err != nil { + lastErr = err + break + } + } + } + } + if lastErr != nil { + this.Fail("删除失败:" + lastErr.Error()) + } else { + this.Success() + } +} diff --git a/teaweb/actions/default/settings/backup/init.go b/teaweb/actions/default/settings/backup/init.go index 019b1a6..d766dfb 100644 --- a/teaweb/actions/default/settings/backup/init.go +++ b/teaweb/actions/default/settings/backup/init.go @@ -31,6 +31,7 @@ func init() { Post("/delete", new(DeleteAction)). Post("/restore", new(RestoreAction)). Get("/download", new(DownloadAction)). + Post("/clean", new(CleanAction)). EndAll() }) @@ -65,10 +66,14 @@ func backupTask() error { if err != nil { return err } - defer fp.Close() + defer func() { + _ = fp.Close() + }() z := zip.NewWriter(fp) - defer z.Close() + defer func() { + _ = z.Close() + }() configsDir := files.NewFile(Tea.Root + "/configs") configsAbs, _ := configsDir.AbsPath() diff --git a/teaweb/actions/default/settings/backup/init_test.go b/teaweb/actions/default/settings/backup/init_test.go index 0939717..2d9af1b 100644 --- a/teaweb/actions/default/settings/backup/init_test.go +++ b/teaweb/actions/default/settings/backup/init_test.go @@ -18,10 +18,14 @@ func TestBackupZip(t *testing.T) { if err != nil { t.Fatal(err) } - defer fp.Close() + defer func() { + _ = fp.Close() + }() z := zip.NewWriter(fp) - defer z.Close() + defer func() { + _ = z.Close() + }() { h := &zip.FileHeader{ @@ -32,7 +36,7 @@ func TestBackupZip(t *testing.T) { if err != nil { t.Fatal(err) } - w.Write([]byte("Hello, World")) + _, _ = w.Write([]byte("Hello, World")) } { @@ -44,7 +48,7 @@ func TestBackupZip(t *testing.T) { if err != nil { t.Fatal(err) } - w.Write([]byte("Hello, Hello")) + _, _ = w.Write([]byte("Hello, Hello")) } } From f3d260419274c46662c6cbe19399337c4b43799e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月23日 11:38:25 +0800 Subject: [PATCH 009/121] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B8=85=E9=99=A4?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=E6=96=87=E4=BB=B6=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/settings/backup/clean.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/teaweb/actions/default/settings/backup/clean.go b/teaweb/actions/default/settings/backup/clean.go index d58788d..26c8969 100644 --- a/teaweb/actions/default/settings/backup/clean.go +++ b/teaweb/actions/default/settings/backup/clean.go @@ -18,6 +18,7 @@ func (this *CleanAction) RunPost(params struct{}) { reg := regexp.MustCompile(`^\d{8}\.zip$`) dir := files.NewFile(Tea.Root + "/backups/") var lastErr error = nil + count := 0 if dir.Exists() { for _, f := range dir.List() { if !reg.MatchString(f.Name()) { @@ -29,11 +30,14 @@ func (this *CleanAction) RunPost(params struct{}) { lastErr = err break } + count++ } } } + + this.Data["count"] = count if lastErr != nil { - this.Fail("删除失败:" + lastErr.Error()) + this.Fail("清除失败:" + lastErr.Error()) } else { this.Success() } From cc6fccb669b7fc10e6aae236e3ef8436c4cff72b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月23日 18:14:01 +0800 Subject: [PATCH 010/121] =?UTF-8?q?[proxy]=E8=87=AA=E5=8A=A8=E4=B8=BA?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E8=A7=84=E5=88=99=E5=89=8D=E9=9D=A2=E5=8A=A0?= =?UTF-8?q?=E4=B8=8A=E6=96=9C=E6=9D=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/proxy/locations/add.go | 7 +++++++ teaweb/actions/default/proxy/locations/update.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/teaweb/actions/default/proxy/locations/add.go b/teaweb/actions/default/proxy/locations/add.go index 08d8f50..4bda160 100644 --- a/teaweb/actions/default/proxy/locations/add.go +++ b/teaweb/actions/default/proxy/locations/add.go @@ -11,6 +11,7 @@ import ( "github.com/iwind/TeaGo/types" "regexp" "strconv" + "strings" ) type AddAction actions.Action @@ -111,6 +112,12 @@ func (this *AddAction) RunPost(params struct { } } + // 自动加上前缀斜杠 + if params.PatternType == teaconfigs.LocationPatternTypePrefix || + params.PatternType == teaconfigs.LocationPatternTypeExact { + params.Pattern = "/" + strings.TrimLeft(params.Pattern, "/") + } + location := teaconfigs.NewLocation() // 匹配条件 diff --git a/teaweb/actions/default/proxy/locations/update.go b/teaweb/actions/default/proxy/locations/update.go index 62b3b82..d14342b 100644 --- a/teaweb/actions/default/proxy/locations/update.go +++ b/teaweb/actions/default/proxy/locations/update.go @@ -13,6 +13,7 @@ import ( "github.com/iwind/TeaGo/types" "regexp" "strconv" + "strings" ) type UpdateAction actions.Action @@ -153,6 +154,12 @@ func (this *UpdateAction) RunPost(params struct { } } + // 自动加上前缀斜杠 + if params.PatternType == teaconfigs.LocationPatternTypePrefix || + params.PatternType == teaconfigs.LocationPatternTypeExact { + params.Pattern = "/" + strings.TrimLeft(params.Pattern, "/") + } + // 匹配条件 conds, breakCond, err := proxyutils.ParseRequestConds(this.Request, "request") if err != nil { From 67feb794c33109f0d20dba1c22402e6cdf6f2084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月23日 20:10:04 +0800 Subject: [PATCH 011/121] =?UTF-8?q?[proxy]=E8=AE=BF=E9=97=AE=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E8=A2=AB=E9=99=90=E5=88=B6=E9=A1=B5=E9=9D=A2=E5=8F=AF?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=89=B9=E6=AE=8A=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/teaproxy/request.go b/teaproxy/request.go index ed37457..a64d0d6 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -755,15 +755,19 @@ func (this *Request) callBegin(writer *ResponseWriter) error { // access policy if this.accessPolicy != nil { if !this.accessPolicy.AllowAccess(this.requestRemoteAddr()) { - writer.WriteHeader(http.StatusForbidden) - _, _ = writer.Write([]byte("Forbidden Request")) + if !this.callPage(writer, http.StatusForbidden) { + writer.WriteHeader(http.StatusForbidden) + _, _ = writer.Write([]byte("Request Forbidden")) + } return nil } reason, allowed := this.accessPolicy.AllowTraffic() if !allowed { - writer.WriteHeader(http.StatusTooManyRequests) - _, _ = writer.Write([]byte("[" + reason + "]Request Quota Exceeded")) + if !this.callPage(writer, http.StatusTooManyRequests) { + writer.WriteHeader(http.StatusTooManyRequests) + _, _ = writer.Write([]byte("[" + reason + "]Request Quota Exceeded")) + } return nil } } From c1106d343cf559383e6cc2aedceae6ddef72df46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月24日 16:26:56 +0800 Subject: [PATCH 012/121] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=95=8C=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../actions/default/proxy/proxyutils/menu.go | 10 +++++----- .../default/settings/server/httpUpdate.go | 19 ++++++++++--------- .../default/settings/server/httpsUpdate.go | 16 +++++++++------- .../default/settings/server/security.go | 8 ++++---- teaweb/utils/log_writer.go | 17 +++++++++++++---- 5 files changed, 41 insertions(+), 29 deletions(-) diff --git a/teaweb/actions/default/proxy/proxyutils/menu.go b/teaweb/actions/default/proxy/proxyutils/menu.go index bca3ade..cf30ec8 100644 --- a/teaweb/actions/default/proxy/proxyutils/menu.go +++ b/teaweb/actions/default/proxy/proxyutils/menu.go @@ -60,7 +60,7 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { for _, listen := range server.Listen { index := strings.LastIndex(listen, ":") if index> -1 { - ports = append(ports, listen[index+1:]) + ports = append(ports, ":"+listen[index+1:]) } } } @@ -68,15 +68,15 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { for _, listen := range server.SSL.Listen { index := strings.LastIndex(listen, ":") if index> -1 { - ports = append(ports, listen[index+1:]) + ports = append(ports, ":"+listen[index+1:]) } } } if len(ports)> 0 { - if len(ports)> 1 { - item.SubName = "Port: " + ports[0] + "等 " + if len(ports)> 2 { + item.SubName = ports[0] + ", " + ports[1] + "等 " } else { - item.SubName = "Port: " + ports[0] + item.SubName = strings.Join(ports, ", ") + " " } } diff --git a/teaweb/actions/default/settings/server/httpUpdate.go b/teaweb/actions/default/settings/server/httpUpdate.go index dc4aeba..3b3ad15 100644 --- a/teaweb/actions/default/settings/server/httpUpdate.go +++ b/teaweb/actions/default/settings/server/httpUpdate.go @@ -2,22 +2,23 @@ package server import ( "github.com/TeaWeb/code/teaconfigs" + "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teaweb/actions/default/settings" "github.com/iwind/TeaGo/actions" "net" - "strings" ) type HttpUpdateAction actions.Action +// 保存HTTP设置 func (this *HttpUpdateAction) Run(params struct { - On bool - Addresses string - Must *actions.Must + On bool + ListenValues []string + Must *actions.Must }) { - params.Must. - Field("addresses", params.Addresses). - Require("请输入绑定地址") + if len(params.ListenValues) == 0 { + this.Fail("请输入绑定地址") + } server, err := teaconfigs.LoadWebConfig() if err != nil { @@ -27,8 +28,8 @@ func (this *HttpUpdateAction) Run(params struct { server.Http.On = params.On listen := []string{} - for _, addr := range strings.Split(params.Addresses, "\n") { - addr = strings.TrimSpace(addr) + for _, addr := range params.ListenValues { + addr = teautils.FormatAddress(addr) if len(addr) == 0 { continue } diff --git a/teaweb/actions/default/settings/server/httpsUpdate.go b/teaweb/actions/default/settings/server/httpsUpdate.go index 0bc5947..fe4a937 100644 --- a/teaweb/actions/default/settings/server/httpsUpdate.go +++ b/teaweb/actions/default/settings/server/httpsUpdate.go @@ -2,6 +2,7 @@ package server import ( "github.com/TeaWeb/code/teaconfigs" + "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teaweb/actions/default/settings" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/actions" @@ -12,9 +13,10 @@ import ( type HttpsUpdateAction actions.Action +// 保存HTTPS设置 func (this *HttpsUpdateAction) Run(params struct { - On bool - Addresses string + On bool + ListenValues []string CertType string CertFile *actions.File @@ -23,9 +25,9 @@ func (this *HttpsUpdateAction) Run(params struct { Must *actions.Must }) { - params.Must. - Field("addresses", params.Addresses). - Require("请输入绑定地址") + if len(params.ListenValues) == 0 { + this.Fail("请输入绑定地址") + } server, err := teaconfigs.LoadWebConfig() if err != nil { @@ -35,8 +37,8 @@ func (this *HttpsUpdateAction) Run(params struct { server.Https.On = params.On listen := []string{} - for _, addr := range strings.Split(params.Addresses, "\n") { - addr = strings.TrimSpace(addr) + for _, addr := range params.ListenValues { + addr = teautils.FormatAddress(addr) if len(addr) == 0 { continue } diff --git a/teaweb/actions/default/settings/server/security.go b/teaweb/actions/default/settings/server/security.go index bc5bc62..c138c36 100644 --- a/teaweb/actions/default/settings/server/security.go +++ b/teaweb/actions/default/settings/server/security.go @@ -26,8 +26,8 @@ func (this *SecurityAction) Run(params struct{}) { } func (this *SecurityAction) RunPost(params struct { - AllowIPs []string - DenyIPs []string + AllowIPValues []string + DenyIPValues []string AllowAll bool DirAutoComplete bool LoginURL string @@ -42,7 +42,7 @@ func (this *SecurityAction) RunPost(params struct { admin.Security.Allow = []string{"all"} } else { ips := []string{} - for _, ip := range params.AllowIPs { + for _, ip := range params.AllowIPValues { if len(ip)> 0 { ips = append(ips, ip) } @@ -57,7 +57,7 @@ func (this *SecurityAction) RunPost(params struct { { ips := []string{} - for _, ip := range params.DenyIPs { + for _, ip := range params.DenyIPValues { if len(ip)> 0 { ips = append(ips, ip) } diff --git a/teaweb/utils/log_writer.go b/teaweb/utils/log_writer.go index 4d61e24..9cc89ca 100644 --- a/teaweb/utils/log_writer.go +++ b/teaweb/utils/log_writer.go @@ -16,13 +16,19 @@ func (this *LogWriter) Init() { // 创建目录 dir := files.NewFile(Tea.LogDir()) if !dir.Exists() { - dir.Mkdir() + err := dir.Mkdir() + if err != nil { + log.Println("[error]" + err.Error()) + } } // 先删除原来的 logFile := files.NewFile(Tea.LogFile("teaweb.log")) if logFile.Exists() { - logFile.Delete() + err := logFile.Delete() + if err != nil { + log.Println("[error]" + err.Error()) + } } // 打开要写入的日志文件 @@ -38,12 +44,15 @@ func (this *LogWriter) Write(message string) { log.Println(message) if this.fileAppender != nil { - this.fileAppender.AppendString(timeutil.Format("Y/m/d H:i:s ") + message + "\n") + _, err := this.fileAppender.AppendString(timeutil.Format("Y/m/d H:i:s ") + message + "\n") + if err != nil { + log.Println("[error]" + err.Error()) + } } } func (this *LogWriter) Close() { if this.fileAppender != nil { - this.fileAppender.Close() + _ = this.fileAppender.Close() } } From 0ca1a44113cdc63405e90d3e27b7a1d2a2e7728a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月24日 17:04:44 +0800 Subject: [PATCH 013/121] =?UTF-8?q?[proxy]=E8=AE=BF=E9=97=AE=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E9=BB=91=E7=99=BD=E5=90=8D=E5=8D=95=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?IP=E8=8C=83=E5=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/ip_range_list.go | 8 +++-- teaconfigs/request_group.go | 16 +++++----- teaconfigs/request_group_test.go | 20 ++++++------ teaconfigs/shared/client.go | 29 ++++++----------- teaconfigs/shared/client_test.go | 8 ++--- teaconfigs/{ => shared}/ip_range.go | 21 +++++++++--- teaconfigs/{ => shared}/ip_range_test.go | 32 ++++++++++++++++++- teaweb/actions/default/proxy/groups/add.go | 8 ++--- teaweb/actions/default/proxy/groups/update.go | 10 +++--- .../default/proxy/locations/access/update.go | 14 ++++---- teaweb/configs/admin_security.go | 14 ++++---- 11 files changed, 107 insertions(+), 73 deletions(-) rename teaconfigs/{ => shared}/ip_range.go (83%) rename teaconfigs/{ => shared}/ip_range_test.go (77%) diff --git a/teaconfigs/ip_range_list.go b/teaconfigs/ip_range_list.go index c7ee82d..46953d3 100644 --- a/teaconfigs/ip_range_list.go +++ b/teaconfigs/ip_range_list.go @@ -1,8 +1,10 @@ package teaconfigs +import "github.com/TeaWeb/code/teaconfigs/shared" + // IP Range列表 type IPRangeList struct { - IPRanges []*IPRangeConfig + IPRanges []*shared.IPRangeConfig } // 校验 @@ -17,13 +19,13 @@ func (this *IPRangeList) Validate() error { } // 添加 -func (this *IPRangeList) AddIPRange(ipRange *IPRangeConfig) { +func (this *IPRangeList) AddIPRange(ipRange *shared.IPRangeConfig) { this.IPRanges = append(this.IPRanges, ipRange) } // 删除 func (this *IPRangeList) RemoveIPRange(ipRangeId string) { - result := []*IPRangeConfig{} + result := []*shared.IPRangeConfig{} for _, r := range this.IPRanges { if r.Id == ipRangeId { continue diff --git a/teaconfigs/request_group.go b/teaconfigs/request_group.go index c47fa4c..44c4133 100644 --- a/teaconfigs/request_group.go +++ b/teaconfigs/request_group.go @@ -9,13 +9,13 @@ import ( type RequestGroup struct { BackendList `yaml:",inline"` - Id string `yaml:"id" json:"id"` // ID - Name string `yaml:"name" json:"name"` // 名称 - Cond []*shared.RequestCond `yaml:"conds" json:"conds"` // 匹配条件 - IPRanges []*IPRangeConfig `yaml:"ipRanges" json:"ipRanges"` // IP范围 - IsDefault bool `yaml:"isDefault" json:"isDefault"` // 是否为默认分组 - RequestHeaders []*shared.HeaderConfig `yaml:"requestHeaders" json:"requestHeaders"` // 请求Header - ResponseHeaders []*shared.HeaderConfig `yaml:"responseHeaders" json:"responseHeaders"` // 响应Header + Id string `yaml:"id" json:"id"` // ID + Name string `yaml:"name" json:"name"` // 名称 + Cond []*shared.RequestCond `yaml:"conds" json:"conds"` // 匹配条件 + IPRanges []*shared.IPRangeConfig `yaml:"ipRanges" json:"ipRanges"` // IP范围 + IsDefault bool `yaml:"isDefault" json:"isDefault"` // 是否为默认分组 + RequestHeaders []*shared.HeaderConfig `yaml:"requestHeaders" json:"requestHeaders"` // 请求Header + ResponseHeaders []*shared.HeaderConfig `yaml:"responseHeaders" json:"responseHeaders"` // 响应Header hasConds bool hasIPRanges bool @@ -80,7 +80,7 @@ func (this *RequestGroup) AddCond(cond *shared.RequestCond) { } // 添加IP范围 -func (this *RequestGroup) AddIPRange(ipRange *IPRangeConfig) { +func (this *RequestGroup) AddIPRange(ipRange *shared.IPRangeConfig) { this.IPRanges = append(this.IPRanges, ipRange) } diff --git a/teaconfigs/request_group_test.go b/teaconfigs/request_group_test.go index da84e89..a3b8bc4 100644 --- a/teaconfigs/request_group_test.go +++ b/teaconfigs/request_group_test.go @@ -28,8 +28,8 @@ func TestRequestGroup_Match(t *testing.T) { { group := NewRequestGroup() { - ipRange := NewIPRangeConfig() - ipRange.Type = IPRangeTypeRange + ipRange := shared.NewIPRangeConfig() + ipRange.Type = shared.IPRangeTypeRange ipRange.Param = "${remoteAddr}" ipRange.IPFrom = "192.168.1.1" ipRange.IPTo = "192.168.1.200" @@ -45,8 +45,8 @@ func TestRequestGroup_Match(t *testing.T) { { group := NewRequestGroup() { - ipRange := NewIPRangeConfig() - ipRange.Type = IPRangeTypeRange + ipRange := shared.NewIPRangeConfig() + ipRange.Type = shared.IPRangeTypeRange ipRange.Param = "${remoteAddr}" ipRange.IPFrom = "192.168.1.1" ipRange.IPTo = "192.168.1.100" @@ -62,8 +62,8 @@ func TestRequestGroup_Match(t *testing.T) { { group := NewRequestGroup() { - ipRange := NewIPRangeConfig() - ipRange.Type = IPRangeTypeRange + ipRange := shared.NewIPRangeConfig() + ipRange.Type = shared.IPRangeTypeRange ipRange.Param = "${remoteAddr}" ipRange.IPFrom = "192.168.1.1" ipRange.IPTo = "192.168.1.99" @@ -79,8 +79,8 @@ func TestRequestGroup_Match(t *testing.T) { { group := NewRequestGroup() { - ipRange := NewIPRangeConfig() - ipRange.Type = IPRangeTypeCIDR + ipRange := shared.NewIPRangeConfig() + ipRange.Type = shared.IPRangeTypeCIDR ipRange.Param = "${remoteAddr}" ipRange.CIDR = "192.168.1.1/24" group.AddIPRange(ipRange) @@ -102,8 +102,8 @@ func TestRequestGroup_Match(t *testing.T) { group.AddCond(cond) } { - ipRange := NewIPRangeConfig() - ipRange.Type = IPRangeTypeCIDR + ipRange := shared.NewIPRangeConfig() + ipRange.Type = shared.IPRangeTypeCIDR ipRange.Param = "${remoteAddr}" ipRange.CIDR = "192.168.1.1/24" group.AddIPRange(ipRange) diff --git a/teaconfigs/shared/client.go b/teaconfigs/shared/client.go index 8c830af..f08d301 100644 --- a/teaconfigs/shared/client.go +++ b/teaconfigs/shared/client.go @@ -2,7 +2,6 @@ package shared import ( "github.com/iwind/TeaGo/utils/string" - "strings" ) // 客户端配置 @@ -12,8 +11,7 @@ type ClientConfig struct { IP string `yaml:"ip" json:"ip"` // IP Description string `yaml:"description" json:"description"` // 描述 - hasWildcard bool - pieces []string + ipRange *IPRangeConfig } // 取得新配置对象 @@ -26,29 +24,20 @@ func NewClientConfig() *ClientConfig { // 校验 func (this *ClientConfig) Validate() error { - this.hasWildcard = strings.Contains(this.IP, "*") - if this.hasWildcard && len(this.IP)> 0 { - this.pieces = strings.Split(this.IP, ".") + if len(this.IP)> 0 { + ipRange, err := ParseIPRange(this.IP) + if err != nil { + return err + } + this.ipRange = ipRange } return nil } // 判断是否匹配某个IP func (this *ClientConfig) Match(ip string) bool { - if len(ip) == 0 { + if len(ip) == 0 || this.ipRange == nil { return false } - if this.hasWildcard { - pieces2 := strings.Split(ip, ".") - if len(pieces2) != len(this.pieces) { - return false - } - for index, piece2 := range pieces2 { - if this.pieces[index] != "*" && this.pieces[index] != piece2 { - return false - } - } - return true - } - return this.IP == ip + return this.ipRange.Contains(ip) } diff --git a/teaconfigs/shared/client_test.go b/teaconfigs/shared/client_test.go index ebe8b30..e4be8d6 100644 --- a/teaconfigs/shared/client_test.go +++ b/teaconfigs/shared/client_test.go @@ -9,22 +9,22 @@ func TestClientConfig_Allow(t *testing.T) { a := assert.NewAssertion(t) client := NewClientConfig() - client.Validate() + a.IsNil(client.Validate()) a.IsFalse(client.Match("127.0.0.1")) client.IP = "192.168.1.100" - client.Validate() + a.IsNil(client.Validate()) a.IsTrue(client.Match("192.168.1.100")) a.IsFalse(client.Match("192.168.1.101")) client.IP = "192.168.1.*" - client.Validate() + a.IsNil(client.Validate()) a.IsTrue(client.Match("192.168.1.100")) a.IsTrue(client.Match("192.168.1.101")) a.IsFalse(client.Match("192.168.2.100")) client.IP = "192.168.*.*" - client.Validate() + a.IsNil(client.Validate()) a.IsTrue(client.Match("192.168.1.100")) a.IsTrue(client.Match("192.168.1.101")) a.IsTrue(client.Match("192.168.2.100")) diff --git a/teaconfigs/ip_range.go b/teaconfigs/shared/ip_range.go similarity index 83% rename from teaconfigs/ip_range.go rename to teaconfigs/shared/ip_range.go index 7f107ac..53c2623 100644 --- a/teaconfigs/ip_range.go +++ b/teaconfigs/shared/ip_range.go @@ -1,10 +1,11 @@ -package teaconfigs +package shared import ( "bytes" "errors" "github.com/iwind/TeaGo/utils/string" "net" + "regexp" "strings" ) @@ -12,9 +13,10 @@ import ( type IPRangeType = int const ( - IPRangeTypeRange IPRangeType = 1 - IPRangeTypeCIDR IPRangeType = 2 - IPRangeTypeAll IPRangeType = 3 + IPRangeTypeRange IPRangeType = 1 + IPRangeTypeCIDR IPRangeType = 2 + IPRangeTypeAll IPRangeType = 3 + IPRangeTypeWildcard IPRangeType = 4 // 通配符,可以使用* ) // IP Range @@ -31,6 +33,7 @@ type IPRangeConfig struct { cidr *net.IPNet ipFrom net.IP ipTo net.IP + reg *regexp.Regexp } // 获取新对象 @@ -66,6 +69,10 @@ func ParseIPRange(s string) (*IPRangeConfig, error) { pieces := strings.SplitN(s, ",", 2) ipRange.IPFrom = strings.TrimSpace(pieces[0]) ipRange.IPTo = strings.TrimSpace(pieces[1]) + } else if strings.Contains(s, "*") { + ipRange.Type = IPRangeTypeWildcard + s = "^" + strings.Replace(regexp.QuoteMeta(s), `\*`, `\d+`, -1) + "$" + ipRange.reg = regexp.MustCompile(s) } else { ipRange.Type = IPRangeTypeRange ipRange.IPFrom = s @@ -127,6 +134,12 @@ func (this *IPRangeConfig) Contains(ipString string) bool { } return bytes.Compare(ip, this.ipFrom)>= 0 && bytes.Compare(ip, this.ipTo) <= 0 } + if this.Type == IPRangeTypeWildcard { + if this.reg == nil { + return false + } + return this.reg.MatchString(ipString) + } if this.Type == IPRangeTypeAll { return true } diff --git a/teaconfigs/ip_range_test.go b/teaconfigs/shared/ip_range_test.go similarity index 77% rename from teaconfigs/ip_range_test.go rename to teaconfigs/shared/ip_range_test.go index 383e519..1046979 100644 --- a/teaconfigs/ip_range_test.go +++ b/teaconfigs/shared/ip_range_test.go @@ -1,4 +1,4 @@ -package teaconfigs +package shared import ( "github.com/iwind/TeaGo/assert" @@ -92,4 +92,34 @@ func TestParseIPRange(t *testing.T) { a.IsTrue(r.Contains("192.168.1.150")) a.IsTrue(r.Contains("192.168.2.100")) } + + { + r, err := ParseIPRange("192.168.1.*") + a.IsNil(err) + if r != nil { + a.IsTrue(r.Type == IPRangeTypeWildcard) + a.IsTrue(r.Contains("192.168.1.100")) + a.IsFalse(r.Contains("192.168.2.100")) + } + } + + { + r, err := ParseIPRange("192.168.*.*") + a.IsNil(err) + if r != nil { + a.IsTrue(r.Type == IPRangeTypeWildcard) + a.IsTrue(r.Contains("192.168.1.100")) + a.IsTrue(r.Contains("192.168.2.100")) + } + } +} + +func BenchmarkIPRangeConfig_Contains(b *testing.B) { + r, err := ParseIPRange("192.168.1.*") + if err != nil { + b.Fatal(err) + } + for i := 0; i < b.N; i++ { + _ = r.Contains("192.168.1.100") + } } diff --git a/teaweb/actions/default/proxy/groups/add.go b/teaweb/actions/default/proxy/groups/add.go index d808070..48960d1 100644 --- a/teaweb/actions/default/proxy/groups/add.go +++ b/teaweb/actions/default/proxy/groups/add.go @@ -86,8 +86,8 @@ func (this *AddAction) RunPost(params struct { for index, ipRangeType := range params.IPRangeTypeList { if index < len(params.IPRangeFromList) && index < len(params.IPRangeToList) && index < len(params.IPRangeCIDRIPList) && index < len(params.IPRangeCIDRBitsList) && index < len(params.IPRangeVarList) { if ipRangeType == "range" { - config := teaconfigs.NewIPRangeConfig() - config.Type = teaconfigs.IPRangeTypeRange + config := shared.NewIPRangeConfig() + config.Type = shared.IPRangeTypeRange config.IPFrom = params.IPRangeFromList[index] config.IPTo = params.IPRangeToList[index] config.Param = params.IPRangeVarList[index] @@ -97,8 +97,8 @@ func (this *AddAction) RunPost(params struct { } group.AddIPRange(config) } else if ipRangeType == "cidr" { - config := teaconfigs.NewIPRangeConfig() - config.Type = teaconfigs.IPRangeTypeCIDR + config := shared.NewIPRangeConfig() + config.Type = shared.IPRangeTypeCIDR config.CIDR = params.IPRangeCIDRIPList[index] + "/" + params.IPRangeCIDRBitsList[index] config.Param = params.IPRangeVarList[index] err := config.Validate() diff --git a/teaweb/actions/default/proxy/groups/update.go b/teaweb/actions/default/proxy/groups/update.go index 2533fb4..95658ec 100644 --- a/teaweb/actions/default/proxy/groups/update.go +++ b/teaweb/actions/default/proxy/groups/update.go @@ -87,7 +87,7 @@ func (this *UpdateAction) RunPost(params struct { group.Name = params.Name group.Cond = []*shared.RequestCond{} - group.IPRanges = []*teaconfigs.IPRangeConfig{} + group.IPRanges = []*shared.IPRangeConfig{} group.RequestHeaders = []*shared.HeaderConfig{} group.ResponseHeaders = []*shared.HeaderConfig{} @@ -103,8 +103,8 @@ func (this *UpdateAction) RunPost(params struct { for index, ipRangeType := range params.IPRangeTypeList { if index < len(params.IPRangeFromList) && index < len(params.IPRangeToList) && index < len(params.IPRangeCIDRIPList) && index < len(params.IPRangeCIDRBitsList) { if ipRangeType == "range" { - config := teaconfigs.NewIPRangeConfig() - config.Type = teaconfigs.IPRangeTypeRange + config := shared.NewIPRangeConfig() + config.Type = shared.IPRangeTypeRange config.IPFrom = params.IPRangeFromList[index] config.IPTo = params.IPRangeToList[index] config.Param = params.IPRangeVarList[index] @@ -114,8 +114,8 @@ func (this *UpdateAction) RunPost(params struct { } group.AddIPRange(config) } else if ipRangeType == "cidr" { - config := teaconfigs.NewIPRangeConfig() - config.Type = teaconfigs.IPRangeTypeCIDR + config := shared.NewIPRangeConfig() + config.Type = shared.IPRangeTypeCIDR config.CIDR = params.IPRangeCIDRIPList[index] + "/" + params.IPRangeCIDRBitsList[index] config.Param = params.IPRangeVarList[index] err := config.Validate() diff --git a/teaweb/actions/default/proxy/locations/access/update.go b/teaweb/actions/default/proxy/locations/access/update.go index 7ce639e..19de9bd 100644 --- a/teaweb/actions/default/proxy/locations/access/update.go +++ b/teaweb/actions/default/proxy/locations/access/update.go @@ -55,12 +55,12 @@ func (this *UpdateAction) RunPost(params struct { TrafficMonthTotal int64 TrafficMonthDuration int64 - AccessOn bool - AccessAllowOn bool - AccessAllowIPs []string + AccessOn bool + AccessAllowOn bool + AccessAllowIPValues []string - AccessDenyOn bool - AccessDenyIPs []string + AccessDenyOn bool + AccessDenyIPValues []string Must *actions.Must }) { @@ -107,7 +107,7 @@ func (this *UpdateAction) RunPost(params struct { policy.Access.On = params.AccessOn policy.Access.AllowOn = params.AccessAllowOn policy.Access.Allow = []*shared.ClientConfig{} - for _, ip := range params.AccessAllowIPs { + for _, ip := range params.AccessAllowIPValues { if len(ip)> 0 { client := shared.NewClientConfig() client.IP = ip @@ -117,7 +117,7 @@ func (this *UpdateAction) RunPost(params struct { policy.Access.DenyOn = params.AccessDenyOn policy.Access.Deny = []*shared.ClientConfig{} - for _, ip := range params.AccessDenyIPs { + for _, ip := range params.AccessDenyIPValues { if len(ip)> 0 { client := shared.NewClientConfig() client.IP = ip diff --git a/teaweb/configs/admin_security.go b/teaweb/configs/admin_security.go index 5539ce5..3725ec5 100644 --- a/teaweb/configs/admin_security.go +++ b/teaweb/configs/admin_security.go @@ -1,7 +1,7 @@ package configs import ( - "github.com/TeaWeb/code/teaconfigs" + "github.com/TeaWeb/code/teaconfigs/shared" "github.com/TeaWeb/code/teaconst" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" @@ -20,8 +20,8 @@ type AdminSecurity struct { LoginURL string `yaml:"loginURL" json:"loginURL"` // 登录页面的URL PasswordEncryptType string `yaml:"passwordEncryptType" json:"passwordEncryptType"` // 密码加密方式 - allowIPRanges []*teaconfigs.IPRangeConfig - denyIPRanges []*teaconfigs.IPRangeConfig + allowIPRanges []*shared.IPRangeConfig + denyIPRanges []*shared.IPRangeConfig } // 获取新对象 @@ -55,9 +55,9 @@ func (this *AdminSecurity) Validate() error { this.PasswordEncryptType = "clear" } - this.allowIPRanges = []*teaconfigs.IPRangeConfig{} + this.allowIPRanges = []*shared.IPRangeConfig{} for _, s := range this.Allow { - r, err := teaconfigs.ParseIPRange(s) + r, err := shared.ParseIPRange(s) if err != nil { logs.Error(err) } else { @@ -65,9 +65,9 @@ func (this *AdminSecurity) Validate() error { } } - this.denyIPRanges = []*teaconfigs.IPRangeConfig{} + this.denyIPRanges = []*shared.IPRangeConfig{} for _, s := range this.Deny { - r, err := teaconfigs.ParseIPRange(s) + r, err := shared.ParseIPRange(s) if err != nil { logs.Error(err) } else { From 57a303fbfd902bf6629ce457285c0b60f96ac279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月26日 11:22:29 +0800 Subject: [PATCH 014/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8D=E5=BD=93?= =?UTF-8?q?=E5=89=8D=E8=BF=9E=E6=8E=A5=E6=95=B0=E6=98=BE=E7=A4=BA=E4=B8=8D?= =?UTF-8?q?=E5=87=86=E7=A1=AE=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 以前版本当在有连接时修改后端服务器设置时可能会出现此问题 --- teaconfigs/backend.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/teaconfigs/backend.go b/teaconfigs/backend.go index 572f718..2ea2967 100644 --- a/teaconfigs/backend.go +++ b/teaconfigs/backend.go @@ -19,6 +19,8 @@ import ( // 服务后端配置 type BackendConfig struct { + nextBackend *BackendConfig // 等待切换的下一个Backend + shared.HeaderList `yaml:",inline"` TeaVersion string `yaml:"teaVersion" json:"teaVersion"` @@ -218,6 +220,9 @@ func (this *BackendConfig) IncreaseConn() int32 { // 减少连接数,并返回减少之后的数字 func (this *BackendConfig) DecreaseConn() int32 { + if this.nextBackend != nil { + return this.nextBackend.DecreaseConn() + } return atomic.AddInt32(&this.CurrentConns, -1) } @@ -438,6 +443,7 @@ func (this *BackendConfig) CloneState(oldBackend *BackendConfig) { if oldBackend == nil { return } + oldBackend.nextBackend = this this.IsDown = oldBackend.IsDown this.DownTime = oldBackend.DownTime this.CurrentFails = oldBackend.CurrentFails From e32886104141231f7dee42c8eed6b2d814c6a3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月26日 12:08:48 +0800 Subject: [PATCH 015/121] =?UTF-8?q?[proxy]=E5=A2=9E=E5=BC=BA=E7=99=BB?= =?UTF-8?q?=E5=BD=95=E7=95=8C=E9=9D=A2=E5=AE=89=E5=85=A8=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/login/index.go | 27 +++++++++++++++++++++++++++ teaweb/configs/admin.go | 8 ++++---- teaweb/configs/admin_test.go | 8 ++++---- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/teaweb/actions/default/login/index.go b/teaweb/actions/default/login/index.go index 741172e..1a01bde 100644 --- a/teaweb/actions/default/login/index.go +++ b/teaweb/actions/default/login/index.go @@ -1,6 +1,7 @@ package login import ( + "fmt" "github.com/TeaWeb/code/teaconfigs/audits" "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teadb" @@ -9,12 +10,16 @@ import ( "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/types" + stringutil "github.com/iwind/TeaGo/utils/string" "net/http" "time" ) type IndexAction actions.Action +var TokenSalt = stringutil.Rand(32) + // 登录 func (this *IndexAction) RunGet() { // 检查IP限制 @@ -31,6 +36,9 @@ func (this *IndexAction) RunGet() { this.Data["teaDemoEnabled"] = teaconst.DemoEnabled + timestamp := fmt.Sprintf("%d", time.Now().Unix()) + this.Data["token"] = stringutil.Md5(TokenSalt+timestamp) + timestamp + this.Show() } @@ -38,6 +46,7 @@ func (this *IndexAction) RunGet() { func (this *IndexAction) RunPost(params struct { Username string Password string + Token string Remember bool Must *actions.Must Auth *helpers.UserShouldAuth @@ -70,6 +79,24 @@ func (this *IndexAction) RunPost(params struct { Field("password", params.Password). Require("请输入密码") + if params.Password == stringutil.Md5("") { + this.FailField("password", "请输入密码") + } + + // 检查token + if len(params.Token) <= 32 { + this.Fail("请通过登录页面登录") + } + timestampString := params.Token[32:] + if stringutil.Md5(TokenSalt+timestampString) != params.Token[:32] { + this.Fail("登录页面已过期,请刷新后重试") + } + timestamp := types.Int64(timestampString) + if timestamp < time.Now().Unix()-1800 { + this.Fail("登录页面已过期,请刷新后重试") + } + + // 查找用户 adminConfig := configs.SharedAdminConfig() user := adminConfig.FindActiveUser(params.Username) if user != nil { diff --git a/teaweb/configs/admin.go b/teaweb/configs/admin.go index fc618a1..be49f96 100644 --- a/teaweb/configs/admin.go +++ b/teaweb/configs/admin.go @@ -111,14 +111,14 @@ func (this *AdminConfig) EncryptPassword(password string) string { } // 对比密码 -func (this *AdminConfig) ComparePassword(rawPassword, encryptedPassword string) bool { +func (this *AdminConfig) ComparePassword(inputPassword, encryptedPassword string) bool { if strings.HasPrefix(encryptedPassword, "clear:") { - return "clear:"+rawPassword == encryptedPassword + return inputPassword == stringutil.Md5(encryptedPassword[len("clear:"):]) } if strings.HasPrefix(encryptedPassword, "md5:") { - return "md5:"+stringutil.Md5(rawPassword) == encryptedPassword + return inputPassword == encryptedPassword[len("md5:"):] } - return rawPassword == encryptedPassword + return inputPassword == stringutil.Md5(encryptedPassword) } // 是否包含某个激活的用户名 diff --git a/teaweb/configs/admin_test.go b/teaweb/configs/admin_test.go index 01ec4d5..bb76dbe 100644 --- a/teaweb/configs/admin_test.go +++ b/teaweb/configs/admin_test.go @@ -16,17 +16,17 @@ func TestAdminConfig_ComparePassword(t *testing.T) { { config := AdminConfig{} - a.IsTrue(config.ComparePassword("123456", "123456")) + a.IsTrue(config.ComparePassword(stringutil.Md5("123456"), "123456")) } { config := AdminConfig{} - a.IsTrue(config.ComparePassword("123456", "clear:123456")) + a.IsTrue(config.ComparePassword(stringutil.Md5("123456"), "clear:123456")) } { config := AdminConfig{} - a.IsTrue(config.ComparePassword("123456", "md5:"+stringutil.Md5("123456"))) - a.IsFalse(config.ComparePassword("123456789", "md5:"+stringutil.Md5("123456"))) + a.IsTrue(config.ComparePassword(stringutil.Md5("123456"), "md5:"+stringutil.Md5("123456"))) + a.IsFalse(config.ComparePassword(stringutil.Md5("123456789"), "md5:"+stringutil.Md5("123456"))) } } From 5bc4030e41be8a9708baa565354c4a797cd1cdf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月26日 16:20:23 +0800 Subject: [PATCH 016/121] =?UTF-8?q?=E7=95=8C=E9=9D=A2=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/login/index.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/teaweb/actions/default/login/index.go b/teaweb/actions/default/login/index.go index 1a01bde..90f71b0 100644 --- a/teaweb/actions/default/login/index.go +++ b/teaweb/actions/default/login/index.go @@ -89,11 +89,11 @@ func (this *IndexAction) RunPost(params struct { } timestampString := params.Token[32:] if stringutil.Md5(TokenSalt+timestampString) != params.Token[:32] { - this.Fail("登录页面已过期,请刷新后重试") + this.FailField("refresh", "登录页面已过期,请刷新后重试") } timestamp := types.Int64(timestampString) if timestamp < time.Now().Unix()-1800 { - this.Fail("登录页面已过期,请刷新后重试") + this.FailField("refresh", "登录页面已过期,请刷新后重试") } // 查找用户 From 169c15001d8e0662b88d2ab6918b17374eb6d665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月26日 16:21:01 +0800 Subject: [PATCH 017/121] =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/shared/access_policy.go | 14 +++++++------- teaconfigs/shared/access_policy_test.go | 12 ++++-------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/teaconfigs/shared/access_policy.go b/teaconfigs/shared/access_policy.go index ca0fa99..fb1ed60 100644 --- a/teaconfigs/shared/access_policy.go +++ b/teaconfigs/shared/access_policy.go @@ -17,7 +17,7 @@ type AccessPolicy struct { On bool `yaml:"on" json:"on"` // 是否开启 Total int64 `yaml:"total" json:"total"` // 总量 Used int64 `yaml:"used" json:"used"` // 已使用量 - } `yaml:"total" json:"total"` // 总量控制 + } `yaml:"total" json:"total"` // 总量控制 Second struct { On bool `yaml:"on" json:"on"` // 是否开启 Total int64 `yaml:"total" json:"total"` // 总量 @@ -201,13 +201,13 @@ func (this *AccessPolicy) IncreaseTraffic() { // total if this.Traffic.Total.On { - this.Traffic.Total.Used ++ + this.Traffic.Total.Used++ } // second if this.Traffic.Second.On && this.Traffic.Second.Duration> 0 { if timestamp-this.Traffic.Second.FromTime < this.Traffic.Second.Duration { - this.Traffic.Second.Used ++ + this.Traffic.Second.Used++ } else { this.Traffic.Second.FromTime = timestamp this.Traffic.Second.Used = 1 @@ -217,7 +217,7 @@ func (this *AccessPolicy) IncreaseTraffic() { // minute if this.Traffic.Minute.On && this.Traffic.Minute.Duration> 0 { if timestamp>= this.Traffic.Minute.FromTime && timestamp < this.Traffic.Minute.ToTime { - this.Traffic.Minute.Used ++ + this.Traffic.Minute.Used++ } else { this.Traffic.Minute.Used = 1 fromTime := time.Date(year, month, day, hour, minute, 0, 0, time.Local) @@ -229,7 +229,7 @@ func (this *AccessPolicy) IncreaseTraffic() { // hour if this.Traffic.Hour.On && this.Traffic.Hour.Duration> 0 { if timestamp>= this.Traffic.Hour.FromTime && timestamp < this.Traffic.Hour.ToTime { - this.Traffic.Hour.Used ++ + this.Traffic.Hour.Used++ } else { this.Traffic.Hour.Used = 1 fromTime := time.Date(year, month, day, hour, 0, 0, 0, time.Local) @@ -241,7 +241,7 @@ func (this *AccessPolicy) IncreaseTraffic() { // day if this.Traffic.Day.On && this.Traffic.Day.Duration> 0 { if timestamp>= this.Traffic.Day.FromTime && timestamp < this.Traffic.Day.ToTime { - this.Traffic.Day.Used ++ + this.Traffic.Day.Used++ } else { this.Traffic.Day.Used = 1 fromTime := time.Date(year, month, day, 0, 0, 0, 0, time.Local) @@ -253,7 +253,7 @@ func (this *AccessPolicy) IncreaseTraffic() { // month if this.Traffic.Month.On && this.Traffic.Month.Duration> 0 { if timestamp>= this.Traffic.Month.FromTime && timestamp < this.Traffic.Month.ToTime { - this.Traffic.Month.Used ++ + this.Traffic.Month.Used++ } else { this.Traffic.Month.Used = 1 fromTime := time.Date(year, month, 1, 0, 0, 0, 0, time.Local) diff --git a/teaconfigs/shared/access_policy_test.go b/teaconfigs/shared/access_policy_test.go index 4200cdf..9724ef1 100644 --- a/teaconfigs/shared/access_policy_test.go +++ b/teaconfigs/shared/access_policy_test.go @@ -135,15 +135,13 @@ func TestAPIAccessPolicyMinute(t *testing.T) { //a.IsTrue(p.AllowTraffic()) } -func TestAPIAccessPolicyPerformance(t *testing.T) { - times := 100000 - before := time.Now() +func BenchmarkAccessPolicyPerformance(b *testing.B) { + p := AccessPolicy{} - locker := sync.Mutex{} - for i := 0; i < times; i ++ { + for i := 0; i < b.N; i++ { + locker := sync.Mutex{} locker.Lock() - p := AccessPolicy{} p.Traffic.On = true p.Traffic.Second.On = true p.Traffic.Second.Duration = 1 @@ -165,8 +163,6 @@ func TestAPIAccessPolicyPerformance(t *testing.T) { locker.Unlock() } - - t.Log(int(float64(times) / time.Since(before).Seconds())) } func TestAccessPolicy_AllowAccess(t *testing.T) { From 8ea2ce99ba8c8310d182500fb53cbb1934bf5304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年10月28日 16:46:17 +0800 Subject: [PATCH 018/121] =?UTF-8?q?[search]=E5=85=A8=E5=B1=80=E6=90=9C?= =?UTF-8?q?=E7=B4=A2=E5=8F=AF=E4=BB=A5=E6=90=9C=E7=B4=A2=E7=BD=91=E7=BB=9C?= =?UTF-8?q?=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/server.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/teaconfigs/server.go b/teaconfigs/server.go index 95252c4..da19a0b 100644 --- a/teaconfigs/server.go +++ b/teaconfigs/server.go @@ -1093,6 +1093,27 @@ func (this *ServerConfig) MatchKeyword(keyword string) (matched bool, name strin } } + // 地址 + for _, addr := range this.Listen { + if strings.Index(addr, keyword)> -1 { + matched = true + name = this.Description + tags = []string{addr} + return + } + } + + if this.SSL != nil { + for _, addr := range this.SSL.Listen { + if strings.Index(addr, keyword)> -1 { + matched = true + name = this.Description + tags = []string{addr} + return + } + } + } + return } From 45f4f34c554af12d0ca54976b4e32d285736a260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Fri, 1 Nov 2019 11:02:17 +0800 Subject: [PATCH 019/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8D=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E5=9F=9F=E5=90=8D=E7=9A=84=E6=AD=A3=E5=88=99=E5=8F=AF?= =?UTF-8?q?=E8=83=BD=E4=BA=A7=E7=94=9F=E7=9A=84nil=20panic=EF=BC=8C?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E6=94=AF=E6=8C=81=E5=8C=B9=E9=85=8D=E5=90=8E?= =?UTF-8?q?=E7=BC=80=E5=9F=9F=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teautils/domain.go | 9 +++++++-- teautils/domain_test.go | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/teautils/domain.go b/teautils/domain.go index a03a3dc..1205a52 100644 --- a/teautils/domain.go +++ b/teautils/domain.go @@ -28,14 +28,19 @@ func matchDomain(pattern string, domain string) (isMatched bool) { } // 正则表达式 - if pattern[0:1] == "~" { - reg, err := stringutil.RegexpCompile(pattern[1:]) + if pattern[0] == '~' { + reg, err := stringutil.RegexpCompile(strings.TrimSpace(pattern[1:])) if err != nil { logs.Error(err) + return false } return reg.MatchString(domain) } + if pattern[0] == '.' { + return strings.HasSuffix(domain, pattern) + } + // 其他匹配 patternPieces := strings.Split(pattern, ".") domainPieces := strings.Split(domain, ".") diff --git a/teautils/domain_test.go b/teautils/domain_test.go index ba96fab..457d5ce 100644 --- a/teautils/domain_test.go +++ b/teautils/domain_test.go @@ -27,6 +27,16 @@ func TestMatchDomain(t *testing.T) { a.IsTrue(ok) } + { + ok := MatchDomains([]string{".example.com"}, "a.www.example.com") + a.IsTrue(ok) + } + + { + ok := MatchDomains([]string{".example.com"}, "a.www.example123.com") + a.IsFalse(ok) + } + { ok := MatchDomains([]string{"*.example.com"}, "www.example.com") a.IsTrue(ok) @@ -52,6 +62,11 @@ func TestMatchDomain(t *testing.T) { a.IsTrue(ok) } + { + ok := MatchDomains([]string{"~\\w+.example.com"}, "a.www.example.com") + a.IsTrue(ok) + } + { ok := MatchDomains([]string{"~^\\d+.example.com$"}, "www.example.com") a.IsFalse(ok) From 4f4c7c57ddd39f608aa75b39d332210462255295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月11日 12:50:07 +0800 Subject: [PATCH 020/121] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9C=A8Windows?= =?UTF-8?q?=E4=B8=8B=E4=B8=8D=E8=83=BD=E6=AD=A3=E7=A1=AE=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E6=98=AF=E5=90=A6=E6=AD=A3=E5=9C=A8=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C=E5=B0=86"TeaWeb"?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E4=B8=BA=E5=B8=B8=E9=87=8F=EF=BC=8C=E6=96=B9?= =?UTF-8?q?=E4=BE=BF=E7=AC=AC=E4=B8=89=E6=96=B9=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/service_install.go | 5 ++- main/service_uninstall.go | 5 ++- teaconst/const.go | 3 ++ teautils/service_linux.go | 5 ++- teautils/service_test.go | 3 +- teaweb/cmd/web_shell.go | 74 ++++++++++++++++++++++---------- teaweb/cmd/web_shell_windows.go | 3 +- teaweb/helpers/user_must_auth.go | 8 ++-- teaweb/utils/log_writer.go | 3 +- 9 files changed, 73 insertions(+), 36 deletions(-) diff --git a/main/service_install.go b/main/service_install.go index d352d47..eb97bea 100644 --- a/main/service_install.go +++ b/main/service_install.go @@ -1,6 +1,7 @@ package main import ( + "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" "github.com/iwind/TeaGo/Tea" "log" @@ -10,9 +11,9 @@ import ( // 安装服务 func main() { log.Println("installing ...") - manager := teautils.NewServiceManager("TeaWeb", "TeaWeb Server") + manager := teautils.NewServiceManager(teaconst.TeaProductName, teaconst.TeaProductName+" Server") - var exePath = Tea.Root + Tea.DS + "bin" + Tea.DS + "teaweb" + var exePath = Tea.Root + Tea.DS + "bin" + Tea.DS + teaconst.TeaProcessName if runtime.GOOS == "windows" { exePath += ".exe" } diff --git a/main/service_uninstall.go b/main/service_uninstall.go index 743420a..a6eb844 100644 --- a/main/service_uninstall.go +++ b/main/service_uninstall.go @@ -1,6 +1,7 @@ package main import ( + "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" "log" ) @@ -8,7 +9,7 @@ import ( // 卸载服务 func main() { log.Println("uninstalling ...") - manager := teautils.NewServiceManager("TeaWeb", "TeaWeb Server") + manager := teautils.NewServiceManager(teaconst.TeaProductName, teaconst.TeaProductName+" Server") err := manager.Uninstall() if err != nil { log.Println("ERROR: " + err.Error()) @@ -19,6 +20,6 @@ func main() { log.Println("uninstalled service successfully") log.Println("done.") - + manager.PauseWindow() } diff --git a/teaconst/const.go b/teaconst/const.go index 3feab11..2ebf31f 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -2,4 +2,7 @@ package teaconst const ( TeaVersion = "0.1.8.1" + + TeaProcessName = "teaweb" // 进程名 + TeaProductName = "TeaWeb" // 产品名 ) diff --git a/teautils/service_linux.go b/teautils/service_linux.go index 2816b01..738dbcf 100644 --- a/teautils/service_linux.go +++ b/teautils/service_linux.go @@ -4,6 +4,7 @@ package teautils import ( "errors" + "github.com/TeaWeb/code/teaconst" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" "io/ioutil" @@ -43,7 +44,7 @@ func (this *ServiceManager) Start() error { return exec.Command(systemd, "start", "teaweb.service").Start() } - return exec.Command("service", "teaweb", "start").Start() + return exec.Command("service", teaconst.TeaProcessName, "start").Start() } // 删除服务 @@ -97,7 +98,7 @@ func (this *ServiceManager) installInitService(exePath string, args []string) er return err } - err = exec.Command(chkCmd, "--add", "teaweb").Start() + err = exec.Command(chkCmd, "--add", teaconst.TeaProcessName).Start() if err != nil { return err } diff --git a/teautils/service_test.go b/teautils/service_test.go index 6f7bcd1..03c36fe 100644 --- a/teautils/service_test.go +++ b/teautils/service_test.go @@ -1,11 +1,12 @@ package teautils import ( + "github.com/TeaWeb/code/teaconst" "testing" ) func TestServiceManager_Log(t *testing.T) { - manager := NewServiceManager("TeaWeb", "TeaWeb Server") + manager := NewServiceManager(teaconst.TeaProductName, teaconst.TeaProductName+" Server") manager.Log("Hello, World") manager.LogError("Hello, World") } diff --git a/teaweb/cmd/web_shell.go b/teaweb/cmd/web_shell.go index 7adef8a..22909c2 100644 --- a/teaweb/cmd/web_shell.go +++ b/teaweb/cmd/web_shell.go @@ -28,6 +28,7 @@ import ( ) var sharedShell *WebShell = nil +var pidFP *os.File // hold pid file pointer on windows // 命令行相关封装 type WebShell struct { @@ -51,11 +52,11 @@ func (this *WebShell) Start(server *TeaGo.Server) { return } - // 当前PID - err := files.NewFile(Tea.Root + Tea.DS + "bin" + Tea.DS + "pid"). - WriteString(fmt.Sprintf("%d", os.Getpid())) + // 当前PID文件句柄 + err := this.writePid() if err != nil { - logs.Error(err) + logs.Println("[error]write pid file failed: '" + err.Error() + "'") + return } // 信号 @@ -139,7 +140,7 @@ func (this *WebShell) execArgs(writer io.Writer) bool { // 检查是否已经启动 proc := this.checkPid() if proc != nil { - this.write(writer, "TeaWeb is already running, pid:", proc.Pid) + this.write(writer, teaconst.TeaProductName+" is already running, pid:", proc.Pid) return true } return false @@ -174,7 +175,7 @@ func (this *WebShell) execArgs(writer io.Writer) bool { } if len(args)> 0 { - this.write(writer, "Unknown command option '"+strings.Join(args, " ")+"', run './bin/teaweb -h' to lookup the usage.") + this.write(writer, "Unknown command option '"+strings.Join(args, " ")+"', run './bin/"+teaconst.TeaProcessName+" -h' to lookup the usage.") return true } return false @@ -182,8 +183,8 @@ func (this *WebShell) execArgs(writer io.Writer) bool { // 帮助 func (this *WebShell) ExecHelp(writer io.Writer) bool { - this.write(writer, "TeaWeb v"+teaconst.TeaVersion) - this.write(writer, "Usage:", "\n ./bin/teaweb [option]") + this.write(writer, teaconst.TeaProductName+" v"+teaconst.TeaVersion) + this.write(writer, "Usage:", "\n ./bin/"+teaconst.TeaProcessName+" [option]") this.write(writer, "") this.write(writer, "Options:") this.write(writer, fmt.Sprintf(" %-20s%s", "-h", ": print this help")) @@ -197,14 +198,14 @@ func (this *WebShell) ExecHelp(writer io.Writer) bool { this.write(writer, fmt.Sprintf(" %-20s%s", "sync", ": sync config files with cluster")) this.write(writer, fmt.Sprintf(" %-20s%s", "pprof [address]", ": start pprof server")) this.write(writer, "") - this.write(writer, "To run the server in foreground:", "\n ./bin/teaweb\n") + this.write(writer, "To run the server in foreground:", "\n ./bin/"+teaconst.TeaProcessName+"\n") return true } // 版本号 func (this *WebShell) ExecVersion(writer io.Writer) bool { - this.write(writer, "TeaWeb v"+teaconst.TeaVersion, "(build: "+runtime.Version(), runtime.GOOS, runtime.GOARCH+")") + this.write(writer, teaconst.TeaProductName+" v"+teaconst.TeaVersion, "(build: "+runtime.Version(), runtime.GOOS, runtime.GOARCH+")") return true } @@ -212,17 +213,17 @@ func (this *WebShell) ExecVersion(writer io.Writer) bool { func (this *WebShell) ExecStart(writer io.Writer) bool { proc := this.checkPid() if proc != nil { - this.write(writer, "TeaWeb already started, pid:", proc.Pid) + this.write(writer, teaconst.TeaProductName+" already started, pid:", proc.Pid) return true } cmd := exec.Command(os.Args[0]) err := cmd.Start() if err != nil { - this.write(writer, "TeaWeb start failed:", err.Error()) + this.write(writer, teaconst.TeaProductName+" start failed:", err.Error()) return true } - this.write(writer, "TeaWeb started ok, pid:", cmd.Process.Pid) + this.write(writer, teaconst.TeaProductName+" started ok, pid:", cmd.Process.Pid) return true } @@ -231,18 +232,18 @@ func (this *WebShell) ExecStart(writer io.Writer) bool { func (this *WebShell) ExecStop(writer io.Writer) bool { proc := this.checkPid() if proc == nil { - this.write(writer, "TeaWeb not started") + this.write(writer, teaconst.TeaProductName+" not started") return true } err := proc.Kill() if err != nil { - this.write(writer, "TeaWeb stop error:", err.Error()) + this.write(writer, teaconst.TeaProductName+" stop error:", err.Error()) return true } err = files.NewFile(Tea.Root + "/bin/pid").Delete() - this.write(writer, "TeaWeb stopped ok, pid:", proc.Pid) + this.write(writer, teaconst.TeaProductName+" stopped ok, pid:", proc.Pid) if err != nil { this.write(writer, "ERROR:", err.Error()) @@ -284,7 +285,7 @@ func (this *WebShell) ExecRestart(writer io.Writer) bool { if proc != nil { err := proc.Kill() if err != nil { - this.write(writer, "TeaWeb stop error:", err.Error()) + this.write(writer, teaconst.TeaProductName+" stop error:", err.Error()) return true } @@ -295,10 +296,10 @@ func (this *WebShell) ExecRestart(writer io.Writer) bool { cmd := exec.Command(os.Args[0]) err := cmd.Start() if err != nil { - this.write(writer, "TeaWeb restart failed:", err.Error()) + this.write(writer, teaconst.TeaProductName+" restart failed:", err.Error()) return true } - this.write(writer, "TeaWeb restarted ok, pid:", cmd.Process.Pid) + this.write(writer, teaconst.TeaProductName+" restarted ok, pid:", cmd.Process.Pid) return true } @@ -334,9 +335,9 @@ func (this *WebShell) ExecReset(writer io.Writer) bool { func (this *WebShell) ExecStatus(writer io.Writer) bool { proc := this.checkPid() if proc == nil { - this.write(writer, "TeaWeb not started yet") + this.write(writer, teaconst.TeaProductName+" not started yet") } else { - this.write(writer, "TeaWeb is running, pid:"+fmt.Sprintf("%d", proc.Pid)) + this.write(writer, teaconst.TeaProductName+" is running, pid:"+fmt.Sprintf("%d", proc.Pid)) } return true } @@ -345,7 +346,7 @@ func (this *WebShell) ExecStatus(writer io.Writer) bool { func (this *WebShell) ExecSync(writer io.Writer) bool { proc := this.checkPid() if proc == nil { - this.write(writer, "TeaWeb not started yet") + this.write(writer, teaconst.TeaProductName+" not started yet") } else { err := proc.Signal(syscall.Signal(0x1f /**syscall.SIGUSR2**/)) if err != nil { @@ -374,6 +375,29 @@ func (this *WebShell) ExecPprof(writer io.Writer) bool { return false } +// 写入PID +func (this *WebShell) writePid() error { + fp, err := os.OpenFile(Tea.Root+Tea.DS+"bin"+Tea.DS+"pid", os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_RDONLY, 0666) + if err != nil { + return err + } + + if runtime.GOOS == "windows" { + pidFP = fp // hold the fp to lock file + } else { + defer func() { + _ = fp.Close() + }() + } + + _, err = fp.WriteString(fmt.Sprintf("%d", os.Getpid())) + if err != nil { + return err + } + + return nil +} + // 检查PID func (this *WebShell) checkPid() *os.Process { // check pid file @@ -402,6 +426,10 @@ func (this *WebShell) checkPid() *os.Process { } if runtime.GOOS == "windows" { + // windows上打开的文件是不能删除的 + if pidFile.Delete() == nil { + return nil + } return proc } @@ -435,7 +463,7 @@ func (this *WebShell) checkPid() *os.Process { if index2> 0 { outputString = outputString[index2+1:] } - if strings.Contains(outputString, "teaweb") && !strings.Contains(outputString, "teaweb-") { + if strings.Contains(outputString, teaconst.TeaProcessName) && !strings.Contains(outputString, teaconst.TeaProcessName+"-") { return proc } diff --git a/teaweb/cmd/web_shell_windows.go b/teaweb/cmd/web_shell_windows.go index c7a493e..579f66c 100644 --- a/teaweb/cmd/web_shell_windows.go +++ b/teaweb/cmd/web_shell_windows.go @@ -3,6 +3,7 @@ package cmd import ( + "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" "io" ) @@ -10,7 +11,7 @@ import ( // 启动服务模式 func (this *WebShell) ExecService(writer io.Writer) bool { // start the manager - manager := teautils.NewServiceManager("TeaWeb", "TeaWeb Server") + manager := teautils.NewServiceManager(teaconst.TeaProductName, teaconst.TeaProductName+" Server") manager.Run() return true diff --git a/teaweb/helpers/user_must_auth.go b/teaweb/helpers/user_must_auth.go index 76d8a4d..8a5ddda 100644 --- a/teaweb/helpers/user_must_auth.go +++ b/teaweb/helpers/user_must_auth.go @@ -123,11 +123,11 @@ func (this *UserMustAuth) BeforeAction(actionPtr actions.ActionWrapper, paramNam } if teaconst.PlusEnabled { - action.Data["teaTitle"] = "TeaWeb+管理平台" - action.Data["teaName"] = "TeaWeb+" + action.Data["teaTitle"] = teaconst.TeaProductName + "+管理平台" + action.Data["teaName"] = teaconst.TeaProductName + "+" } else { - action.Data["teaTitle"] = "TeaWeb管理平台" - action.Data["teaName"] = "TeaWeb" + action.Data["teaTitle"] = teaconst.TeaProductName + "管理平台" + action.Data["teaName"] = teaconst.TeaProductName } if len(user.Name) == 0 { diff --git a/teaweb/utils/log_writer.go b/teaweb/utils/log_writer.go index 9cc89ca..236fb8e 100644 --- a/teaweb/utils/log_writer.go +++ b/teaweb/utils/log_writer.go @@ -1,6 +1,7 @@ package utils import ( + "github.com/TeaWeb/code/teaconst" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" "github.com/iwind/TeaGo/logs" @@ -23,7 +24,7 @@ func (this *LogWriter) Init() { } // 先删除原来的 - logFile := files.NewFile(Tea.LogFile("teaweb.log")) + logFile := files.NewFile(Tea.LogFile(teaconst.TeaProcessName + ".log")) if logFile.Exists() { err := logFile.Delete() if err != nil { From acf78df9cd542cc1318e787c6494ad4d1161c0f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月11日 16:44:14 +0800 Subject: [PATCH 021/121] =?UTF-8?q?=E4=BC=98=E5=8C=96teaweb=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teautils/file_others.go | 17 +++++++ teautils/file_windows.go | 17 +++++++ teautils/process.go | 73 +++++++++++++++++++++++++++ teautils/process_test.go | 10 ++++ teaweb/cmd/web_shell.go | 92 ++-------------------------------- teaweb/cmd/web_shell_others.go | 4 +- 6 files changed, 124 insertions(+), 89 deletions(-) create mode 100644 teautils/file_others.go create mode 100644 teautils/file_windows.go create mode 100644 teautils/process.go create mode 100644 teautils/process_test.go diff --git a/teautils/file_others.go b/teautils/file_others.go new file mode 100644 index 0000000..62f4429 --- /dev/null +++ b/teautils/file_others.go @@ -0,0 +1,17 @@ +// +build !windows + +package teautils + +import ( + "os" + "syscall" +) + +// lock file +func LockFile(fp *os.File) error { + return syscall.Flock(int(fp.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) +} + +func UnlockFile(fp *os.File) error { + return syscall.Flock(int(fp.Fd()), syscall.LOCK_UN) +} diff --git a/teautils/file_windows.go b/teautils/file_windows.go new file mode 100644 index 0000000..ad9be1d --- /dev/null +++ b/teautils/file_windows.go @@ -0,0 +1,17 @@ +// +build windows + +package teautils + +import ( + "errors" + "os" +) + +// lock file +func LockFile(fp *os.File) error { + return errors.New("not implemented on windows") +} + +func UnlockFile(fp *os.File) error { + return errors.New("not implemented on windows") +} diff --git a/teautils/process.go b/teautils/process.go new file mode 100644 index 0000000..f4bcdd6 --- /dev/null +++ b/teautils/process.go @@ -0,0 +1,73 @@ +package teautils + +import ( + "fmt" + "github.com/iwind/TeaGo/types" + "io/ioutil" + "os" + "runtime" +) + +var pidFileList = []*os.File{} + +// 检查Pid +func CheckPid(path string) *os.Process { + file, err := os.Open(path) + if err != nil { + return nil + } + + defer func() { + _ = file.Close() + }() + + // 是否能取得Lock + err = LockFile(file) + if err == nil { + _ = UnlockFile(file) + return nil + } + + if runtime.GOOS == "windows" { + // windows上打开的文件是不能删除的 + if os.Remove(path) == nil { + return nil + } + } + + pidBytes, err := ioutil.ReadAll(file) + if err != nil { + return nil + } + pid := types.Int(string(pidBytes)) + + if pid <= 0 { + return nil + } + + proc, _ := os.FindProcess(pid) + return proc +} + +// 写入Pid +func WritePid(path string) error { + fp, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_RDONLY, 0666) + if err != nil { + return err + } + + if runtime.GOOS != "windows" { + err = LockFile(fp) + if err != nil { + return err + } + } + pidFileList = append(pidFileList, fp) // hold the file pointers + + _, err = fp.WriteString(fmt.Sprintf("%d", os.Getpid())) + if err != nil { + return err + } + + return nil +} diff --git a/teautils/process_test.go b/teautils/process_test.go new file mode 100644 index 0000000..28c3493 --- /dev/null +++ b/teautils/process_test.go @@ -0,0 +1,10 @@ +package teautils + +import ( + "github.com/iwind/TeaGo/Tea" + "testing" +) + +func TestCheckPid(t *testing.T) { + t.Log(CheckPid(Tea.Root + "/bin/pid")) +} diff --git a/teaweb/cmd/web_shell.go b/teaweb/cmd/web_shell.go index 22909c2..c3e17bd 100644 --- a/teaweb/cmd/web_shell.go +++ b/teaweb/cmd/web_shell.go @@ -6,6 +6,7 @@ import ( "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teaproxy" + "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" "github.com/TeaWeb/code/teaweb/configs" "github.com/iwind/TeaGo" @@ -337,7 +338,7 @@ func (this *WebShell) ExecStatus(writer io.Writer) bool { if proc == nil { this.write(writer, teaconst.TeaProductName+" not started yet") } else { - this.write(writer, teaconst.TeaProductName+" is running, pid:"+fmt.Sprintf("%d", proc.Pid)) + this.write(writer, teaconst.TeaProductName+" is running, pid: "+fmt.Sprintf("%d", proc.Pid)) } return true } @@ -377,97 +378,12 @@ func (this *WebShell) ExecPprof(writer io.Writer) bool { // 写入PID func (this *WebShell) writePid() error { - fp, err := os.OpenFile(Tea.Root+Tea.DS+"bin"+Tea.DS+"pid", os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_RDONLY, 0666) - if err != nil { - return err - } - - if runtime.GOOS == "windows" { - pidFP = fp // hold the fp to lock file - } else { - defer func() { - _ = fp.Close() - }() - } - - _, err = fp.WriteString(fmt.Sprintf("%d", os.Getpid())) - if err != nil { - return err - } - - return nil + return teautils.WritePid(Tea.Root + Tea.DS + "bin" + Tea.DS + "pid") } // 检查PID func (this *WebShell) checkPid() *os.Process { - // check pid file - pidFile := files.NewFile(Tea.Root + "/bin/pid") - if !pidFile.Exists() { - return nil - } - pidString, err := pidFile.ReadAllString() - if err != nil { - return nil - } - pid := types.Int(pidString) - - if pid <= 0 { - return nil - } - - // 如果是当前进程在检查,说明没有启动 - if pid == os.Getpid() { - return nil - } - - proc, err := os.FindProcess(pid) - if err != nil || proc == nil { - return nil - } - - if runtime.GOOS == "windows" { - // windows上打开的文件是不能删除的 - if pidFile.Delete() == nil { - return nil - } - return proc - } - - err = proc.Signal(syscall.Signal(0)) // 根据方法文档:Sending Interrupt on Windows is not implemented - if err != nil { - return nil - } - - // ps? - ps, err := exec.LookPath("ps") - if err != nil { - return proc - } - - cmd := exec.Command(ps, "-p", pidString, "-o", "command=") - output, err := cmd.Output() - if err != nil { - return proc - } - - if len(output) == 0 { - return nil - } - - outputString := string(output) - index := strings.LastIndex(outputString, "/") - if index> -1 { - outputString = outputString[index+1:] - } - index2 := strings.LastIndex(outputString, "\\") - if index2> 0 { - outputString = outputString[index2+1:] - } - if strings.Contains(outputString, teaconst.TeaProcessName) && !strings.Contains(outputString, teaconst.TeaProcessName+"-") { - return proc - } - - return nil + return teautils.CheckPid(Tea.Root + "/bin/pid") } // 写入string到writer diff --git a/teaweb/cmd/web_shell_others.go b/teaweb/cmd/web_shell_others.go index a853273..9f481e0 100644 --- a/teaweb/cmd/web_shell_others.go +++ b/teaweb/cmd/web_shell_others.go @@ -2,7 +2,9 @@ package cmd -import "io" +import ( + "io" +) // 启动服务模式 func (this *WebShell) ExecService(writer io.Writer) bool { From 16128596ca48c21752eb84a401f3e8e1f0af63d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月11日 16:54:34 +0800 Subject: [PATCH 022/121] =?UTF-8?q?[proxy]http=E5=92=8Cftp=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=E8=BF=9E=E6=8E=A5=E6=95=B0=E9=BB=98=E8=AE=A4=E7=9A=84?= =?UTF-8?q?=E6=9C=80=E5=B0=8F=E5=80=BC=E8=B0=83=E6=95=B4=E4=B8=BA8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/ftp_client_pool.go | 6 +++++- teaproxy/http_client_pool.go | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/teaproxy/ftp_client_pool.go b/teaproxy/ftp_client_pool.go index 8a1919b..3e0d2cf 100644 --- a/teaproxy/ftp_client_pool.go +++ b/teaproxy/ftp_client_pool.go @@ -41,8 +41,12 @@ func (this *FTPClientPool) client(backend *teaconfigs.BackendConfig) *FTPClient if backend.FTP == nil { backend.FTP = &teaconfigs.FTPBackendConfig{} } + numberCPU := runtime.NumCPU() + if numberCPU < 8 { + numberCPU = 8 + } if backend.IdleConns <= 0 { - backend.IdleConns = int32(runtime.NumCPU()) + backend.IdleConns = int32(numberCPU) } client = &FTPClient{ pool: &FTPConnectionPool{ diff --git a/teaproxy/http_client_pool.go b/teaproxy/http_client_pool.go index 0a17bc4..f4675a6 100644 --- a/teaproxy/http_client_pool.go +++ b/teaproxy/http_client_pool.go @@ -59,8 +59,8 @@ func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig) *http.Clie } numberCPU := runtime.NumCPU() - if numberCPU == 0 { - numberCPU = 1 + if numberCPU < 8 { + numberCPU = 8 } if maxConnections <= 0 { maxConnections = numberCPU From e8e1780b5e37b1cfa08491e20751a6f507113223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月11日 20:55:20 +0800 Subject: [PATCH 023/121] =?UTF-8?q?Windows=E4=B8=8A=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=AE=8C=E6=95=B4=E7=9A=84=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teautils/process.go | 31 +++++++-- teautils/signal.go | 24 +++++++ teautils/signal_windows.go | 84 ++++++++++++++++++++++ teaweb/cmd/web_shell.go | 138 ++++++++++++++----------------------- 4 files changed, 182 insertions(+), 95 deletions(-) create mode 100644 teautils/signal.go create mode 100644 teautils/signal_windows.go diff --git a/teautils/process.go b/teautils/process.go index f4bcdd6..b314302 100644 --- a/teautils/process.go +++ b/teautils/process.go @@ -12,6 +12,13 @@ var pidFileList = []*os.File{} // 检查Pid func CheckPid(path string) *os.Process { + // windows上打开的文件是不能删除的 + if runtime.GOOS == "windows" { + if os.Remove(path) == nil { + return nil + } + } + file, err := os.Open(path) if err != nil { return nil @@ -28,13 +35,6 @@ func CheckPid(path string) *os.Process { return nil } - if runtime.GOOS == "windows" { - // windows上打开的文件是不能删除的 - if os.Remove(path) == nil { - return nil - } - } - pidBytes, err := ioutil.ReadAll(file) if err != nil { return nil @@ -71,3 +71,20 @@ func WritePid(path string) error { return nil } + +// 删除Pid +func DeletePid(path string) error { + _, err := os.Stat(path) + if err != nil { + if !os.IsNotExist(err) { + return nil + } + return err + } + + for _, fp := range pidFileList { + _ = UnlockFile(fp) + _ = fp.Close() + } + return os.Remove(path) +} diff --git a/teautils/signal.go b/teautils/signal.go new file mode 100644 index 0000000..2ae77f8 --- /dev/null +++ b/teautils/signal.go @@ -0,0 +1,24 @@ +// +build !windows + +package teautils + +import ( + "os" + "os/signal" +) + +// 监听Signal +func ListenSignal(f func(sig os.Signal), sig ...os.Signal) { + ch := make(chan os.Signal, 8) + signal.Notify(ch, sig...) + go func() { + for r := range ch { + f(r) + } + }() +} + +// 通知Signal +func NotifySignal(proc *os.Process, sig os.Signal) error { + return proc.Signal(sig) +} diff --git a/teautils/signal_windows.go b/teautils/signal_windows.go new file mode 100644 index 0000000..f48c22c --- /dev/null +++ b/teautils/signal_windows.go @@ -0,0 +1,84 @@ +// +build windows + +package teautils + +import ( + "errors" + "fmt" + "github.com/Microsoft/go-winio" + "github.com/TeaWeb/code/teaconst" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/types" + "net" + "os" + "syscall" +) + +var signalPipe net.Listener + +const signalPipePath = `\\.\pipe\` + teaconst.TeaProcessName + `.signal.pipe` + +// 监听Signal +func ListenSignal(f func(sig os.Signal), sig ...os.Signal) { + pipe, err := winio.ListenPipe(signalPipePath, nil) + if err != nil { + logs.Error(err) + return + } + signalPipe = pipe + + go func() { + for { + conn, err := signalPipe.Accept() + if err != nil { + logs.Error(err) + continue + } + + go func(conn net.Conn) { + buf := make([]byte, 16) + for { + n, err := conn.Read(buf) + if n> 0 { + data := buf[:n] + r := syscall.Signal(types.Int(string(data))) + + // 是否存在 + found := false + for _, s := range sig { + if r == s { + found = true + break + } + } + if !found { + logs.Println("[ERROR]undefined signal '" + r.String() + "'") + continue + } + + f(r) + } + if err != nil { + break + } + } + }(conn) + } + }() +} + +// 通知Signal +func NotifySignal(proc *os.Process, sig os.Signal) error { + conn, err := winio.DialPipe(signalPipePath, nil) + if err != nil { + return errors.New("can not connect to signal pipe: " + err.Error()) + } + _, err = conn.Write([]byte(fmt.Sprintf("%d", sig))) + if err != nil { + return errors.New("signal sending failed: " + err.Error()) + } + + _ = conn.Close() + + return nil +} diff --git a/teaweb/cmd/web_shell.go b/teaweb/cmd/web_shell.go index c3e17bd..8fcccb3 100644 --- a/teaweb/cmd/web_shell.go +++ b/teaweb/cmd/web_shell.go @@ -11,16 +11,13 @@ import ( "github.com/TeaWeb/code/teaweb/configs" "github.com/iwind/TeaGo" "github.com/iwind/TeaGo/Tea" - "github.com/iwind/TeaGo/files" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" - "github.com/iwind/TeaGo/types" "io" "net/http" _ "net/http/pprof" "os" "os/exec" - "os/signal" "path/filepath" "runtime" "strings" @@ -29,7 +26,6 @@ import ( ) var sharedShell *WebShell = nil -var pidFP *os.File // hold pid file pointer on windows // 命令行相关封装 type WebShell struct { @@ -61,57 +57,48 @@ func (this *WebShell) Start(server *TeaGo.Server) { } // 信号 - signalsChannel := make(chan os.Signal, 16) - signal.Notify(signalsChannel, syscall.SIGINT, syscall.SIGHUP, syscall.Signal(0x1e /**syscall.SIGUSR1**/), syscall.Signal(0x1f /**syscall.SIGUSR2**/), syscall.SIGTERM) - go func() { - for { - sig := <-signalschannel - - if sig == syscall.SIGHUP { // 重置 - configs.SharedAdminConfig().Reset() - } else if sig == syscall.Signal(0x1e /**syscall.SIGUSR1**/) { // 刷新代理状态 - err := teaproxy.SharedManager.Restart() - if err != nil { - logs.Println("[error]" + err.Error()) - } else { - proxyutils.FinishChange() - } - } else if sig == syscall.Signal(0x1f /**syscall.SIGUSR2**/) { // 同步 - node := teaconfigs.SharedNodeConfig() - if node == nil { - logs.Println("[cluster]not a node yet") - return - } + teautils.ListenSignal(func(sig os.Signal) { + if sig == syscall.SIGHUP { // 重置 + configs.SharedAdminConfig().Reset() + } else if sig == syscall.Signal(0x1e /**syscall.SIGUSR1**/) { // 刷新代理状态 + err := teaproxy.SharedManager.Restart() + if err != nil { + logs.Println("[error]" + err.Error()) + } else { + proxyutils.FinishChange() + } + } else if sig == syscall.Signal(0x1f /**syscall.SIGUSR2**/) { // 同步 + node := teaconfigs.SharedNodeConfig() + if node == nil { + logs.Println("[cluster]not a node yet") + return + } - if node.IsMaster() { - logs.Println("[cluster]push items") - teacluster.SharedManager.BuildSum() - teacluster.SharedManager.PushItems() - } else { - logs.Println("[cluster]pull items") - teacluster.SharedManager.BuildSum() - teacluster.SharedManager.PullItems() - } + if node.IsMaster() { + logs.Println("[cluster]push items") + teacluster.SharedManager.BuildSum() + teacluster.SharedManager.PushItems() } else { - if sig == syscall.SIGINT { // 终止进程 - if server != nil { - server.Stop() - time.Sleep(1 * time.Second) - } + logs.Println("[cluster]pull items") + teacluster.SharedManager.BuildSum() + teacluster.SharedManager.PullItems() + } + } else { + if sig == syscall.SIGINT { // 终止进程 + if server != nil { + server.Stop() + time.Sleep(1 * time.Second) } + } - // 删除PID - pidFile := files.NewFile(Tea.Root + "/bin/pid") - if pidFile.Exists() { - err = pidFile.Delete() - if err != nil { - logs.Error(err) - } - } - os.Exit(0) + // 删除PID + err = teautils.DeletePid(Tea.Root + "/bin/pid") + if err != nil { + logs.Error(err) } + os.Exit(0) } - }() + }, syscall.SIGINT, syscall.SIGHUP, syscall.Signal(0x1e /**syscall.SIGUSR1**/), syscall.Signal(0x1f /**syscall.SIGUSR2**/), syscall.SIGTERM) } // 重置Root @@ -233,7 +220,7 @@ func (this *WebShell) ExecStart(writer io.Writer) bool { func (this *WebShell) ExecStop(writer io.Writer) bool { proc := this.checkPid() if proc == nil { - this.write(writer, teaconst.TeaProductName+" not started") + this.write(writer, teaconst.TeaProductName+" not started yet") return true } @@ -243,37 +230,23 @@ func (this *WebShell) ExecStop(writer io.Writer) bool { return true } - err = files.NewFile(Tea.Root + "/bin/pid").Delete() + // 在Windows上经常不能即使释放资源 + _ = teautils.DeletePid(Tea.Root + "/bin/pid") this.write(writer, teaconst.TeaProductName+" stopped ok, pid:", proc.Pid) - if err != nil { - this.write(writer, "ERROR:", err.Error()) - } - return true } // 重载代理配置 func (this *WebShell) ExecReload(writer io.Writer) bool { - pidString, err := files.NewFile(Tea.Root + Tea.DS + "bin" + Tea.DS + "pid").ReadAllString() - if err != nil { - this.write(writer, err.Error()) - return true - } - - pid := types.Int(pidString) - proc, err := os.FindProcess(pid) - if err != nil { - this.write(writer, err.Error()) - return true - } + proc := this.checkPid() if proc == nil { - this.write(writer, "can not find process") + this.write(writer, teaconst.TeaProductName+" not started yet") return true } - err = proc.Signal(syscall.Signal(0x1e /**syscall.SIGUSR1**/)) + err := teautils.NotifySignal(proc, syscall.Signal(0x1e /**syscall.SIGUSR1**/)) if err != nil { - logs.Error(err) + this.write(writer, "[ERROR]"+err.Error()) return true } this.write(writer, "reload success") @@ -300,32 +273,21 @@ func (this *WebShell) ExecRestart(writer io.Writer) bool { this.write(writer, teaconst.TeaProductName+" restart failed:", err.Error()) return true } - this.write(writer, teaconst.TeaProductName+" restarted ok, pid:", cmd.Process.Pid) + this.write(writer, teaconst.TeaProductName+" restarted ok, new pid:", cmd.Process.Pid) return true } // 重置 func (this *WebShell) ExecReset(writer io.Writer) bool { - pidString, err := files.NewFile(Tea.Root + Tea.DS + "bin" + Tea.DS + "pid").ReadAllString() - if err != nil { - this.write(writer, err.Error()) - return true - } - - pid := types.Int(pidString) - proc, err := os.FindProcess(pid) - if err != nil { - this.write(writer, err.Error()) - return true - } + proc := this.checkPid() if proc == nil { - this.write(writer, "can not find process") + this.write(writer, teaconst.TeaProductName+" not started yet") return true } - err = proc.Signal(syscall.SIGHUP) + err := teautils.NotifySignal(proc, syscall.SIGHUP) if err != nil { - this.write(writer, err.Error()) + this.write(writer, "[ERROR]"+err.Error()) return true } this.write(writer, "reset success") @@ -349,9 +311,9 @@ func (this *WebShell) ExecSync(writer io.Writer) bool { if proc == nil { this.write(writer, teaconst.TeaProductName+" not started yet") } else { - err := proc.Signal(syscall.Signal(0x1f /**syscall.SIGUSR2**/)) + err := teautils.NotifySignal(proc, syscall.Signal(0x1f /**syscall.SIGUSR2**/)) if err != nil { - logs.Error(err) + this.write(writer, "[ERROR]"+err.Error()) return true } this.write(writer, "signal sent successfully") From c538870189a72d3dac7592c2945eb4763ac0563e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月12日 10:44:33 +0800 Subject: [PATCH 024/121] =?UTF-8?q?[proxy]=E5=AF=B9=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=E5=85=B3=E9=97=AD=E8=BF=9E=E6=8E=A5=E7=9A=84=E6=8F=90?= =?UTF-8?q?=E7=A4=BA=E6=9B=B4=E5=8A=A0=E5=8F=8B=E5=A5=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_backend.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/teaproxy/request_backend.go b/teaproxy/request_backend.go index 27c27d1..fe249fb 100644 --- a/teaproxy/request_backend.go +++ b/teaproxy/request_backend.go @@ -152,8 +152,20 @@ func (this *Request) callBackend(writer *ResponseWriter) error { logs.Println("[proxy]'" + this.raw.URL.String() + "': " + err.Error()) this.addError(err) } else { - this.serverError(writer) - this.addError(err) + // 是否为客户端方面的错误 + isClientError := false + if ok { + if httpErr.Err == context.Canceled { + isClientError = true + this.addError(errors.New(httpErr.Op + " " + httpErr.URL + ": client closed the connection")) + writer.WriteHeader(499) // 仿照nginx + } + } + + if !isClientError { + this.serverError(writer) + this.addError(err) + } } if resp != nil && resp.Body != nil { _ = resp.Body.Close() From 1c45ab0c51f4f5b09661d0feeffb16223a1252e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月12日 11:19:07 +0800 Subject: [PATCH 025/121] =?UTF-8?q?[notice]=E5=A2=9E=E5=8A=A0Telegram?= =?UTF-8?q?=E5=AA=92=E4=BB=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/notices/media_telegram.go | 33 ++++++++++++ teaconfigs/notices/media_type.go | 9 ++++ teaweb/actions/default/notices/addMedia.go | 53 +++++++++++++++--- teaweb/actions/default/notices/updateMedia.go | 54 +++++++++++++++---- 4 files changed, 132 insertions(+), 17 deletions(-) create mode 100644 teaconfigs/notices/media_telegram.go diff --git a/teaconfigs/notices/media_telegram.go b/teaconfigs/notices/media_telegram.go new file mode 100644 index 0000000..96e11ed --- /dev/null +++ b/teaconfigs/notices/media_telegram.go @@ -0,0 +1,33 @@ +package notices + +import ( + tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api" + "github.com/iwind/TeaGo/types" +) + +// Telegram媒介 +type NoticeTelegramMedia struct { + Token string `yaml:"token" json:"token"` +} + +// 获取新对象 +func NewNoticeTelegramMedia() *NoticeTelegramMedia { + return &NoticeTelegramMedia{} +} + +// 发送消息 +func (this *NoticeTelegramMedia) Send(user string, subject string, body string) (respBytes []byte, err error) { + bot, err := tgbotapi.NewBotAPI(this.Token) + if err != nil { + return nil, err + } + + msg := tgbotapi.NewMessage(types.Int64(user), subject+"\n"+body) + _, err = bot.Send(msg) + return nil, err +} + +// 是否需要用户标识 +func (this *NoticeTelegramMedia) RequireUser() bool { + return true +} diff --git a/teaconfigs/notices/media_type.go b/teaconfigs/notices/media_type.go index 8380971..cf14d16 100644 --- a/teaconfigs/notices/media_type.go +++ b/teaconfigs/notices/media_type.go @@ -13,6 +13,7 @@ const ( NoticeMediaTypeQyWeixin = "qyWeixin" NoticeMediaTypeQyWeixinRobot = "qyWeixinRobot" NoticeMediaTypeAliyunSms = "aliyunSms" + NoticeMediaTypeTelegram = "telegram" NoticeMediaTypeTeaSms = "teaSms" ) @@ -75,6 +76,14 @@ func AllNoticeMediaTypes() []maps.Map { "description": "通过阿里云短信服务发送短信,相关文档»", "user": "接收消息的手机号", }, + { + "name": "Telegram机器人", + "code": NoticeMediaTypeTelegram, + "supportsHTML": false, + "instance": new(NoticeTelegramMedia), + "description": "通过机器人向群或者某个用户发送消息,需要确保所在网络能够访问Telegram API服务", + "user": "群或用户的Chat ID,通常是一个数字,可以通过和 @get_id_bot 建立对话并发送任意消息获得", + }, { "name": "TeaOS云短信", "code": NoticeMediaTypeTeaSms, diff --git a/teaweb/actions/default/notices/addMedia.go b/teaweb/actions/default/notices/addMedia.go index a8467cc..fa3c9f4 100644 --- a/teaweb/actions/default/notices/addMedia.go +++ b/teaweb/actions/default/notices/addMedia.go @@ -7,6 +7,7 @@ import ( "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/logs" "net/http" ) @@ -65,6 +66,8 @@ func (this *AddMediaAction) RunPost(params struct { AliyunSmsAccessKeyId string AliyunSmsAccessKeySecret string + TelegramToken string + TeaSmsAccessId string TeaSmsAccessSecret string @@ -111,7 +114,10 @@ func (this *AddMediaAction) RunPost(params struct { media.Username = params.EmailUsername media.Password = params.EmailPassword media.From = params.EmailFrom - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeWebhook: params.Must. Field("webhookURL", params.WebhookURL). @@ -143,7 +149,10 @@ func (this *AddMediaAction) RunPost(params struct { media.Body = params.WebhookBody } - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeScript: if params.ScriptType == "path" { params.Must. @@ -172,7 +181,10 @@ func (this *AddMediaAction) RunPost(params struct { } } - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeDingTalk: params.Must. Field("dingTalkWebhookURL", params.DingTalkWebhookURL). @@ -181,7 +193,10 @@ func (this *AddMediaAction) RunPost(params struct { media := notices.NewNoticeDingTalkMedia() media.WebhookURL = params.DingTalkWebhookURL - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeQyWeixin: params.Must. Field("qyWeixinCorporateId", params.QyWeixinCorporateId). @@ -196,7 +211,10 @@ func (this *AddMediaAction) RunPost(params struct { media.AgentId = params.QyWeixinAgentId media.AppSecret = params.QyWeixinAppSecret media.TextFormat = params.QyWeixinTextFormat - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeQyWeixinRobot: params.Must. Field("qyWeixinRobotWebhookURL", params.QyWeixinRobotWebhookURL). @@ -206,7 +224,10 @@ func (this *AddMediaAction) RunPost(params struct { media := notices.NewNoticeQyWeixinRobotMedia() media.WebhookURL = params.QyWeixinRobotWebhookURL media.TextFormat = params.QyWeixinRobotTextFormat - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeAliyunSms: params.Must. Field("aliyunSmsSign", params.AliyunSmsSign). @@ -233,7 +254,20 @@ func (this *AddMediaAction) RunPost(params struct { } } - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } + case notices.NoticeMediaTypeTelegram: + params.Must. + Field("telegramToken", params.TelegramToken). + Require("请输入机器人Token") + media := notices.NewNoticeTelegramMedia() + media.Token = params.TelegramToken + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeTeaSms: params.Must. Field("teaSmsAccessId", params.TeaSmsAccessId). @@ -243,7 +277,10 @@ func (this *AddMediaAction) RunPost(params struct { media := notices.NewNoticeTeaSmsMedia() media.AccessId = params.TeaSmsAccessId media.AccessSecret = params.TeaSmsAccessSecret - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } } // 时间 diff --git a/teaweb/actions/default/notices/updateMedia.go b/teaweb/actions/default/notices/updateMedia.go index c69ee80..77e98a2 100644 --- a/teaweb/actions/default/notices/updateMedia.go +++ b/teaweb/actions/default/notices/updateMedia.go @@ -7,6 +7,7 @@ import ( "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/logs" "net/http" ) @@ -78,6 +79,8 @@ func (this *UpdateMediaAction) RunPost(params struct { AliyunSmsAccessKeyId string AliyunSmsAccessKeySecret string + TelegramToken string + TeaSmsAccessId string TeaSmsAccessSecret string @@ -129,7 +132,10 @@ func (this *UpdateMediaAction) RunPost(params struct { media.Username = params.EmailUsername media.Password = params.EmailPassword media.From = params.EmailFrom - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeWebhook: params.Must. Field("webhookURL", params.WebhookURL). @@ -160,7 +166,10 @@ func (this *UpdateMediaAction) RunPost(params struct { } else if params.WebhookContentType == "body" { media.Body = params.WebhookBody } - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeScript: if params.ScriptType == "path" { params.Must. @@ -189,7 +198,10 @@ func (this *UpdateMediaAction) RunPost(params struct { } } - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeDingTalk: params.Must. Field("dingTalkWebhookURL", params.DingTalkWebhookURL). @@ -198,7 +210,10 @@ func (this *UpdateMediaAction) RunPost(params struct { media := notices.NewNoticeDingTalkMedia() media.WebhookURL = params.DingTalkWebhookURL - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeQyWeixin: params.Must. Field("qyWeixinCorporateId", params.QyWeixinCorporateId). @@ -213,7 +228,10 @@ func (this *UpdateMediaAction) RunPost(params struct { media.AgentId = params.QyWeixinAgentId media.AppSecret = params.QyWeixinAppSecret media.TextFormat = params.QyWeixinTextFormat - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeQyWeixinRobot: params.Must. Field("qyWeixinRobotWebhookURL", params.QyWeixinRobotWebhookURL). @@ -223,7 +241,10 @@ func (this *UpdateMediaAction) RunPost(params struct { media := notices.NewNoticeQyWeixinRobotMedia() media.WebhookURL = params.QyWeixinRobotWebhookURL media.TextFormat = params.QyWeixinRobotTextFormat - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeAliyunSms: params.Must. Field("aliyunSmsSign", params.AliyunSmsSign). @@ -250,8 +271,20 @@ func (this *UpdateMediaAction) RunPost(params struct { } } - teautils.ObjectToMapJSON(media, &mediaConfig.Options) - + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } + case notices.NoticeMediaTypeTelegram: + params.Must. + Field("telegramToken", params.TelegramToken). + Require("请输入机器人Token") + media := notices.NewNoticeTelegramMedia() + media.Token = params.TelegramToken + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } case notices.NoticeMediaTypeTeaSms: params.Must. Field("teaSmsAccessId", params.TeaSmsAccessId). @@ -261,7 +294,10 @@ func (this *UpdateMediaAction) RunPost(params struct { media := notices.NewNoticeTeaSmsMedia() media.AccessId = params.TeaSmsAccessId media.AccessSecret = params.TeaSmsAccessSecret - teautils.ObjectToMapJSON(media, &mediaConfig.Options) + err := teautils.ObjectToMapJSON(media, &mediaConfig.Options) + if err != nil { + logs.Error(err) + } } // 时间 From cf192caefc3da28eb83dbe1c7d24e1dcb4a8266c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月13日 15:01:37 +0800 Subject: [PATCH 026/121] =?UTF-8?q?[proxy]=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E7=AD=96=E7=95=A5=E5=A2=9E=E5=8A=A0syslog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tealogs/storage_command.go | 2 +- tealogs/storage_manager.go | 4 +- tealogs/storage_syslog.go | 184 ++++++++++++++++++ tealogs/storage_tcp.go | 4 +- tealogs/storage_types.go | 1 + tealogs/storage_utils.go | 5 + .../actions/default/proxy/log/policies/add.go | 37 +++- .../default/proxy/log/policies/policy.go | 3 + .../default/proxy/log/policies/update.go | 35 +++- 9 files changed, 268 insertions(+), 7 deletions(-) create mode 100644 tealogs/storage_syslog.go diff --git a/tealogs/storage_command.go b/tealogs/storage_command.go index 2f3fe9a..6335643 100644 --- a/tealogs/storage_command.go +++ b/tealogs/storage_command.go @@ -69,7 +69,7 @@ func (this *CommandStorage) Write(accessLogs []*accesslogs.AccessLog) error { logs.Error(err) } } - w.Close() + _ = w.Close() err = cmd.Wait() if err != nil { logs.Error(err) diff --git a/tealogs/storage_manager.go b/tealogs/storage_manager.go index f2a33e3..6cd588d 100644 --- a/tealogs/storage_manager.go +++ b/tealogs/storage_manager.go @@ -58,7 +58,7 @@ func FindPolicyStorage(policyId string) StorageInterface { if storage != nil { err := storage.Start() if err != nil { - logs.Println("access log storage '"+policyId+"/"+FindPolicyName(policyId)+"' start failed:", err.Error()) + logs.Println("access log storage '"+policyId+"/"+policy.Name+"' start failed:", err.Error()) storage = nil } } @@ -108,6 +108,8 @@ func DecodePolicyStorage(policy *teaconfigs.AccessLogStoragePolicy) StorageInter instance = new(MySQLStorage) case StorageTypeTCP: instance = new(TCPStorage) + case StorageTypeSyslog: + instance = new(SyslogStorage) case StorageTypeCommand: instance = new(CommandStorage) } diff --git a/tealogs/storage_syslog.go b/tealogs/storage_syslog.go new file mode 100644 index 0000000..eae5d00 --- /dev/null +++ b/tealogs/storage_syslog.go @@ -0,0 +1,184 @@ +package tealogs + +import ( + "errors" + "github.com/TeaWeb/code/tealogs/accesslogs" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" + "os/exec" + "runtime" + "strconv" +) + +type SyslogStorageProtocol = string + +const ( + SyslogStorageProtocolTCP SyslogStorageProtocol = "tcp" + SyslogStorageProtocolUDP SyslogStorageProtocol = "udp" + SyslogStorageProtocolNone SyslogStorageProtocol = "none" + SyslogStorageProtocolSocket SyslogStorageProtocol = "socket" +) + +type SyslogStoragePriority = int + +const ( + SyslogStoragePriorityEmerg SyslogStoragePriority = iota + SyslogStoragePriorityAlert + SyslogStoragePriorityCrit + SyslogStoragePriorityErr + SyslogStoragePriorityWarning + SyslogStoragePriorityNotice + SyslogStoragePriorityInfo + SyslogStoragePriorityDebug +) + +var SyslogStoragePriorities = []maps.Map{ + { + "name": "[无]", + "value": -1, + }, + { + "name": "EMERG", + "value": SyslogStoragePriorityEmerg, + }, + { + "name": "ALERT", + "value": SyslogStoragePriorityAlert, + }, + { + "name": "CRIT", + "value": SyslogStoragePriorityCrit, + }, + { + "name": "ERR", + "value": SyslogStoragePriorityErr, + }, + { + "name": "WARNING", + "value": SyslogStoragePriorityWarning, + }, + { + "name": "NOTICE", + "value": SyslogStoragePriorityNotice, + }, + { + "name": "INFO", + "value": SyslogStoragePriorityInfo, + }, + { + "name": "DEBUG", + "value": SyslogStoragePriorityDebug, + }, +} + +// syslog存储策略 +type SyslogStorage struct { + Storage `yaml:", inline"` + + Protocol string `yaml:"protocol" json:"protocol"` // SysLogStorageProtocol* + ServerAddr string `yaml:"serverAddr" json:"serverAddr"` + ServerPort int `yaml:"serverPort" json:"serverPort"` + Socket string `yaml:"socket" json:"socket"` // sock file + Tag string `yaml:"tag" json:"tag"` + Priority SyslogStoragePriority `yaml:"priority" json:"priority"` + + exe string +} + +// 开启 +func (this *SyslogStorage) Start() error { + if runtime.GOOS != "linux" { + return errors.New("'syslog' storage only works on linux") + } + + exe, err := exec.LookPath("logger") + if err != nil { + return err + } + + this.exe = exe + + return nil +} + +// 写入日志 +func (this *SyslogStorage) Write(accessLogs []*accesslogs.AccessLog) error { + if len(accessLogs) == 0 { + return nil + } + + args := []string{} + if len(this.Tag)> 0 { + args = append(args, "-t", this.Tag) + } + + if this.Priority>= 0 { + args = append(args, "-p", strconv.Itoa(this.Priority)) + } + + switch this.Protocol { + case SyslogStorageProtocolTCP: + args = append(args, "-T") + if len(this.ServerAddr)> 0 { + args = append(args, "-n", this.ServerAddr) + } + if this.ServerPort> 0 { + args = append(args, "-P", strconv.Itoa(this.ServerPort)) + } + case SyslogStorageProtocolUDP: + args = append(args, "-d") + if len(this.ServerAddr)> 0 { + args = append(args, "-n", this.ServerAddr) + } + if this.ServerPort> 0 { + args = append(args, "-P", strconv.Itoa(this.ServerPort)) + } + case SyslogStorageProtocolSocket: + args = append(args, "-u") + args = append(args, this.Socket) + case SyslogStorageProtocolNone: + // do nothing + } + + args = append(args, "-S", "10240") + + cmd := exec.Command(this.exe, args...) + w, err := cmd.StdinPipe() + if err != nil { + return err + } + err = cmd.Start() + if err != nil { + return err + } + + for _, accessLog := range accessLogs { + data, err := this.FormatAccessLogBytes(accessLog) + if err != nil { + logs.Error(err) + continue + } + _, err = w.Write(data) + if err != nil { + logs.Error(err) + } + + _, err = w.Write([]byte("\n")) + if err != nil { + logs.Error(err) + } + } + + _ = w.Close() + err = cmd.Wait() + if err != nil { + return err + } + + return nil +} + +// 关闭 +func (this *SyslogStorage) Close() error { + return nil +} diff --git a/tealogs/storage_tcp.go b/tealogs/storage_tcp.go index ce2f343..e6dd2b2 100644 --- a/tealogs/storage_tcp.go +++ b/tealogs/storage_tcp.go @@ -59,12 +59,12 @@ func (this *TCPStorage) Write(accessLogs []*accesslogs.AccessLog) error { } _, err = conn.Write(data) if err != nil { - this.Close() + _ = this.Close() break } _, err = conn.Write([]byte("\n")) if err != nil { - this.Close() + _ = this.Close() break } } diff --git a/tealogs/storage_types.go b/tealogs/storage_types.go index b26575c..49a3daf 100644 --- a/tealogs/storage_types.go +++ b/tealogs/storage_types.go @@ -8,5 +8,6 @@ const ( StorageTypeES StorageType = "es" StorageTypeMySQL StorageType = "mysql" StorageTypeTCP StorageType = "tcp" + StorageTypeSyslog StorageType = "syslog" StorageTypeCommand StorageType = "command" ) diff --git a/tealogs/storage_utils.go b/tealogs/storage_utils.go index 04e6fe1..8312865 100644 --- a/tealogs/storage_utils.go +++ b/tealogs/storage_utils.go @@ -27,6 +27,11 @@ func AllStorages() []maps.Map { "type": StorageTypeTCP, "description": "将日志通过TCP套接字输出", }, + { + "name": "Syslog", + "type": StorageTypeSyslog, + "description": "将日志通过syslog输出,仅支持Linux", + }, { "name": "命令行输入流", "type": StorageTypeCommand, diff --git a/teaweb/actions/default/proxy/log/policies/add.go b/teaweb/actions/default/proxy/log/policies/add.go index 11a2e30..a7bc9ef 100644 --- a/teaweb/actions/default/proxy/log/policies/add.go +++ b/teaweb/actions/default/proxy/log/policies/add.go @@ -23,6 +23,9 @@ func (this *AddAction) RunGet(params struct{}) { this.Data["condOperators"] = shared.AllRequestOperators() this.Data["condVariables"] = proxyutils.DefaultRequestVariables() + // syslog + this.Data["syslogPriorities"] = tealogs.SyslogStoragePriorities + this.Show() } @@ -54,6 +57,14 @@ func (this *AddAction) RunPost(params struct { TcpNetwork string TcpAddr string + // syslog + SyslogProtocol string + SyslogServerAddr string + SyslogServerPort int + SyslogSocket string + SyslogTag string + SyslogPriority int + // command CommandCommand string CommandArgs string @@ -133,6 +144,28 @@ func (this *AddAction) RunPost(params struct { storage.Network = params.TcpNetwork storage.Addr = params.TcpAddr instance = storage + case tealogs.StorageTypeSyslog: + switch params.SyslogProtocol { + case tealogs.SyslogStorageProtocolTCP, tealogs.SyslogStorageProtocolUDP: + params.Must. + Field("syslogServerAddr", params.SyslogServerAddr). + Require("请输入网络地址") + case tealogs.SyslogStorageProtocolSocket: + params.Must. + Field("syslogSocket", params.SyslogSocket). + Require("请输入Socket路径") + } + + storage := new(tealogs.SyslogStorage) + storage.Format = params.StorageFormat + storage.Template = params.StorageTemplate + storage.Protocol = params.SyslogProtocol + storage.ServerAddr = params.SyslogServerAddr + storage.ServerPort = params.SyslogServerPort + storage.Socket = params.SyslogSocket + storage.Tag = params.SyslogTag + storage.Priority = params.SyslogPriority + instance = storage case tealogs.StorageTypeCommand: params.Must. Field("commandCommand", params.CommandCommand). @@ -148,7 +181,7 @@ func (this *AddAction) RunPost(params struct { } if instance == nil { - this.Fail("请选择存储类型") + this.Fail("找不到选择的存储类型") } policy := teaconfigs.NewAccessLogStoragePolicy() @@ -179,7 +212,7 @@ func (this *AddAction) RunPost(params struct { policyList.AddId(policy.Id) err = policyList.Save() if err != nil { - policy.Delete() + _ = policy.Delete() this.Fail("保存失败:" + err.Error()) } diff --git a/teaweb/actions/default/proxy/log/policies/policy.go b/teaweb/actions/default/proxy/log/policies/policy.go index a344406..0d693ff 100644 --- a/teaweb/actions/default/proxy/log/policies/policy.go +++ b/teaweb/actions/default/proxy/log/policies/policy.go @@ -37,5 +37,8 @@ func (this *PolicyAction) RunGet(params struct { this.Data["configItems"] = FindAllUsingPolicy(policy.Id) + // syslog + this.Data["syslogPriorities"] = tealogs.SyslogStoragePriorities + this.Show() } diff --git a/teaweb/actions/default/proxy/log/policies/update.go b/teaweb/actions/default/proxy/log/policies/update.go index 4995514..387f9ac 100644 --- a/teaweb/actions/default/proxy/log/policies/update.go +++ b/teaweb/actions/default/proxy/log/policies/update.go @@ -40,6 +40,9 @@ func (this *UpdateAction) RunGet(params struct { this.Data["condOperators"] = shared.AllRequestOperators() this.Data["condVariables"] = proxyutils.DefaultRequestVariables() + // syslog + this.Data["syslogPriorities"] = tealogs.SyslogStoragePriorities + this.Show() } @@ -73,6 +76,14 @@ func (this *UpdateAction) RunPost(params struct { TcpNetwork string TcpAddr string + // syslog + SyslogProtocol string + SyslogServerAddr string + SyslogServerPort int + SyslogSocket string + SyslogTag string + SyslogPriority int + // command CommandCommand string CommandArgs string @@ -157,6 +168,28 @@ func (this *UpdateAction) RunPost(params struct { storage.Network = params.TcpNetwork storage.Addr = params.TcpAddr instance = storage + case tealogs.StorageTypeSyslog: + switch params.SyslogProtocol { + case tealogs.SyslogStorageProtocolTCP, tealogs.SyslogStorageProtocolUDP: + params.Must. + Field("syslogServerAddr", params.SyslogServerAddr). + Require("请输入网络地址") + case tealogs.SyslogStorageProtocolSocket: + params.Must. + Field("syslogSocket", params.SyslogSocket). + Require("请输入Socket路径") + } + + storage := new(tealogs.SyslogStorage) + storage.Format = params.StorageFormat + storage.Template = params.StorageTemplate + storage.Protocol = params.SyslogProtocol + storage.ServerAddr = params.SyslogServerAddr + storage.ServerPort = params.SyslogServerPort + storage.Socket = params.SyslogSocket + storage.Tag = params.SyslogTag + storage.Priority = params.SyslogPriority + instance = storage case tealogs.StorageTypeCommand: params.Must. Field("commandCommand", params.CommandCommand). @@ -172,7 +205,7 @@ func (this *UpdateAction) RunPost(params struct { } if instance == nil { - this.Fail("请选择存储类型") + this.Fail("找不到选择存储类型") } policy.Type = params.StorageType From 5cd284383e3f4e5cac626c07ca929bf554dce5c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月13日 16:58:40 +0800 Subject: [PATCH 027/121] =?UTF-8?q?[proxy]=E8=AF=B7=E6=B1=82=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E6=9D=A1=E4=BB=B6=E5=A2=9E=E5=8A=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E5=92=8C=E6=96=87=E4=BB=B6=E4=B8=8D=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E4=B8=A4=E4=B8=AA=E6=93=8D=E4=BD=9C=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/shared/request_cond.go | 28 +++++++++ teaconfigs/shared/request_cond_test.go | 78 ++++++++++++++++++++++++++ teaconfigs/shared/request_operators.go | 16 ++++++ teaproxy/request.go | 47 ++++++++++++++-- 4 files changed, 165 insertions(+), 4 deletions(-) diff --git a/teaconfigs/shared/request_cond.go b/teaconfigs/shared/request_cond.go index 661b51a..11f9bd2 100644 --- a/teaconfigs/shared/request_cond.go +++ b/teaconfigs/shared/request_cond.go @@ -5,10 +5,12 @@ import ( "encoding/binary" "encoding/json" "errors" + "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/types" "github.com/iwind/TeaGo/utils/string" "net" + "os" "path/filepath" "regexp" "strings" @@ -328,6 +330,32 @@ func (this *RequestCond) Match(formatter func(source string) string) bool { return this.ipToInt64(net.ParseIP(paramValue))%10 == types.Int64(this.Value) case RequestCondOperatorIPMod100: return this.ipToInt64(net.ParseIP(paramValue))%100 == types.Int64(this.Value) + case RequestCondOperatorFileExist: + index := strings.Index(paramValue, "?") + if index> -1 { + paramValue = paramValue[:index] + } + if len(paramValue) == 0 { + return false + } + if !filepath.IsAbs(paramValue) { + paramValue = Tea.Root + Tea.DS + paramValue + } + stat, err := os.Stat(paramValue) + return err == nil && !stat.IsDir() + case RequestCondOperatorFileNotExist: + index := strings.Index(paramValue, "?") + if index> -1 { + paramValue = paramValue[:index] + } + if len(paramValue) == 0 { + return true + } + if !filepath.IsAbs(paramValue) { + paramValue = Tea.Root + Tea.DS + paramValue + } + stat, err := os.Stat(paramValue) + return err != nil || stat.IsDir() } return false diff --git a/teaconfigs/shared/request_cond_test.go b/teaconfigs/shared/request_cond_test.go index 7206284..b62ecbf 100644 --- a/teaconfigs/shared/request_cond_test.go +++ b/teaconfigs/shared/request_cond_test.go @@ -3,6 +3,7 @@ package shared import ( "bytes" "fmt" + "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/assert" "net" "regexp" @@ -685,6 +686,83 @@ func TestRequestCond_File(t *testing.T) { return source })) } + + { + cond := RequestCond{ + Param: "a.png", + Operator: RequestCondOperatorFileExist, + } + a.IsNil(cond.Validate()) + a.IsFalse(cond.Match(func(source string) string { + return source + })) + } + + { + cond := RequestCond{ + Param: Tea.Root + "/README.md", + Operator: RequestCondOperatorFileExist, + } + a.IsNil(cond.Validate()) + a.IsTrue(cond.Match(func(source string) string { + return source + })) + } + + { + cond := RequestCond{ + Param: Tea.Root + "/README.md?v=1", + Operator: RequestCondOperatorFileExist, + } + a.IsNil(cond.Validate()) + a.IsTrue(cond.Match(func(source string) string { + return source + })) + } + + { + cond := RequestCond{ + Param: Tea.Root, + Operator: RequestCondOperatorFileExist, + } + a.IsNil(cond.Validate()) + a.IsFalse(cond.Match(func(source string) string { + return source + })) + } + + { + cond := RequestCond{ + Param: Tea.Root, + Operator: RequestCondOperatorFileExist, + } + a.IsNil(cond.Validate()) + a.IsFalse(cond.Match(func(source string) string { + return source + })) + } + + { + cond := RequestCond{ + Param: "a.png", + Operator: RequestCondOperatorFileNotExist, + } + a.IsNil(cond.Validate()) + a.IsTrue(cond.Match(func(source string) string { + return source + })) + } + + { + cond := RequestCond{ + Param: Tea.Root + "/README.md", + Operator: RequestCondOperatorFileNotExist, + } + a.IsNil(cond.Validate()) + a.IsFalse(cond.Match(func(source string) string { + return source + })) + } } func TestRequestCond_MimeType(t *testing.T) { diff --git a/teaconfigs/shared/request_operators.go b/teaconfigs/shared/request_operators.go index 0c6d52a..ce88cea 100644 --- a/teaconfigs/shared/request_operators.go +++ b/teaconfigs/shared/request_operators.go @@ -46,6 +46,10 @@ const ( RequestCondOperatorIPMod10 RequestCondOperator = "ip mod 10" RequestCondOperatorIPMod100 RequestCondOperator = "ip mod 100" RequestCondOperatorIPMod RequestCondOperator = "ip mod" + + // 文件相关 + RequestCondOperatorFileExist RequestCondOperator = "file exist" + RequestCondOperatorFileNotExist RequestCondOperator = "file not exist" ) // 所有的运算符 @@ -206,5 +210,17 @@ func AllRequestOperators() []maps.Map { "op": RequestCondOperatorIPMod, "description": "对IP参数值取模,对比值格式为:除数,余数,比如10,1", }, + + { + "name": "文件存在", + "op": RequestCondOperatorFileExist, + "description": "判断参数值解析后的文件是否存在", + }, + + { + "name": "文件不存在", + "op": RequestCondOperatorFileNotExist, + "description": "判断参数值解析后的文件是否不存在", + }, } } diff --git a/teaproxy/request.go b/teaproxy/request.go index a64d0d6..edf5914 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -23,6 +23,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "time" ) @@ -129,6 +130,8 @@ type Request struct { gzip *teaconfigs.GzipConfig debug bool + locationContext *teaconfigs.LocationConfig // 当前变量的上下文 *Location ... + hasForwardHeader bool } @@ -219,6 +222,8 @@ func (this *Request) reset(rawRequest *http.Request) { this.accessLog = nil + this.locationContext = nil + this.gzip = nil this.debug = false @@ -331,6 +336,7 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b if !location.On { continue } + this.locationContext = location if locationMatches, ok := location.Match(rawPath, this.Format); ok { this.addVarMapping(locationMatches) @@ -363,6 +369,7 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b } if location.RedirectToHttps && this.rawScheme == "http" { this.redirectToHttps = true + this.locationContext = nil return nil } @@ -428,6 +435,7 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b this.rewriteIsExternal = true this.rewriteRedirectMode = rule.RedirectMode() this.rewriteProxyHost = rule.ProxyHost + this.locationContext = nil return nil } @@ -436,12 +444,14 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b this.rewriteReplace = replace this.rewriteIsExternal = false this.rewriteRedirectMode = teaconfigs.RewriteFlagRedirect + this.locationContext = nil return nil } newURI, err := url.ParseRequestURI(replace) if err != nil { this.uri = replace + this.locationContext = nil return nil } if len(newURI.RawQuery)> 0 { @@ -458,18 +468,23 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b switch rule.TargetType() { case teaconfigs.RewriteTargetURL: + this.locationContext = nil return this.configure(server, redirects, rule.IsBreak) case teaconfigs.RewriteTargetProxy: proxyId := rule.TargetProxy() server := SharedManager.FindServer(proxyId) if server == nil { + this.locationContext = nil return errors.New("server with '" + proxyId + "' not found") } if !server.On { + this.locationContext = nil return errors.New("server with '" + proxyId + "' not available now") } + this.locationContext = nil return this.configure(server, redirects, rule.IsBreak) } + this.locationContext = nil return nil } } @@ -497,11 +512,14 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b if len(location.Proxy)> 0 { server := SharedManager.FindServer(location.Proxy) if server == nil { + this.locationContext = nil return errors.New("server with '" + location.Proxy + "' not found") } if !server.On { + this.locationContext = nil return errors.New("server with '" + location.Proxy + "' not available now") } + this.locationContext = nil return this.configure(server, redirects, breakRewrite) } @@ -509,6 +527,7 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b if len(location.Backends)> 0 { backend := location.NextBackend(this.backendCall) if backend == nil { + this.locationContext = nil return errors.New("no backends available") } if len(this.backendCall.ResponseCallbacks)> 0 { @@ -536,9 +555,11 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b if location.Websocket != nil && location.Websocket.On { this.backend = location.Websocket.NextBackend(this.backendCall) this.websocket = location.Websocket + this.locationContext = nil return nil } } + this.locationContext = nil } // 如果经过location找到了相关配置,就终止 @@ -1137,7 +1158,7 @@ func (this *Request) Format(source string) string { } return addr case "remotePort": - return fmt.Sprintf("%d", this.requestRemotePort()) + return strconv.Itoa(this.requestRemotePort()) case "remoteUser": return this.requestRemoteUser() case "requestURI", "requestUri": @@ -1151,7 +1172,20 @@ func (this *Request) Format(source string) string { case "requestMethod": return this.requestMethod() case "requestFilename": - return this.requestFilename() + filename := this.requestFilename() + if len(filename)> 0 { + return filename + } + + if this.locationContext != nil && len(this.locationContext.Root)> 0 { + return filepath.Clean(this.locationContext.Root + this.requestPath()) + } + + if len(this.root)> 0 { + return filepath.Clean(this.root + this.requestPath()) + } + + return "" case "scheme": return this.rawScheme case "serverProtocol", "proto": @@ -1161,7 +1195,7 @@ func (this *Request) Format(source string) string { case "bodyBytesSent": return fmt.Sprintf("%d", this.responseWriter.SentBodyBytes()) case "status": - return fmt.Sprintf("%d", this.responseWriter.StatusCode()) + return strconv.Itoa(this.responseWriter.StatusCode()) case "statusMessage": return http.StatusText(this.responseWriter.StatusCode()) case "timeISO8601": @@ -1191,7 +1225,12 @@ func (this *Request) Format(source string) string { case "serverName": return this.serverName case "serverPort": - return fmt.Sprintf("%d", this.requestServerPort()) + return strconv.Itoa(this.requestServerPort()) + case "documentRoot": + if this.locationContext != nil && len(this.locationContext.Root)> 0 { + return this.locationContext.Root + } + return this.root } dotIndex := strings.Index(varName, ".") From 97bbec4422362cd3e80a2475a613727d74590cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月13日 19:12:30 +0800 Subject: [PATCH 028/121] =?UTF-8?q?[proxy]=E4=BC=98=E5=8C=96JavascriptAPI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teastats/counter_value.go | 6 +++--- .../default/proxy/board/scripts/engine.go | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/teastats/counter_value.go b/teastats/counter_value.go index aa2ffb2..a10c568 100644 --- a/teastats/counter_value.go +++ b/teastats/counter_value.go @@ -2,7 +2,7 @@ package teastats // 数值增长型的统计值 type CounterValue struct { - Timestamp int64 `json:"timestamp"` - Params map[string]string `json:"params"` - Value map[string]interface{} `json:"value"` + Timestamp int64 `json:"timestamp"` // 时间戳 + Params map[string]string `json:"params"` // 参数,用来区分单个统计项内的不同的项目 + Value map[string]interface{} `json:"value"` // 数值 } diff --git a/teaweb/actions/default/proxy/board/scripts/engine.go b/teaweb/actions/default/proxy/board/scripts/engine.go index a62e95c..81ee3b5 100644 --- a/teaweb/actions/default/proxy/board/scripts/engine.go +++ b/teaweb/actions/default/proxy/board/scripts/engine.go @@ -83,6 +83,7 @@ func (this *Engine) SetContext(context *Context) { "isDown": backend.IsDown, "isBackup": backend.IsBackup, "address": backend.Address, + "code": backend.Code, } }), "locations": lists.Map(context.Server.Locations, func(k int, v interface{}) interface{} { @@ -116,6 +117,26 @@ func (this *Engine) SetContext(context *Context) { "root": location.Root, "index": location.Index, "headers": location.Headers, + "backends": lists.Map(location.Backends, func(k int, v interface{}) interface{} { + backend := v.(*teaconfigs.BackendConfig) + + if runningServer != nil { + runningBackend := runningServer.FindBackend(backend.Id) + if runningBackend != nil { + backend.IsDown = runningBackend.IsDown + } + } + + return map[string]interface{}{ + "isOn": backend.On, + "weight": backend.Weight, + "id": backend.Id, + "isDown": backend.IsDown, + "isBackup": backend.IsBackup, + "address": backend.Address, + "code": backend.Code, + } + }), } if location.Websocket != nil && location.Websocket.On { locationOptions["websocket"] = maps.Map{ From 08af23015f91be482ac26cbc112a9a681d0a0dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月14日 16:23:10 +0800 Subject: [PATCH 029/121] =?UTF-8?q?=E5=87=86=E5=A4=87v0.1.8.2=EF=BC=88v0.1?= =?UTF-8?q?.9=E7=9A=84=E9=A2=84=E8=A7=88=E7=89=88=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/agents/agent.go | 4 +- teaconst/const.go | 2 +- teautils/command_help.go | 94 +++++++++++++++++++++++++++++++ teautils/file.go | 11 ++++ teautils/logbuffer/buffer.go | 12 ++++ teautils/os.go | 105 ----------------------------------- teautils/os_test.go | 72 ------------------------ 7 files changed, 121 insertions(+), 179 deletions(-) create mode 100644 teautils/command_help.go create mode 100644 teautils/file.go delete mode 100644 teautils/os.go delete mode 100644 teautils/os_test.go diff --git a/teaconfigs/agents/agent.go b/teaconfigs/agents/agent.go index fc8fde8..a8afdb8 100644 --- a/teaconfigs/agents/agent.go +++ b/teaconfigs/agents/agent.go @@ -635,7 +635,9 @@ func (this *AgentConfig) FirstGroup() *Group { // 判断是否匹配关键词 func (this *AgentConfig) MatchKeyword(keyword string) (matched bool, name string, tags []string) { - if teautils.MatchKeyword(this.Name, keyword) || teautils.MatchKeyword(this.Host, keyword) { + if teautils.MatchKeyword(this.Name, keyword) || + teautils.MatchKeyword(this.Host, keyword) || + this.Id == keyword { matched = true name = this.Name if len(this.Host)> 0 { diff --git a/teaconst/const.go b/teaconst/const.go index 2ebf31f..2d5d4a2 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.8.1" + TeaVersion = "0.1.8.2" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 diff --git a/teautils/command_help.go b/teautils/command_help.go new file mode 100644 index 0000000..5466204 --- /dev/null +++ b/teautils/command_help.go @@ -0,0 +1,94 @@ +package teautils + +import ( + "fmt" + "strconv" +) + +// 命令帮助 +type CommandHelp struct { + product string + version string + usage string + options []*CommandHelpOption + appendStrings []string +} + +func NewCommandHelp() *CommandHelp { + return &CommandHelp{} +} + +type CommandHelpOption struct { + Code string + Description string +} + +// 产品 +func (this *CommandHelp) Product(product string) *CommandHelp { + this.product = product + return this +} + +// 版本 +func (this *CommandHelp) Version(version string) *CommandHelp { + this.version = version + return this +} + +// 使用方法 +func (this *CommandHelp) Usage(usage string) *CommandHelp { + this.usage = usage + return this +} + +// 选项 +func (this *CommandHelp) Option(code string, description string) *CommandHelp { + this.options = append(this.options, &CommandHelpOption{ + Code: code, + Description: description, + }) + return this +} + +// 附加内容 +func (this *CommandHelp) Append(appendString string) *CommandHelp { + this.appendStrings = append(this.appendStrings, appendString) + return this +} + +// 打印 +func (this *CommandHelp) Print() { + fmt.Println(this.product + " v" + this.version) + fmt.Println("Usage:", "\n "+this.usage) + + if len(this.options)> 0 { + fmt.Println("") + fmt.Println("Options:") + + spaces := 20 + max := 40 + for _, option := range this.options { + l := len(option.Code) + if l < max && l> spaces { + spaces = l + 4 + } + } + + for _, option := range this.options { + if len(option.Code)> max { + fmt.Println("") + fmt.Println(" " + option.Code) + option.Code = "" + } + + fmt.Printf(" %-"+strconv.Itoa(spaces)+"s%s\n", option.Code, ": "+option.Description) + } + } + + if len(this.appendStrings)> 0 { + fmt.Println("") + for _, s := range this.appendStrings { + fmt.Println(s) + } + } +} diff --git a/teautils/file.go b/teautils/file.go new file mode 100644 index 0000000..4e90b0c --- /dev/null +++ b/teautils/file.go @@ -0,0 +1,11 @@ +package teautils + +import ( + "github.com/iwind/TeaGo/Tea" + "path/filepath" +) + +// 临时文件 +func TmpFile(path string) string { + return filepath.Clean(Tea.Root + Tea.DS + "web" + Tea.DS + "tmp" + Tea.DS + path) +} diff --git a/teautils/logbuffer/buffer.go b/teautils/logbuffer/buffer.go index ae04f91..12092b1 100644 --- a/teautils/logbuffer/buffer.go +++ b/teautils/logbuffer/buffer.go @@ -101,3 +101,15 @@ func (this *Buffer) Read() (data []byte, err error) { func (this *Buffer) filename(index int) string { return this.prefix + "." + strconv.Itoa(index) + ".log" } + +// 关闭 +func (this *Buffer) Close() error { + var resultErr error + for _, file := range this.files { + err := file.Close() + if err != nil { + resultErr = err + } + } + return resultErr +} diff --git a/teautils/os.go b/teautils/os.go deleted file mode 100644 index cd8d06d..0000000 --- a/teautils/os.go +++ /dev/null @@ -1,105 +0,0 @@ -package teautils - -import ( - "github.com/fsnotify/fsnotify" - "github.com/iwind/TeaGo/Tea" - "github.com/iwind/TeaGo/logs" - "os" - "runtime" - "sync" - "time" -) - -// 文件统计Map -const ( - MaxWatchingFiles = 10240 -) - -var ( - fileStatMap = map[string]*FileInfo{} // path => os.FileInfo - fileStatLocker = &sync.RWMutex{} - fileWatcher, _ = fsnotify.NewWatcher() - fileStatEnabled = runtime.GOOS == "linux" || runtime.GOOS == "darwin" -) - -type FileInfo struct { - info os.FileInfo - accessTime int64 -} - -// 初始化 -func init() { - if fileStatEnabled && fileWatcher != nil { - go func() { - for event := range fileWatcher.Events { - logs.Println("[watcher]changed:", event.Op, event.Name) - - fileStatLocker.Lock() - delete(fileStatMap, event.Name) - fileStatLocker.Unlock() - } - }() - } -} - -// 临时文件 -func TmpFile(path string) string { - return Tea.Root + Tea.DS + "web" + Tea.DS + "tmp" + Tea.DS + path -} - -// 文件统计信息 -func StatFile(path string) (os.FileInfo, error) { - if fileWatcher == nil || !fileStatEnabled { - return os.Stat(path) - } - - fileStatLocker.RLock() - fileInfo, ok := fileStatMap[path] - fileStatLocker.RUnlock() - if ok { - fileInfo.accessTime = time.Now().Unix() - return fileInfo.info, nil - } - - stat, err := os.Stat(path) - if err != nil { - return stat, err - } - - fileStatLocker.Lock() - if len(fileStatMap)>= MaxWatchingFiles { - timestamp := time.Now().Unix() - hasDeleted := false - for k, v := range fileStatMap { - if timestamp-v.accessTime>= 1800 { // 1800 seconds - delete(fileStatMap, k) - _ = fileWatcher.Remove(k) - hasDeleted = true - } - } - - // no space left - if !hasDeleted { - fileStatLocker.Unlock() - return stat, err - } - } - fileStatLocker.Unlock() - - // watch file - err = fileWatcher.Add(path) - - if err != nil { - logs.Error(err) - _ = fileWatcher.Remove(path) - } else { - fileStatLocker.Lock() - fileStatMap[path] = &FileInfo{ - info: stat, - accessTime: time.Now().Unix(), - } - fileStatLocker.Unlock() - } - - return stat, nil -} diff --git a/teautils/os_test.go b/teautils/os_test.go deleted file mode 100644 index 131266b..0000000 --- a/teautils/os_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package teautils - -import ( - "fmt" - "github.com/TeaWeb/code/teatesting" - "os" - "testing" - "time" -) - -func TestStatFile(t *testing.T) { - t.Log(StatFile("os.go")) - t.Log(StatFile("os.go")) - - if !teatesting.IsGlobal() { - time.Sleep(10 * time.Second) - } - - t.Log(StatFile("os.go")) -} - -func TestStatFile2(t *testing.T) { - _, _ = StatFile("os.go") - _, _ = StatFile("os_test.go") - _, _ = StatFile("service.go") - _, _ = StatFile("string.go") - _, _ = StatFile("string_test.go") - - time.Sleep(1 * time.Second) - - _, _ = StatFile("string_test.go") -} - -func TestStatManyFiles(t *testing.T) { - if teatesting.IsGlobal() { - return - } - _, err := os.Stat("/tmp") - if err != nil { - return - } - for i := 0; i < 10000; i++ { - file := "/tmp/stat." + fmt.Sprintf("%d", i) + ".log" - fp, err := os.Create(file) - if err != nil { - t.Fatal(err) - } else { - _ = fp.Close() - } - - _, _ = StatFile(file) - } - - time.Sleep(30 * time.Second) - - for i := 0; i < 10000; i++ { - file := "/tmp/stat." + fmt.Sprintf("%d", i) + ".log" - _ = os.Remove(file) - } -} - -func BenchmarkStatFile(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = StatFile("os.go") - } -} - -func BenchmarkStatFile_Raw(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = os.Stat("os.go") - } -} From 30a17266743571a3ebe1a9c388dd11eb945f3623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月15日 15:48:47 +0800 Subject: [PATCH 030/121] =?UTF-8?q?=E7=AE=80=E5=8C=96Agent=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../default/agents/agentutils/agent_event.go | 89 +------------------ .../default/agents/agentutils/agent_queue.go | 61 +++++++++++++ .../default/agents/agentutils/agent_state.go | 29 ++++-- .../actions/default/agents/agentutils/init.go | 4 +- .../actions/default/agents/agentutils/menu.go | 8 +- teaweb/actions/default/agents/apps/addTask.go | 20 +++-- .../default/agents/apps/execItemSource.go | 6 +- .../actions/default/agents/apps/itemDetail.go | 4 +- teaweb/actions/default/agents/apps/itemOn.go | 6 +- teaweb/actions/default/agents/apps/off.go | 6 +- .../actions/default/agents/groups/detail.go | 4 +- .../actions/default/agents/settings/index.go | 4 +- teaweb/actions/default/api/agent/pull.go | 38 ++++---- 13 files changed, 140 insertions(+), 139 deletions(-) create mode 100644 teaweb/actions/default/agents/agentutils/agent_queue.go diff --git a/teaweb/actions/default/agents/agentutils/agent_event.go b/teaweb/actions/default/agents/agentutils/agent_event.go index 1a18db2..bfae03c 100644 --- a/teaweb/actions/default/agents/agentutils/agent_event.go +++ b/teaweb/actions/default/agents/agentutils/agent_event.go @@ -1,102 +1,15 @@ package agentutils -import ( - "sync" -) - // Agent事件 type AgentEvent struct { Name string `json:"name"` Data interface{} `json:"data"` } -var eventQueueMap = map[string]map[chan *AgentEvent]*AgentState{} // agentId => { chan => State } -var eventQueueLocker = sync.Mutex{} - -// 新Agent事件 +// 获取新对象 func NewAgentEvent(name string, data interface{}) *AgentEvent { return &AgentEvent{ Name: name, Data: data, } } - -// 等待Agent事件 -func WaitAgentQueue(agentId string, agentVersion string, osName string, speed float64, ip string, c chan *AgentEvent) { - eventQueueLocker.Lock() - defer eventQueueLocker.Unlock() - _, found := eventQueueMap[agentId] - if found { - eventQueueMap[agentId][c] = &AgentState{ - Version: agentVersion, - OsName: osName, - Speed: speed, - IP: ip, - IsAvailable: true, - } - } else { - eventQueueMap[agentId] = map[chan *AgentEvent]*AgentState{ - c: { - Version: agentVersion, - OsName: osName, - Speed: speed, - IP: ip, - IsAvailable: true, - }, - } - } -} - -// 禁用Channel -func DisableAgentQueue(agentId string, c chan *AgentEvent) { - eventQueueLocker.Lock() - defer eventQueueLocker.Unlock() - m, found := eventQueueMap[agentId] - if found { - state, ok := m[c] - if ok { - state.IsAvailable = false - } - } -} - -// 删除Agent -func RemoveAgentQueue(agentId string, c chan *AgentEvent) { - eventQueueLocker.Lock() - defer eventQueueLocker.Unlock() - _, ok := eventQueueMap[agentId] - if ok { - delete(eventQueueMap[agentId], c) - - if len(eventQueueMap[agentId]) == 0 { - delete(eventQueueMap, agentId) - } - } -} - -// 发送Agent事件 -func PostAgentEvent(agentId string, event *AgentEvent) { - eventQueueLocker.Lock() - defer eventQueueLocker.Unlock() - m, found := eventQueueMap[agentId] - if found { - for c, state := range m { - if state.IsAvailable { - c <- event - } - } - } -} - -// 检查Agent是否正在运行 -func CheckAgentIsWaiting(agentId string) (state *AgentState, isWaiting bool) { - eventQueueLocker.Lock() - defer eventQueueLocker.Unlock() - queue, _ := eventQueueMap[agentId] - if len(queue)> 0 { - for _, v := range queue { - return v, true - } - } - return nil, false -} diff --git a/teaweb/actions/default/agents/agentutils/agent_queue.go b/teaweb/actions/default/agents/agentutils/agent_queue.go new file mode 100644 index 0000000..d9d06de --- /dev/null +++ b/teaweb/actions/default/agents/agentutils/agent_queue.go @@ -0,0 +1,61 @@ +package agentutils + +import ( + "sync" + "time" +) + +var agentQueueMap = map[string]*AgentQueue{} // agentId => queue +var agentQueueLocker = &sync.Mutex{} + +type AgentQueue struct { + c chan *AgentEvent +} + +func NewAgentQueue() *AgentQueue { + return &AgentQueue{ + c: make(chan *AgentEvent, 32), + } +} + +// 通知事件 +func PostAgentEvent(agentId string, event *AgentEvent) { + state := FindAgentState(agentId) + if !state.IsActive { + return + } + + agentQueueLocker.Lock() + queue, ok := agentQueueMap[agentId] + if !ok { + agentQueueLocker.Unlock() + return + } + agentQueueLocker.Unlock() + select { + case queue.c <- event: + default: + } +} + +// 等待事件 +func Wait(agentId string) *AgentEvent { + agentQueueLocker.Lock() + queue, ok := agentQueueMap[agentId] + if !ok { + queue = NewAgentQueue() + agentQueueMap[agentId] = queue + } + agentQueueLocker.Unlock() + + timer := time.NewTimer(59 * time.Second) + + select { + case event := <-queue.c: + timer.Stop() + return event + case <-timer.c: + } + + return nil +} diff --git a/teaweb/actions/default/agents/agentutils/agent_state.go b/teaweb/actions/default/agents/agentutils/agent_state.go index c7f225d..2d1076c 100644 --- a/teaweb/actions/default/agents/agentutils/agent_state.go +++ b/teaweb/actions/default/agents/agentutils/agent_state.go @@ -1,10 +1,29 @@ package agentutils +import "sync" + +var agentStateMap = map[string]*AgentState{} //agentId => state +var agentStateLocker = sync.Mutex{} + // Agent状态 type AgentState struct { - Version string // 版本号 - OsName string // 操作系统 - Speed float64 // 连接速度,ms - IP string // IP地址 - IsAvailable bool // 是否可用 + Version string // 版本号 + OsName string // 操作系统 + Speed float64 // 连接速度,ms + IP string // IP地址 + IsActive bool // 是否在线 +} + +// 查找Agent状态 +func FindAgentState(agentId string) *AgentState { + agentStateLocker.Lock() + defer agentStateLocker.Unlock() + + state, ok := agentStateMap[agentId] + if ok { + return state + } + state = &AgentState{} + agentStateMap[agentId] = state + return state } diff --git a/teaweb/actions/default/agents/agentutils/init.go b/teaweb/actions/default/agents/agentutils/init.go index ad47ff9..131f5bc 100644 --- a/teaweb/actions/default/agents/agentutils/init.go +++ b/teaweb/actions/default/agents/agentutils/init.go @@ -39,8 +39,8 @@ func checkConnecting() { runtimeAgent := FindAgentRuntime(agent) // 监控连通性 - _, isWaiting := CheckAgentIsWaiting(agent.Id) - if !isWaiting { + state := FindAgentState(agent.Id) + if !state.IsActive { runtimeAgent.CountDisconnections++ if runtimeAgent.CountDisconnections> 0 && runtimeAgent.CountDisconnections%maxDisconnections == 0 { // 失去连接 N 次则提醒 diff --git a/teaweb/actions/default/agents/agentutils/menu.go b/teaweb/actions/default/agents/agentutils/menu.go index 6a9ad81..bf78999 100644 --- a/teaweb/actions/default/agents/agentutils/menu.go +++ b/teaweb/actions/default/agents/agentutils/menu.go @@ -47,9 +47,9 @@ func AddTabbar(actionWrapper actions.ActionWrapper) { } defaultGroup := agents.SharedGroupList().FindGroup("default") - state, isWaiting := CheckAgentIsWaiting("local") + state := FindAgentState("local") menu := menuGroup.FindMenu("default", defaultGroup.Name+topSubName) - if isWaiting { + if state.IsActive { subName := "已连接" if state != nil && len(state.OsName)> 0 { subName = state.OsName @@ -67,7 +67,7 @@ func AddTabbar(actionWrapper actions.ActionWrapper) { counterMapping := map[string]int{} // groupId => count maxCount := 50 for _, agent := range allAgents { - state, isWaiting := CheckAgentIsWaiting(agent.Id) + state := FindAgentState(agent.Id) var menu *utils.Menu = nil if len(agent.GroupIds)> 0 { @@ -111,7 +111,7 @@ func AddTabbar(actionWrapper actions.ActionWrapper) { } } - if isWaiting { + if state.IsActive { subName := "已连接" if state != nil && len(state.OsName)> 0 { subName = state.OsName diff --git a/teaweb/actions/default/agents/apps/addTask.go b/teaweb/actions/default/agents/apps/addTask.go index c28b95b..2c6b679 100644 --- a/teaweb/actions/default/agents/apps/addTask.go +++ b/teaweb/actions/default/agents/apps/addTask.go @@ -7,6 +7,7 @@ import ( "github.com/TeaWeb/code/teaweb/actions/default/agents/agentutils" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" + "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/types" "reflect" @@ -143,19 +144,19 @@ func (this *AddTaskAction) RunPost(params struct { switch timeTypeString { case "second": - schedule.AddSecondRanges(ranges ...) + schedule.AddSecondRanges(ranges...) case "minute": - schedule.AddMinuteRanges(ranges ...) + schedule.AddMinuteRanges(ranges...) case "hour": - schedule.AddHourRanges(ranges ...) + schedule.AddHourRanges(ranges...) case "day": - schedule.AddDayRanges(ranges ...) + schedule.AddDayRanges(ranges...) case "month": - schedule.AddMonthRanges(ranges ...) + schedule.AddMonthRanges(ranges...) case "year": - schedule.AddYearRanges(ranges ...) + schedule.AddYearRanges(ranges...) case "weekDay": - schedule.AddWeekDayRanges(ranges ...) + schedule.AddWeekDayRanges(ranges...) } } @@ -180,10 +181,13 @@ func (this *AddTaskAction) RunPost(params struct { })) if app.IsSharedWithGroup { - agentutils.SyncApp(agent.Id, agent.GroupIds, app, agentutils.NewAgentEvent("ADD_TASK", maps.Map{ + err := agentutils.SyncApp(agent.Id, agent.GroupIds, app, agentutils.NewAgentEvent("ADD_TASK", maps.Map{ "appId": app.Id, "taskId": task.Id, }), nil) + if err != nil { + logs.Error(err) + } } this.Success() diff --git a/teaweb/actions/default/agents/apps/execItemSource.go b/teaweb/actions/default/agents/apps/execItemSource.go index 0c85a49..60327bf 100644 --- a/teaweb/actions/default/agents/apps/execItemSource.go +++ b/teaweb/actions/default/agents/apps/execItemSource.go @@ -4,6 +4,7 @@ import ( "github.com/TeaWeb/code/teaconfigs/agents" "github.com/TeaWeb/code/teaweb/actions/default/agents/agentutils" "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" ) @@ -38,13 +39,16 @@ func (this *ExecItemSourceAction) Run(params struct { // 同步 if app.IsSharedWithGroup { - agentutils.SyncAppEvent(agent.Id, agent.GroupIds, app, &agentutils.AgentEvent{ + err := agentutils.SyncAppEvent(agent.Id, agent.GroupIds, app, &agentutils.AgentEvent{ Name: "RUN_ITEM", Data: maps.Map{ "appId": app.Id, "itemId": params.ItemId, }, }) + if err != nil { + logs.Error(err) + } } this.Success() diff --git a/teaweb/actions/default/agents/apps/itemDetail.go b/teaweb/actions/default/agents/apps/itemDetail.go index 999fb63..5fd54c8 100644 --- a/teaweb/actions/default/agents/apps/itemDetail.go +++ b/teaweb/actions/default/agents/apps/itemDetail.go @@ -67,8 +67,8 @@ func (this *ItemDetailAction) Run(params struct { // 是否在线 this.Data["isWaiting"] = false if agent.On && app.On && item.On { - state, isWaiting := agentutils.CheckAgentIsWaiting(params.AgentId) - if state != nil && isWaiting { + state := agentutils.FindAgentState(params.AgentId) + if state.IsActive { if stringutil.VersionCompare(state.Version, "0.1")> 0 { this.Data["isWaiting"] = true } diff --git a/teaweb/actions/default/agents/apps/itemOn.go b/teaweb/actions/default/agents/apps/itemOn.go index e15a4a4..d296297 100644 --- a/teaweb/actions/default/agents/apps/itemOn.go +++ b/teaweb/actions/default/agents/apps/itemOn.go @@ -4,6 +4,7 @@ import ( "github.com/TeaWeb/code/teaconfigs/agents" "github.com/TeaWeb/code/teaweb/actions/default/agents/agentutils" "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" ) @@ -43,10 +44,13 @@ func (this *ItemOnAction) Run(params struct { })) if app.IsSharedWithGroup { - agentutils.SyncApp(agent.Id, agent.GroupIds, app, agentutils.NewAgentEvent("UPDATE_ITEM", maps.Map{ + err := agentutils.SyncApp(agent.Id, agent.GroupIds, app, agentutils.NewAgentEvent("UPDATE_ITEM", maps.Map{ "appId": app.Id, "itemId": params.ItemId, }), nil) + if err != nil { + logs.Error(err) + } } this.Success() diff --git a/teaweb/actions/default/agents/apps/off.go b/teaweb/actions/default/agents/apps/off.go index fa4c1a0..cd467ca 100644 --- a/teaweb/actions/default/agents/apps/off.go +++ b/teaweb/actions/default/agents/apps/off.go @@ -4,6 +4,7 @@ import ( "github.com/TeaWeb/code/teaconfigs/agents" "github.com/TeaWeb/code/teaweb/actions/default/agents/agentutils" "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" ) @@ -37,9 +38,12 @@ func (this *OffAction) Run(params struct { // 同步 if app.IsSharedWithGroup { - agentutils.SyncApp(agent.Id, agent.GroupIds, app, agentutils.NewAgentEvent("UPDATE_APP", maps.Map{ + err := agentutils.SyncApp(agent.Id, agent.GroupIds, app, agentutils.NewAgentEvent("UPDATE_APP", maps.Map{ "appId": app.Id, }), nil) + if err != nil { + logs.Error(err) + } } this.Success() diff --git a/teaweb/actions/default/agents/groups/detail.go b/teaweb/actions/default/agents/groups/detail.go index 68fd3db..62ab7d9 100644 --- a/teaweb/actions/default/agents/groups/detail.go +++ b/teaweb/actions/default/agents/groups/detail.go @@ -32,13 +32,13 @@ func (this *DetailAction) Run(params struct { allAgents := agentList.FindAllAgents() for _, a := range allAgents { if a.BelongsToGroup(params.GroupId) { - _, isWaiting := agentutils.CheckAgentIsWaiting(a.Id) + state := agentutils.FindAgentState(a.Id) groupAgents = append(groupAgents, maps.Map{ "on": a.On, "id": a.Id, "name": a.Name, "host": a.Host, - "isWaiting": isWaiting, + "isWaiting": state.IsActive, }) } } diff --git a/teaweb/actions/default/agents/settings/index.go b/teaweb/actions/default/agents/settings/index.go index 8e34a3b..1050d65 100644 --- a/teaweb/actions/default/agents/settings/index.go +++ b/teaweb/actions/default/agents/settings/index.go @@ -20,8 +20,8 @@ func (this *IndexAction) Run(params struct { this.Fail("找不到Agent") } - state, isWaiting := agentutils.CheckAgentIsWaiting(agent.Id) - if isWaiting { + state := agentutils.FindAgentState(agent.Id) + if state.IsActive { this.Data["agentVersion"] = state.Version this.Data["agentSpeed"] = state.Speed this.Data["agentIP"] = state.IP diff --git a/teaweb/actions/default/api/agent/pull.go b/teaweb/actions/default/api/agent/pull.go index 9e20c65..d4d351c 100644 --- a/teaweb/actions/default/api/agent/pull.go +++ b/teaweb/actions/default/api/agent/pull.go @@ -34,37 +34,29 @@ func (this *PullAction) Run(params struct{}) { } } - c := make(chan *agentutils.AgentEvent) - agentutils.WaitAgentQueue(agentId, agentVersion, osName, speed, this.RequestRemoteIP(), c) + state := agentutils.FindAgentState(agentId) + state.IsActive = true + state.Version = agentVersion + state.OsName = osName + state.Speed = speed + state.IP = this.RequestRemoteIP() - events := []*agentutils.AgentEvent{} + isDone := false - // 监控是否中断请求 go func() { <-this.request.context().done() - - if len(events)> 0 { - agentutils.DisableAgentQueue(agentId, c) - go func() { - time.Sleep(1 * time.Second) - agentutils.RemoveAgentQueue(agentId, c) - }() - } else { - agentutils.RemoveAgentQueue(agentId, c) + if !isDone { + state.IsActive = false } - - // 关闭channel - c <- nil - close(c) }() - for { - event := <-c - if event != nil { - events = append(events, event) - } + event := agentutils.Wait(agentId) + state.IsActive = false + isDone = true - break + events := []*agentutils.AgentEvent{} + if event != nil { + events = append(events, event) } this.Data["events"] = events From c05f6205e51ba1a2cb584a283677c4b6d8a55d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月25日 10:01:26 +0800 Subject: [PATCH 031/121] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconst/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/teaconst/const.go b/teaconst/const.go index 2d5d4a2..9ad2912 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.8.2" + TeaVersion = "0.1.9" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 From 7d8cf2707e08d1696bfbeebf3bc747088857dd5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月25日 16:32:23 +0800 Subject: [PATCH 032/121] =?UTF-8?q?[proxy]=E8=A7=A3=E5=86=B3=E5=9B=A0?= =?UTF-8?q?=E4=B8=BAURL=E4=B8=AD=E5=8C=85=E5=90=AB=E5=A4=9A=E4=B8=AA/?= =?UTF-8?q?=E8=80=8C=E8=87=AA=E5=8A=A8=E8=B7=B3=E8=BD=AC=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/http_serve_mux.go | 18 ++++++++++++++++++ teaproxy/listener.go | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 teaproxy/http_serve_mux.go diff --git a/teaproxy/http_serve_mux.go b/teaproxy/http_serve_mux.go new file mode 100644 index 0000000..936999c --- /dev/null +++ b/teaproxy/http_serve_mux.go @@ -0,0 +1,18 @@ +package teaproxy + +import ( + "net/http" + "path/filepath" +) + +// 自定义ServeMux +type HTTPServeMux struct { + http.ServeMux +} + +func (this *HTTPServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // 解决因为URL中包含多个/而自动跳转的问题 + r.URL.Path = filepath.Clean(r.URL.Path) + + this.ServeMux.ServeHTTP(w, r) +} diff --git a/teaproxy/listener.go b/teaproxy/listener.go index 4ee7555..c64ebc5 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -284,7 +284,7 @@ func (this *Listener) startHTTPServer() error { var err error // 如果没启动,则启动 - httpHandler := http.NewServeMux() + httpHandler := new(HTTPServeMux) httpHandler.HandleFunc("/", func(writer http.ResponseWriter, req *http.Request) { // QPS计算 atomic.AddInt32(&qps, 1) From e21d38cbf1cb5e8b70d8c933ed371c45fe508080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月25日 16:35:16 +0800 Subject: [PATCH 033/121] =?UTF-8?q?[proxy]=E5=8E=BB=E9=99=A4=E9=87=8D?= =?UTF-8?q?=E5=86=99=E8=A7=84=E5=88=99=E4=B8=AD=E4=BD=BF=E7=94=A8=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E8=BD=AC=E5=8F=91=E6=97=B6=E8=B7=AF=E5=BE=84=E4=B8=AD?= =?UTF-8?q?=E5=A4=9A=E4=BD=99=E7=9A=84=E6=96=9C=E6=9D=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/proxy/rewrite/add.go | 3 ++- teaweb/actions/default/proxy/rewrite/update.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/teaweb/actions/default/proxy/rewrite/add.go b/teaweb/actions/default/proxy/rewrite/add.go index 684924f..a46c16f 100644 --- a/teaweb/actions/default/proxy/rewrite/add.go +++ b/teaweb/actions/default/proxy/rewrite/add.go @@ -9,6 +9,7 @@ import ( "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/maps" "regexp" + "strings" ) type AddAction actions.Action @@ -137,7 +138,7 @@ func (this *AddAction) RunPost(params struct { if params.TargetType == "url" { rewriteRule.Replace = params.Replace } else { - rewriteRule.Replace = "proxy://" + params.ProxyId + params.Replace + rewriteRule.Replace = "proxy://" + params.ProxyId + "/" + strings.TrimLeft(params.Replace, "/") } if len(params.RedirectMode)> 0 { rewriteRule.AddFlag(params.RedirectMode, nil) diff --git a/teaweb/actions/default/proxy/rewrite/update.go b/teaweb/actions/default/proxy/rewrite/update.go index a963baf..90eefd7 100644 --- a/teaweb/actions/default/proxy/rewrite/update.go +++ b/teaweb/actions/default/proxy/rewrite/update.go @@ -9,6 +9,7 @@ import ( "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/maps" "regexp" + "strings" ) type UpdateAction actions.Action @@ -168,7 +169,7 @@ func (this *UpdateAction) RunPost(params struct { if params.TargetType == "url" { rewriteRule.Replace = params.Replace } else { - rewriteRule.Replace = "proxy://" + params.ProxyId + "/" + params.Replace + rewriteRule.Replace = "proxy://" + params.ProxyId + "/" + strings.TrimLeft(params.Replace, "/") } rewriteRule.Flags = []string{} rewriteRule.FlagOptions = maps.Map{} From 922ba71be31708f8a6bc59bb5daa09fbe8c2c454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月26日 16:08:04 +0800 Subject: [PATCH 034/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8D=E4=B8=80?= =?UTF-8?q?=E4=B8=AAgzip=E5=92=8C=E7=BC=93=E5=AD=98=E5=86=B2=E7=AA=81?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconst/const.go | 2 +- teaproxy/request.go | 9 ++++++++- teaproxy/request_backend.go | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/teaconst/const.go b/teaconst/const.go index 9ad2912..a67dfb3 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.9" + TeaVersion = "0.1.9.1" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 diff --git a/teaproxy/request.go b/teaproxy/request.go index edf5914..344f197 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -732,12 +732,19 @@ func (this *Request) call(writer *ResponseWriter) error { } // gzip压缩 + hasGzip := false if this.gzip != nil && this.gzip.Level> 0 && this.acceptGzipEncoding() { + hasGzip = true writer.Gzip(this.gzip) - defer writer.Close() } err := this.callBegin(writer) + + // 在结束之前关闭gzip以便能够获取完整的body + if hasGzip { + writer.Close() + } + this.callEnd(writer) return err } diff --git a/teaproxy/request_backend.go b/teaproxy/request_backend.go index fe249fb..8f78a82 100644 --- a/teaproxy/request_backend.go +++ b/teaproxy/request_backend.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "net/url" + "path/filepath" "strings" "time" ) @@ -60,7 +61,7 @@ func (this *Request) callBackend(writer *ResponseWriter) error { // new uri if this.backend.HasRequestURI() { - uri := this.Format(this.backend.RequestPath()) + uri := filepath.Clean(this.Format(this.backend.RequestPath())) u, err := url.ParseRequestURI(uri) if err == nil { From 0fee13a2999df933bfbb0e1857756f7797e6756e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月29日 15:56:30 +0800 Subject: [PATCH 035/121] =?UTF-8?q?[proxy]=E8=B7=AF=E5=BE=84=E8=A7=84?= =?UTF-8?q?=E5=88=99=E5=A2=9E=E5=8A=A0"=E6=98=AF=E5=90=A6=E7=BB=88?= =?UTF-8?q?=E6=AD=A2=E5=BE=80=E4=B8=8B=E5=8C=B9=E9=85=8D"=E9=80=89?= =?UTF-8?q?=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/location.go | 2 ++ teaproxy/request.go | 5 +++++ teaweb/actions/default/proxy/locations/add.go | 12 ++++++++---- teaweb/actions/default/proxy/locations/detail.go | 1 + teaweb/actions/default/proxy/locations/index.go | 7 ++++++- teaweb/actions/default/proxy/locations/update.go | 13 +++++++++---- 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/teaconfigs/location.go b/teaconfigs/location.go index 00f354a..b1cdf07 100644 --- a/teaconfigs/location.go +++ b/teaconfigs/location.go @@ -70,6 +70,8 @@ type LocationConfig struct { // - cond ${requestPath} regexp .*\.png Cond []*shared.RequestCond `yaml:"cond" json:"cond"` + IsBreak bool `yaml:"isBreak" json:"isBreak"` // 终止向下解析 + Pages []*PageConfig `yaml:"pages" json:"pages"` // 特殊页 Shutdown *ShutdownConfig `yaml:"shutdown" json:"shutdown"` // 关闭页 ShutdownPageOn1 bool `yaml:"shutdownPageOn" json:"shutdownPageOn"` // deprecated: v0.1.8, 是否开启临时关闭页面 diff --git a/teaproxy/request.go b/teaproxy/request.go index 344f197..8936c85 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -558,6 +558,11 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b this.locationContext = nil return nil } + + // break + if location.IsBreak { + break + } } this.locationContext = nil } diff --git a/teaweb/actions/default/proxy/locations/add.go b/teaweb/actions/default/proxy/locations/add.go index 4bda160..96a647e 100644 --- a/teaweb/actions/default/proxy/locations/add.go +++ b/teaweb/actions/default/proxy/locations/add.go @@ -66,10 +66,13 @@ func (this *AddAction) Run(params struct { // 保存提交 func (this *AddAction) RunPost(params struct { - ServerId string - Name string - Pattern string - PatternType int + ServerId string + Name string + Pattern string + PatternType int + + IsBreak bool + Root string Charset string Index []string @@ -129,6 +132,7 @@ func (this *AddAction) RunPost(params struct { location.SetPattern(params.Pattern, params.PatternType, params.IsCaseInsensitive, params.IsReverse) location.On = params.On + location.IsBreak = params.IsBreak location.CacheOn = true location.Name = params.Name location.Root = params.Root diff --git a/teaweb/actions/default/proxy/locations/detail.go b/teaweb/actions/default/proxy/locations/detail.go index a53ef60..24f4533 100644 --- a/teaweb/actions/default/proxy/locations/detail.go +++ b/teaweb/actions/default/proxy/locations/detail.go @@ -28,6 +28,7 @@ func (this *DetailAction) Run(params struct { "type": location.PatternType(), "pattern": location.PatternString(), "name": location.Name, + "isBreak": location.IsBreak, "caseInsensitive": location.IsCaseInsensitive(), "reverse": location.IsReverse(), "root": location.Root, diff --git a/teaweb/actions/default/proxy/locations/index.go b/teaweb/actions/default/proxy/locations/index.go index 767da9c..bb83f54 100644 --- a/teaweb/actions/default/proxy/locations/index.go +++ b/teaweb/actions/default/proxy/locations/index.go @@ -3,6 +3,7 @@ package locations import ( "github.com/TeaWeb/code/teaconfigs" "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" ) @@ -22,7 +23,10 @@ func (this *IndexAction) Run(params struct { locations := []maps.Map{} for _, location := range server.Locations { - location.Validate() + err := location.Validate() + if err != nil { + logs.Error(err) + } locations = append(locations, maps.Map{ "on": location.On, "id": location.Id, @@ -30,6 +34,7 @@ func (this *IndexAction) Run(params struct { "type": location.PatternType(), "pattern": location.PatternString(), "patternTypeName": teaconfigs.FindLocationPatternTypeName(location.PatternType()), + "isBreak": location.IsBreak, "isCaseInsensitive": location.IsCaseInsensitive(), "isReverse": location.IsReverse(), "rewrite": location.Rewrite, diff --git a/teaweb/actions/default/proxy/locations/update.go b/teaweb/actions/default/proxy/locations/update.go index d14342b..71d415b 100644 --- a/teaweb/actions/default/proxy/locations/update.go +++ b/teaweb/actions/default/proxy/locations/update.go @@ -60,6 +60,7 @@ func (this *UpdateAction) Run(params struct { "pattern": location.PatternString(), "type": location.PatternType(), "name": location.Name, + "isBreak": location.IsBreak, "isReverse": location.IsReverse(), "isCaseInsensitive": location.IsCaseInsensitive(), "root": location.Root, @@ -104,10 +105,13 @@ func (this *UpdateAction) Run(params struct { // 保存修改 func (this *UpdateAction) RunPost(params struct { - ServerId string - LocationId string - Pattern string - PatternType int + ServerId string + LocationId string + Pattern string + PatternType int + + IsBreak bool + Name string Root string URLPrefix string `alias:"urlPrefix"` @@ -170,6 +174,7 @@ func (this *UpdateAction) RunPost(params struct { location.SetPattern(params.Pattern, params.PatternType, params.IsCaseInsensitive, params.IsReverse) location.On = params.On + location.IsBreak = params.IsBreak location.Name = params.Name location.Root = params.Root location.URLPrefix = params.URLPrefix From 0cbeafb03c0369d3c147abab5e9b6b090567d7fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年11月30日 20:26:14 +0800 Subject: [PATCH 036/121] =?UTF-8?q?[proxy]=E5=A6=82=E6=9E=9C=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=E8=AF=B7=E6=B1=82=E7=9A=84Host=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA=EF=BC=8C=E5=88=99=E8=87=AA=E5=8A=A8=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E4=B8=BA=E5=BD=93=E5=89=8D=E8=BF=9E=E6=8E=A5=E7=9A=84=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E5=99=A8=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/listener.go | 12 ++++++++++++ teaproxy/request_backend.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/teaproxy/listener.go b/teaproxy/listener.go index c64ebc5..ac9d124 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -389,6 +389,18 @@ func (this *Listener) handleHTTP(writer http.ResponseWriter, rawRequest *http.Re // 域名 reqHost := rawRequest.Host + + // 防止空Host + if len(reqHost) == 0 { + ctx := rawRequest.Context() + if ctx != nil { + addr := ctx.Value(http.LocalAddrContextKey) + if addr != nil { + reqHost = addr.(net.Addr).String() + } + } + } + domain, _, err := net.SplitHostPort(reqHost) if err != nil { domain = reqHost diff --git a/teaproxy/request_backend.go b/teaproxy/request_backend.go index 8f78a82..b897bb3 100644 --- a/teaproxy/request_backend.go +++ b/teaproxy/request_backend.go @@ -104,7 +104,7 @@ func (this *Request) callBackend(writer *ResponseWriter) error { } // 支持修改Host - if header.Name == "Host" { + if header.Name == "Host" && len(header.Value)> 0 { this.raw.Host = header.Value } } From 81fbe2d4f76fb3ea7982e6caad85b68ef043b0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Sun, 1 Dec 2019 14:39:40 +0800 Subject: [PATCH 037/121] =?UTF-8?q?[proxy]=E9=98=B2=E6=AD=A2=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=E6=9C=8D=E5=8A=A1=E5=99=A8=E5=BD=93=E5=89=8D=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E6=95=B0=E5=B0=8F=E4=BA=8E0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/backend.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/teaconfigs/backend.go b/teaconfigs/backend.go index 2ea2967..3c4a9d3 100644 --- a/teaconfigs/backend.go +++ b/teaconfigs/backend.go @@ -223,6 +223,9 @@ func (this *BackendConfig) DecreaseConn() int32 { if this.nextBackend != nil { return this.nextBackend.DecreaseConn() } + if this.CurrentConns == 0 { + return 0 + } return atomic.AddInt32(&this.CurrentConns, -1) } @@ -447,7 +450,7 @@ func (this *BackendConfig) CloneState(oldBackend *BackendConfig) { this.IsDown = oldBackend.IsDown this.DownTime = oldBackend.DownTime this.CurrentFails = oldBackend.CurrentFails - this.CurrentConns = oldBackend.CurrentConns + atomic.StoreInt32(&this.CurrentConns, oldBackend.CurrentConns) } // 获取唯一ID From 09e497227d160eff752d79e0510d6dd95e69f181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Mon, 2 Dec 2019 17:40:46 +0800 Subject: [PATCH 038/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8D=E5=9C=A8Windo?= =?UTF-8?q?ws=E4=B8=8AURL=E7=89=B9=E6=AE=8A=E5=AD=97=E7=AC=A6=E4=B8=8D?= =?UTF-8?q?=E8=83=BD=E8=AE=BF=E9=97=AE=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconst/const.go | 2 +- teaproxy/http_serve_mux.go | 3 +-- teaproxy/request_backend.go | 6 ++---- teaproxy/utils.go | 23 +++++++++++++++++++++++ teaproxy/utils_test.go | 23 +++++++++++++++++++++++ 5 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 teaproxy/utils.go create mode 100644 teaproxy/utils_test.go diff --git a/teaconst/const.go b/teaconst/const.go index a67dfb3..5036f3a 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.9.1" + TeaVersion = "0.1.9.2" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 diff --git a/teaproxy/http_serve_mux.go b/teaproxy/http_serve_mux.go index 936999c..cd3918b 100644 --- a/teaproxy/http_serve_mux.go +++ b/teaproxy/http_serve_mux.go @@ -2,7 +2,6 @@ package teaproxy import ( "net/http" - "path/filepath" ) // 自定义ServeMux @@ -12,7 +11,7 @@ type HTTPServeMux struct { func (this *HTTPServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { // 解决因为URL中包含多个/而自动跳转的问题 - r.URL.Path = filepath.Clean(r.URL.Path) + r.URL.Path = CleanPath(r.URL.Path) this.ServeMux.ServeHTTP(w, r) } diff --git a/teaproxy/request_backend.go b/teaproxy/request_backend.go index b897bb3..825eaac 100644 --- a/teaproxy/request_backend.go +++ b/teaproxy/request_backend.go @@ -9,7 +9,6 @@ import ( "io" "net/http" "net/url" - "path/filepath" "strings" "time" ) @@ -61,11 +60,10 @@ func (this *Request) callBackend(writer *ResponseWriter) error { // new uri if this.backend.HasRequestURI() { - uri := filepath.Clean(this.Format(this.backend.RequestPath())) - + uri := this.Format(this.backend.RequestPath()) u, err := url.ParseRequestURI(uri) if err == nil { - this.raw.URL.Path = u.Path + this.raw.URL.Path = CleanPath(u.Path) this.raw.URL.RawQuery = u.RawQuery args := this.Format(this.backend.RequestArgs()) diff --git a/teaproxy/utils.go b/teaproxy/utils.go new file mode 100644 index 0000000..557deef --- /dev/null +++ b/teaproxy/utils.go @@ -0,0 +1,23 @@ +package teaproxy + +// 清理Path中的多于信息 +func CleanPath(path string) string { + l := len(path) + if l == 0 { + return "/" + } + result := []byte{'/'} + isSlash := true + for i := 0; i < l; i++ { + if path[i] == '\\' || path[i] == '/' { + if !isSlash { + isSlash = true + result = append(result, '/') + } + } else { + isSlash = false + result = append(result, path[i]) + } + } + return string(result) +} diff --git a/teaproxy/utils_test.go b/teaproxy/utils_test.go new file mode 100644 index 0000000..1df01d5 --- /dev/null +++ b/teaproxy/utils_test.go @@ -0,0 +1,23 @@ +package teaproxy + +import ( + "github.com/iwind/TeaGo/assert" + "testing" +) + +func TestCleanPath(t *testing.T) { + a := assert.NewAssertion(t) + + a.IsTrue(CleanPath("") == "/") + a.IsTrue(CleanPath("/hello/world") == "/hello/world") + a.IsTrue(CleanPath("\\hello\\world") == "/hello/world") + a.IsTrue(CleanPath("/\\hello\\//world") == "/hello/world") + a.IsTrue(CleanPath("hello/world") == "/hello/world") + a.IsTrue(CleanPath("/hello////world") == "/hello/world") +} + +func BenchmarkCleanPath(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = CleanPath("/hello///world/very/long/very//long") + } +} From f0edbd42339dae6d91ec83d89926414c0a3d29fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Wed, 4 Dec 2019 11:39:15 +0800 Subject: [PATCH 039/121] =?UTF-8?q?[proxy]TCP=E5=90=8E=E7=AB=AF=E9=87=8D?= =?UTF-8?q?=E6=96=B0=E8=BF=9E=E6=8E=A5=E6=88=90=E5=8A=9F=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E9=87=8D=E7=BD=AE=E9=94=99=E8=AF=AF=E6=AC=A1=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/tcp_client.go | 3 +++ teaproxy/utils.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/teaproxy/tcp_client.go b/teaproxy/tcp_client.go index 04d35fe..24de0a7 100644 --- a/teaproxy/tcp_client.go +++ b/teaproxy/tcp_client.go @@ -193,6 +193,9 @@ func (this *TCPClient) connect(server *teaconfigs.ServerConfig) { return } + // 成功连接则重置错误数 + this.backend.CurrentFails = 0 + // 写入 go func() { for data := range this.stream { diff --git a/teaproxy/utils.go b/teaproxy/utils.go index 557deef..6d6c977 100644 --- a/teaproxy/utils.go +++ b/teaproxy/utils.go @@ -1,6 +1,6 @@ package teaproxy -// 清理Path中的多于信息 +// 清理Path中的多余的字符 func CleanPath(path string) string { l := len(path) if l == 0 { From 57449727f328d99080b945bcaddee231c683fd13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年12月12日 10:31:41 +0800 Subject: [PATCH 040/121] =?UTF-8?q?[proxy]=E6=9B=B4=E6=94=B9=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=E6=9C=8D=E5=8A=A1=E5=99=A8=E4=BF=A1=E6=81=AF=E6=97=B6?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E7=89=88=E6=9C=AC=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/proxy/backend/update.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/teaweb/actions/default/proxy/backend/update.go b/teaweb/actions/default/proxy/backend/update.go index 848b2a2..c4284dc 100644 --- a/teaweb/actions/default/proxy/backend/update.go +++ b/teaweb/actions/default/proxy/backend/update.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teaconfigs/shared" + "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/certs/certutils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" @@ -193,6 +194,7 @@ func (this *UpdateAction) RunPost(params struct { this.Fail("找不到要修改的后端服务器") } backend.Touch() + backend.TeaVersion = teaconst.TeaVersion backend.Address = teautils.FormatAddress(params.Address) backend.Scheme = params.Scheme From 6ae5340bd5c64837f3c645fdf968a2926035c121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年12月14日 17:11:19 +0800 Subject: [PATCH 041/121] =?UTF-8?q?=E6=97=A5=E5=BF=97=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=8F=AA=E6=9C=89=E8=B6=85=E8=BF=87128M=E9=87=8D=E5=90=AF?= =?UTF-8?q?=E6=97=B6=E6=89=8D=E4=BC=9A=E8=A6=86=E7=9B=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/utils/log_writer.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/teaweb/utils/log_writer.go b/teaweb/utils/log_writer.go index 236fb8e..3b69bd1 100644 --- a/teaweb/utils/log_writer.go +++ b/teaweb/utils/log_writer.go @@ -26,9 +26,12 @@ func (this *LogWriter) Init() { // 先删除原来的 logFile := files.NewFile(Tea.LogFile(teaconst.TeaProcessName + ".log")) if logFile.Exists() { - err := logFile.Delete() - if err != nil { - log.Println("[error]" + err.Error()) + fileSize, _ := logFile.Size() + if fileSize> 128*1024*1024 { // 如果超过128M则删除,通常来说这个尺寸足够了 + err := logFile.Delete() + if err != nil { + log.Println("[error]" + err.Error()) + } } } From 62bb47ed2d04d9319e462c949b67c0f3d231d8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年12月14日 17:11:43 +0800 Subject: [PATCH 042/121] =?UTF-8?q?=E6=97=A5=E5=BF=97=E4=B8=AD=E8=AE=B0?= =?UTF-8?q?=E5=BD=95TeaWeb=E8=BF=9B=E7=A8=8B=E6=8E=A5=E6=94=B6=E5=88=B0?= =?UTF-8?q?=E7=9A=84signal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/cmd/web_shell.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/teaweb/cmd/web_shell.go b/teaweb/cmd/web_shell.go index 8fcccb3..855eb47 100644 --- a/teaweb/cmd/web_shell.go +++ b/teaweb/cmd/web_shell.go @@ -58,6 +58,8 @@ func (this *WebShell) Start(server *TeaGo.Server) { // 信号 teautils.ListenSignal(func(sig os.Signal) { + logs.Println("[signal]" + sig.String()) + if sig == syscall.SIGHUP { // 重置 configs.SharedAdminConfig().Reset() } else if sig == syscall.Signal(0x1e /**syscall.SIGUSR1**/) { // 刷新代理状态 From 7f931b1f2ec2b23979731bd8f0a5c6c32d4e3251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年12月14日 18:21:38 +0800 Subject: [PATCH 043/121] =?UTF-8?q?[proxy]=E5=9C=A8TCP=E4=BB=A3=E7=90=86?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E4=B8=AD=E5=A2=9E=E5=8A=A0recover()=E6=8D=95?= =?UTF-8?q?=E8=8E=B7=E5=8F=AF=E8=83=BD=E7=9A=84panic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/listener.go | 2 ++ teaproxy/tcp_client.go | 8 +++++++- teautils/panic.go | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 teautils/panic.go diff --git a/teaproxy/listener.go b/teaproxy/listener.go index ac9d124..4ac0323 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -652,6 +652,8 @@ func (this *Listener) buildTLSConfig() *tls.Config { // 连接TCP后端 func (this *Listener) connectTCPBackend(clientConn net.Conn) { + defer teautils.Recover() + client := NewTCPClient(func() *teaconfigs.ServerConfig { if len(this.currentServers) == 0 { return nil diff --git a/teaproxy/tcp_client.go b/teaproxy/tcp_client.go index 24de0a7..b6b5c89 100644 --- a/teaproxy/tcp_client.go +++ b/teaproxy/tcp_client.go @@ -115,6 +115,8 @@ func (this *TCPClient) WriteSpeed() int64 { // 连接后端服务器 func (this *TCPClient) connect(server *teaconfigs.ServerConfig) { + defer teautils.Recover() + if !this.lActive { return } @@ -198,6 +200,8 @@ func (this *TCPClient) connect(server *teaconfigs.ServerConfig) { // 写入 go func() { + defer teautils.Recover() + for data := range this.stream { if data == nil { break @@ -256,7 +260,9 @@ func (this *TCPClient) read(server *teaconfigs.ServerConfig) { for { n, err := this.lConn.Read(buf) if n> 0 { - this.stream <- append([]byte{}, buf[:n]...) + if !this.streamIsClosed { + this.stream <- append([]byte{}, buf[:n]...) + } atomic.AddInt64(&this.writeSpeed, int64(n)) } diff --git a/teautils/panic.go b/teautils/panic.go new file mode 100644 index 0000000..bae9db6 --- /dev/null +++ b/teautils/panic.go @@ -0,0 +1,15 @@ +package teautils + +import ( + "github.com/iwind/TeaGo/logs" + "runtime/debug" +) + +// 记录panic日志 +func Recover() { + p := recover() + if p != nil { + logs.Println("panic:", p) + logs.Println(string(debug.Stack())) + } +} From 387050de8d86f61d66f987b89ce56e72c4063866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年12月17日 11:24:02 +0800 Subject: [PATCH 044/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8D=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E8=A7=84=E5=88=99=E4=B8=AD=E2=80=9C=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E7=BB=88=E6=AD=A2=E5=BE=80=E4=B8=8B=E5=8C=B9=E9=85=8D=E2=80=9D?= =?UTF-8?q?=E5=9C=A8=E6=9F=90=E7=A7=8D=E6=83=85=E5=86=B5=E4=B8=8B=E4=B8=8D?= =?UTF-8?q?=E8=B5=B7=E4=BD=9C=E7=94=A8=E7=9A=84Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/teaproxy/request.go b/teaproxy/request.go index 8936c85..9ef5c95 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -505,6 +505,11 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b this.uppercaseIgnoreHeaders = append(this.uppercaseIgnoreHeaders, fastcgi.UppercaseIgnoreHeaders()...) } + // break + if location.IsBreak { + break + } + continue } @@ -548,6 +553,11 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b this.uppercaseIgnoreHeaders = append(this.uppercaseIgnoreHeaders, backend.UppercaseIgnoreHeaders()...) } + // break + if location.IsBreak { + break + } + continue } From d6e796c0acee23910d0b6876591a3cf0c7a6fd3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2019年12月18日 11:00:29 +0800 Subject: [PATCH 045/121] =?UTF-8?q?[proxy]=E8=A7=A3=E5=86=B3=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E4=BF=9D=E5=AD=98=E6=97=A5=E5=BF=97=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E7=A9=BA=E6=8C=87=E9=92=88=E7=9A=84Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tealogs/accesslogs/access_log.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tealogs/accesslogs/access_log.go b/tealogs/accesslogs/access_log.go index 62e11db..0992e7f 100644 --- a/tealogs/accesslogs/access_log.go +++ b/tealogs/accesslogs/access_log.go @@ -10,6 +10,7 @@ import ( "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" + "github.com/iwind/TeaGo/types" "github.com/mailru/easyjson" "github.com/pquerna/ffjson/ffjson" "path/filepath" @@ -474,6 +475,9 @@ func (this *AccessLog) DBColumns() maps.Map { } func (this *AccessLog) jsonEncode(v interface{}) []byte { + if types.IsNil(v) { + return nil + } data, _ := ffjson.Marshal(v) return data } From 74d81c821c7c00028c61fbde88af612f1369598e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Sat, 8 Feb 2020 16:17:04 +0800 Subject: [PATCH 046/121] =?UTF-8?q?[proxy]ES=E6=97=A5=E5=BF=97=E7=AD=96?= =?UTF-8?q?=E7=95=A5=E5=A2=9E=E5=8A=A0Content-Type:=20application/json?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tealogs/storage_es.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tealogs/storage_es.go b/tealogs/storage_es.go index e43bdc3..a772dd0 100644 --- a/tealogs/storage_es.go +++ b/tealogs/storage_es.go @@ -100,6 +100,7 @@ func (this *ESStorage) Write(accessLogs []*accesslogs.AccessLog) error { if err != nil { return err } + req.Header.Set("Content-Type", "application/json") client := teautils.SharedHttpClient(10 * time.Second) defer func() { _ = req.Body.Close() From 8c22cc283fa8879aa2fe4d1542ff23804b9648de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月14日 11:38:03 +0800 Subject: [PATCH 047/121] =?UTF-8?q?websocket=E6=94=AF=E6=8C=81=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89Host=E3=80=81CA=E8=AF=81=E4=B9=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_websocket.go | 39 +++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/teaproxy/request_websocket.go b/teaproxy/request_websocket.go index 5501986..2fba4a1 100644 --- a/teaproxy/request_websocket.go +++ b/teaproxy/request_websocket.go @@ -2,6 +2,7 @@ package teaproxy import ( "bytes" + "crypto/tls" "errors" "fmt" "github.com/TeaWeb/code/teaconfigs" @@ -9,6 +10,7 @@ import ( "github.com/gorilla/websocket" "github.com/iwind/TeaGo/logs" "io/ioutil" + "net" "net/http" "net/url" "time" @@ -73,9 +75,42 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { if this.backend.Scheme == "https" { scheme = "wss" } - wsURL := url.URL{Scheme: scheme, Host: this.backend.Address, Path: this.raw.RequestURI} + host := this.raw.Host + if this.backend.HasHost() { + host = this.Format(this.backend.Host) + } + wsURL := url.URL{ + Scheme: scheme, + Host: host, + Path: this.raw.RequestURI, + } + + // TLS通讯 + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + if this.backend.Cert != nil { + obj := this.backend.Cert.CertObject() + if obj != nil { + tlsConfig.InsecureSkipVerify = false + tlsConfig.Certificates = []tls.Certificate{*obj} + if len(this.backend.Cert.ServerName)> 0 { + tlsConfig.ServerName = this.backend.Cert.ServerName + } + } + } + + // 超时时间 + connectionTimeout := this.backend.FailTimeoutDuration() + if connectionTimeout <= 0 { + connectionTimeout = 15 * time.Second + } + dialer := websocket.Dialer{ - Proxy: http.ProxyFromEnvironment, + NetDial: func(network, addr string) (conn net.Conn, err error) { + return net.DialTimeout(network, this.backend.Address, connectionTimeout) + }, + TLSClientConfig: tlsConfig, HandshakeTimeout: this.backend.FailTimeoutDuration(), } header := http.Header{} From 255678409c00a2a825b9ecffe39c88848994c634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月14日 12:59:13 +0800 Subject: [PATCH 048/121] =?UTF-8?q?[proxy]websocket=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E4=B8=AD=E5=90=AB=E6=9C=89=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_websocket.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/teaproxy/request_websocket.go b/teaproxy/request_websocket.go index 2fba4a1..ac28da9 100644 --- a/teaproxy/request_websocket.go +++ b/teaproxy/request_websocket.go @@ -79,10 +79,14 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { if this.backend.HasHost() { host = this.Format(this.backend.Host) } + wsURL := url.URL{ - Scheme: scheme, - Host: host, - Path: this.raw.RequestURI, + Scheme: scheme, + Host: host, + User: this.raw.URL.User, + Opaque: this.raw.URL.Opaque, + Path: this.raw.URL.Path, + RawQuery: this.raw.URL.RawQuery, } // TLS通讯 From 05ea471dc17a636b4a40ff3ceaa3605149b838bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月14日 16:07:48 +0800 Subject: [PATCH 049/121] =?UTF-8?q?[proxy]websocket=E5=A2=9E=E5=8A=A0X-For?= =?UTF-8?q?warded-For=E5=92=8CX-Real-IP=E7=AD=89Header?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_websocket.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/teaproxy/request_websocket.go b/teaproxy/request_websocket.go index ac28da9..5efc601 100644 --- a/teaproxy/request_websocket.go +++ b/teaproxy/request_websocket.go @@ -125,6 +125,8 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { } } + this.setProxyHeaders(header) + // 自定义请求Header for _, h := range this.requestHeaders { if !h.On { From d013c6652fe71b12bab842145ca8f4824cd83419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月15日 09:08:00 +0800 Subject: [PATCH 050/121] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconst/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/teaconst/const.go b/teaconst/const.go index 5036f3a..65223fb 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.9.2" + TeaVersion = "0.1.9.3" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 From 801f1061b76f1ab5d4777ca00dfbd2335cd41841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月17日 09:36:43 +0800 Subject: [PATCH 051/121] =?UTF-8?q?[proxy]websocket=E8=BD=AC=E5=8F=91?= =?UTF-8?q?=E6=97=B6=E8=87=AA=E5=8A=A8=E7=BB=A7=E6=89=BF=E8=AF=B7=E6=B1=82?= =?UTF-8?q?Header?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_websocket.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/teaproxy/request_websocket.go b/teaproxy/request_websocket.go index 5efc601..5465b59 100644 --- a/teaproxy/request_websocket.go +++ b/teaproxy/request_websocket.go @@ -13,6 +13,7 @@ import ( "net" "net/http" "net/url" + "strings" "time" ) @@ -118,11 +119,11 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { HandshakeTimeout: this.backend.FailTimeoutDuration(), } header := http.Header{} - { - origin, ok := this.raw.Header["Origin"] - if ok { - header["Origin"] = origin + for k, v := range this.raw.Header { + if strings.HasPrefix(k, "Sec-") || k == "Upgrade" || k == "Connection" { + continue } + header[k] = v } this.setProxyHeaders(header) From 7b6420e15d224f3a1411d5e27504e8e8fd44a9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月17日 10:20:57 +0800 Subject: [PATCH 052/121] =?UTF-8?q?[proxy]ES=E6=97=A5=E5=BF=97=E7=AD=96?= =?UTF-8?q?=E7=95=A5=E6=94=AF=E6=8C=81=E7=94=A8=E6=88=B7=E5=90=8D=E5=92=8C?= =?UTF-8?q?=E5=AF=86=E7=A0=81=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tealogs/storage_es.go | 8 ++++++++ tealogs/storage_es_test.go | 4 ++++ teatesting/require.go | 14 ++++++++++++-- teaweb/actions/default/proxy/log/policies/add.go | 4 ++++ .../actions/default/proxy/log/policies/update.go | 4 ++++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/tealogs/storage_es.go b/tealogs/storage_es.go index a772dd0..2f95f91 100644 --- a/tealogs/storage_es.go +++ b/tealogs/storage_es.go @@ -1,8 +1,10 @@ package tealogs import ( + "encoding/base64" "errors" "fmt" + "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/tealogs/accesslogs" "github.com/TeaWeb/code/teautils" "github.com/iwind/TeaGo/logs" @@ -21,6 +23,8 @@ type ESStorage struct { Endpoint string `yaml:"endpoint" json:"endpoint"` Index string `yaml:"index" json:"index"` MappingType string `yaml:"mappingType" json:"mappingType"` + Username string `yaml:"username" json:"username"` + Password string `yaml:"password" json:"password"` } // 开启 @@ -101,6 +105,10 @@ func (this *ESStorage) Write(accessLogs []*accesslogs.AccessLog) error { return err } req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", teaconst.TeaProductName+"/"+teaconst.TeaVersion) + if len(this.Username)> 0 || len(this.Password)> 0 { + req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(this.Username+":"+this.Password))) + } client := teautils.SharedHttpClient(10 * time.Second) defer func() { _ = req.Body.Close() diff --git a/tealogs/storage_es_test.go b/tealogs/storage_es_test.go index 14a2740..e39b6ea 100644 --- a/tealogs/storage_es_test.go +++ b/tealogs/storage_es_test.go @@ -18,6 +18,8 @@ func TestESStorage_Write(t *testing.T) { Endpoint: "http://127.0.0.1:9200", Index: "logs", MappingType: "accessLogs", + Username: "hello", + Password: "world", } err := storage.Start() if err != nil { @@ -32,6 +34,7 @@ func TestESStorage_Write(t *testing.T) { RequestMethod: "POST", RequestPath: "/1", TimeLocal: time.Now().Format("2/Jan/2006:15:04:05 -0700"), + TimeISO8601: "2018-07-23T22:23:35+08:00", Header: map[string][]string{ "Content-Type": {"text/html"}, }, @@ -40,6 +43,7 @@ func TestESStorage_Write(t *testing.T) { RequestMethod: "GET", RequestPath: "/2", TimeLocal: time.Now().Format("2/Jan/2006:15:04:05 -0700"), + TimeISO8601: "2018-07-23T22:23:35+08:00", Header: map[string][]string{ "Content-Type": {"text/css"}, }, diff --git a/teatesting/require.go b/teatesting/require.go index 520715f..03c4e25 100644 --- a/teatesting/require.go +++ b/teatesting/require.go @@ -1,6 +1,7 @@ package teatesting import ( + "github.com/TeaWeb/code/teautils" "github.com/iwind/TeaGo/logs" "net/http" "time" @@ -100,8 +101,17 @@ func RequireRedis() bool { // 需要ES支持 func RequireElasticSearch() bool { - // TODO - return false + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:9200", nil) + if err != nil { + logs.Error(err) + return false + } + _, err = teautils.SharedHttpClient(1 * time.Second).Do(req) + if err != nil { + return false + } + + return true } // 检查Fastcgi diff --git a/teaweb/actions/default/proxy/log/policies/add.go b/teaweb/actions/default/proxy/log/policies/add.go index a7bc9ef..48d47fd 100644 --- a/teaweb/actions/default/proxy/log/policies/add.go +++ b/teaweb/actions/default/proxy/log/policies/add.go @@ -43,6 +43,8 @@ func (this *AddAction) RunPost(params struct { EsEndpoint string EsIndex string EsMappingType string + EsUsername string + EsPassword string // mysql MysqlHost string @@ -107,6 +109,8 @@ func (this *AddAction) RunPost(params struct { storage.Endpoint = params.EsEndpoint storage.Index = params.EsIndex storage.MappingType = params.EsMappingType + storage.Username = params.EsUsername + storage.Password = params.EsPassword instance = storage case tealogs.StorageTypeMySQL: params.Must. diff --git a/teaweb/actions/default/proxy/log/policies/update.go b/teaweb/actions/default/proxy/log/policies/update.go index 387f9ac..d7bff91 100644 --- a/teaweb/actions/default/proxy/log/policies/update.go +++ b/teaweb/actions/default/proxy/log/policies/update.go @@ -62,6 +62,8 @@ func (this *UpdateAction) RunPost(params struct { EsEndpoint string EsIndex string EsMappingType string + EsUsername string + EsPassword string // mysql MysqlHost string @@ -131,6 +133,8 @@ func (this *UpdateAction) RunPost(params struct { storage.Endpoint = params.EsEndpoint storage.Index = params.EsIndex storage.MappingType = params.EsMappingType + storage.Username = params.EsUsername + storage.Password = params.EsPassword instance = storage case tealogs.StorageTypeMySQL: params.Must. From 952ae56fdb77661bda56fa6922a0757a669d1046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月17日 11:00:54 +0800 Subject: [PATCH 053/121] =?UTF-8?q?MongoDB=E5=8F=AF=E4=BB=A5=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E8=BF=9E=E6=8E=A5=E6=B1=A0=E5=B0=BA=E5=AF=B8=E3=80=81?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E8=B6=85=E6=97=B6=E6=97=B6=E9=97=B4=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E8=B0=83=E6=95=B4=E9=BB=98=E8=AE=A4=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E6=B1=A0=E5=B0=BA=E5=AF=B8=E4=B8=BA32=EF=BC=88=E4=BB=A5?= =?UTF-8?q?=E5=89=8D=E7=89=88=E6=9C=AC=E4=B8=BA10=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/db/mongo.go | 3 +++ teadb/driver_mongo.go | 12 ++++++++++-- teaweb/actions/default/settings/mongo/update.go | 12 +++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/teaconfigs/db/mongo.go b/teaconfigs/db/mongo.go index 57b13ab..9e7280e 100644 --- a/teaconfigs/db/mongo.go +++ b/teaconfigs/db/mongo.go @@ -35,6 +35,9 @@ type MongoConfig struct { RequestURI string `yaml:"requestURI" json:"requestURI"` // @TODO 未来版本需要实现 DBName string `yaml:"dbName" json:"dbName"` + PoolSize int `yaml:"poolSize" json:"poolSize"` // 连接池大小 + Timeout int `yaml:"timeout" json:"timeout"` // 超时时间(秒) + // 日志访问配置 AccessLog *MongoAccessLogConfig `yaml:"accessLog" json:"accessLog"` } diff --git a/teadb/driver_mongo.go b/teadb/driver_mongo.go index e2d3a74..6010428 100644 --- a/teadb/driver_mongo.go +++ b/teadb/driver_mongo.go @@ -672,8 +672,16 @@ func (this *MongoDriver) connect() (*mongo.Client, error) { this.dbName = config.DBName opts := options.Client().ApplyURI(config.URI) - opts.SetMaxPoolSize(10). - SetConnectTimeout(5 * time.Second) + if config.PoolSize> 0 { + opts.SetMaxPoolSize(uint64(config.PoolSize)) + } else { + opts.SetMaxPoolSize(32) + } + if config.Timeout> 0 { + opts.SetConnectTimeout(time.Duration(5) * time.Second) + } else { + opts.SetConnectTimeout(5 * time.Second) + } sharedConfig, err := db.LoadMongoConfig() if err != nil { return nil, err diff --git a/teaweb/actions/default/settings/mongo/update.go b/teaweb/actions/default/settings/mongo/update.go index aaa5369..37bff44 100644 --- a/teaweb/actions/default/settings/mongo/update.go +++ b/teaweb/actions/default/settings/mongo/update.go @@ -31,6 +31,8 @@ func (this *UpdateAction) Run(params struct { "authMechanism": config.AuthMechanism, "authMechanismProperties": config.AuthMechanismPropertiesString(), "requestURI": config.RequestURI, + "poolSize": config.PoolSize, + "timeout": config.Timeout, } this.Show() @@ -46,6 +48,8 @@ func (this *UpdateAction) RunPost(params struct { AuthEnabled bool AuthMechanism string AuthMechanismProperties string + PoolSize int + Timeout int Must *actions.Must }) { @@ -58,7 +62,11 @@ func (this *UpdateAction) RunPost(params struct { Require("请输入主机地址"). Field("port", params.Port). Require("请输入端口"). - Gt(0, "请输入正确的端口") + Gt(0, "请输入正确的端口"). + Field("poolSize", params.PoolSize). + Gte(0, "连接池不能小于0"). + Field("timeout", params.Timeout). + Gte(0, "超时时间不能小于0") config, err := db.LoadMongoConfig() if err != nil { @@ -78,6 +86,8 @@ func (this *UpdateAction) RunPost(params struct { } else { config.Password = "" } + config.PoolSize = params.PoolSize + config.Timeout = params.Timeout err = config.Save() if err != nil { From acbff3d72b79353716f0d21a16947663659ee5c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月17日 11:17:35 +0800 Subject: [PATCH 054/121] =?UTF-8?q?MySQL=E5=92=8CPostgreSQL=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E8=AE=BE=E7=BD=AE=E8=BF=9E=E6=8E=A5=E6=B1=A0=E5=B0=BA?= =?UTF-8?q?=E5=AF=B8=E3=80=81=E8=BF=9E=E6=8E=A5=E8=B6=85=E6=97=B6=E6=97=B6?= =?UTF-8?q?=E9=97=B4=EF=BC=8C=E5=B9=B6=E5=A2=9E=E5=A4=A7=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=B1=A0=E5=B0=BA=E5=AF=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/db/mysql.go | 2 ++ teaconfigs/db/postgres.go | 2 ++ teadb/driver_mysql.go | 10 ++++++++-- teadb/driver_postgres.go | 10 ++++++++-- teaweb/actions/default/settings/mysql/update.go | 2 ++ teaweb/actions/default/settings/postgres/update.go | 2 ++ 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/teaconfigs/db/mysql.go b/teaconfigs/db/mysql.go index 7c4e3bc..8c1356c 100644 --- a/teaconfigs/db/mysql.go +++ b/teaconfigs/db/mysql.go @@ -20,6 +20,8 @@ type MySQLConfig struct { Password string `yaml:"password" json:"password"` DBName string `yaml:"dbName" json:"dbName"` + PoolSize int `yaml:"poolSize" json:"poolSize"` + // 日志访问配置 AccessLog *MySQLAccessLogConfig `yaml:"accessLog" json:"accessLog"` } diff --git a/teaconfigs/db/postgres.go b/teaconfigs/db/postgres.go index 8f2d48e..3555c34 100644 --- a/teaconfigs/db/postgres.go +++ b/teaconfigs/db/postgres.go @@ -20,6 +20,8 @@ type PostgresConfig struct { Password string `yaml:"password" json:"password"` DBName string `yaml:"dbName" json:"dbName"` + PoolSize int `yaml:"poolSize" json:"poolSize"` + // 日志访问配置 AccessLog *PostgresAccessLogConfig `yaml:"accessLog" json:"accessLog"` } diff --git a/teadb/driver_mysql.go b/teadb/driver_mysql.go index 3707bb8..b6680ce 100644 --- a/teadb/driver_mysql.go +++ b/teadb/driver_mysql.go @@ -68,8 +68,14 @@ func (this *MySQLDriver) initDB() error { if err != nil { return err } - dbInstance.SetMaxIdleConns(10) - dbInstance.SetMaxOpenConns(32) + if config.PoolSize> 0 { + half := config.PoolSize / 2 + dbInstance.SetMaxIdleConns(half) + dbInstance.SetMaxOpenConns(config.PoolSize) + } else { + dbInstance.SetMaxIdleConns(32) + dbInstance.SetMaxOpenConns(64) + } dbInstance.SetConnMaxLifetime(0) this.db = dbInstance diff --git a/teadb/driver_postgres.go b/teadb/driver_postgres.go index 98b15e7..656cf4f 100644 --- a/teadb/driver_postgres.go +++ b/teadb/driver_postgres.go @@ -70,8 +70,14 @@ func (this *PostgresDriver) initDB() error { if err != nil { return err } - dbInstance.SetMaxIdleConns(10) - dbInstance.SetMaxOpenConns(32) + if config.PoolSize> 0 { + half := config.PoolSize / 2 + dbInstance.SetMaxIdleConns(half) + dbInstance.SetMaxOpenConns(config.PoolSize) + } else { + dbInstance.SetMaxIdleConns(32) + dbInstance.SetMaxOpenConns(64) + } dbInstance.SetConnMaxLifetime(0) this.db = dbInstance return nil diff --git a/teaweb/actions/default/settings/mysql/update.go b/teaweb/actions/default/settings/mysql/update.go index 251f4be..75c58a0 100644 --- a/teaweb/actions/default/settings/mysql/update.go +++ b/teaweb/actions/default/settings/mysql/update.go @@ -32,6 +32,7 @@ func (this *UpdateAction) RunPost(params struct { Username string Password string DBName string `alias:"dbName"` + PoolSize int Must *actions.Must }) { // 是否已改变 @@ -46,6 +47,7 @@ func (this *UpdateAction) RunPost(params struct { config.Username = params.Username config.Password = params.Password config.DBName = params.DBName + config.PoolSize = params.PoolSize config.DSN = config.ComposeDSN() err = config.Save() if err != nil { diff --git a/teaweb/actions/default/settings/postgres/update.go b/teaweb/actions/default/settings/postgres/update.go index 0aff5ec..487adba 100644 --- a/teaweb/actions/default/settings/postgres/update.go +++ b/teaweb/actions/default/settings/postgres/update.go @@ -32,6 +32,7 @@ func (this *UpdateAction) RunPost(params struct { Username string Password string DBName string `alias:"dbName"` + PoolSize int Must *actions.Must }) { // 是否已改变 @@ -46,6 +47,7 @@ func (this *UpdateAction) RunPost(params struct { config.Username = params.Username config.Password = params.Password config.DBName = params.DBName + config.PoolSize = params.PoolSize config.DSN = config.ComposeDSN() err = config.Save() if err != nil { From 6cdc4a5b4157499880f4d1ce5632a7d51cbc2159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年2月19日 20:45:57 +0800 Subject: [PATCH 055/121] =?UTF-8?q?[=E7=9B=91=E6=8E=A7]=E5=9C=A8=E4=B8=BB?= =?UTF-8?q?=E4=BB=8E=E6=A8=A1=E5=BC=8F=E4=B8=8B=E6=9C=AC=E5=9C=B0Agent?= =?UTF-8?q?=E7=9A=84=E7=9C=8B=E6=9D=BF=E4=B8=8D=E8=A6=86=E7=9B=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacluster/utils.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/teacluster/utils.go b/teacluster/utils.go index 0e45763..a322531 100644 --- a/teacluster/utils.go +++ b/teacluster/utils.go @@ -23,7 +23,12 @@ func RangeFiles(f func(file *files.File, relativePath string)) { strings.HasPrefix(file.Name(), ".") { return } - if lists.ContainsString([]string{"node.conf", "server.conf", "agent.local.conf"}, file.Name()) { + if lists.ContainsString([]string{ + "node.conf", + "server.conf", + "agent.local.conf", + "board.local.conf", + }, file.Name()) { return } absPath, _ := file.AbsPath() From 32f5788d4bdd0b08f4769ba0d643c1f9c7dc1f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Mon, 9 Mar 2020 10:08:01 +0800 Subject: [PATCH 056/121] =?UTF-8?q?[cluster]=E4=BC=A0=E8=BE=93=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=BF=87=E7=A8=8B=E5=8F=91=E7=94=9F=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E6=97=B6=E5=85=B3=E9=97=AD=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacluster/manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/teacluster/manager.go b/teacluster/manager.go index cd82fa4..25d8f65 100644 --- a/teacluster/manager.go +++ b/teacluster/manager.go @@ -79,6 +79,10 @@ func (this *Manager) Start() error { this.error = err.Error() return err } + defer func() { + // close connection + _ = conn.Close() + }() this.isActive = true From 0f1b05df4543a011eec400d5df10949f15f9133d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Mon, 9 Mar 2020 13:51:22 +0800 Subject: [PATCH 057/121] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=99=A8=E5=A2=9E=E5=8A=A0websocket=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teatesting/server.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/teatesting/server.go b/teatesting/server.go index 5c4e2f6..0a8c545 100644 --- a/teatesting/server.go +++ b/teatesting/server.go @@ -3,6 +3,7 @@ package teatesting import ( "compress/gzip" "fmt" + "github.com/gorilla/websocket" "github.com/iwind/TeaGo" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" @@ -177,6 +178,37 @@ func StartTestServer() { `)) }). + Get("/websocket", func(req *http.Request, resp http.ResponseWriter) { + logs.Println("[test]websocket receive request") + + upgrader := websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, + } + c, err := upgrader.Upgrade(resp, req, nil) + if err != nil { + logs.Println("[test]websocket upgrade:", err) + return + } + defer func() { + _ = c.Close() + }() + for { + mt, message, err := c.ReadMessage() + if err != nil { + logs.Println("[test]websocket read:", err) + break + } + logs.Printf("[test]websocket recv: %s", message) + err = c.WriteMessage(mt, message) + if err != nil { + logs.Println("[test]websocket write:", err) + break + } + } + logs.Println("[test]websocket closed") + }). StartOn("127.0.0.1:9991") } @@ -216,3 +248,9 @@ func compressResource(writer http.ResponseWriter, path string, mimeType string) logs.Error(err) } } + +// websocket response +type WebsocketResponse struct { + resp http.ResponseWriter + hijacker http.Hijacker +} From cbe14b140d9a224129268f3aa5a915c99f295996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Mon, 9 Mar 2020 18:02:28 +0800 Subject: [PATCH 058/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8D=E5=9B=A0?= =?UTF-8?q?=E4=B8=BA=E5=A4=8D=E5=88=B6=E8=B7=AF=E5=BE=84=E8=A7=84=E5=88=99?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E5=90=8E=E7=AB=AF=E6=9C=8D=E5=8A=A1=E5=99=A8?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=B1=A0=E4=B8=8D=E8=83=BD=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/ftp_client_pool.go | 5 ++++- teaproxy/http_client_pool.go | 10 +++++++++- teaproxy/request_backend.go | 4 ++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/teaproxy/ftp_client_pool.go b/teaproxy/ftp_client_pool.go index 3e0d2cf..bf3a21b 100644 --- a/teaproxy/ftp_client_pool.go +++ b/teaproxy/ftp_client_pool.go @@ -25,8 +25,11 @@ func NewFTPClientPool() *FTPClientPool { } // 通过Backend配置FTP客户端 -func (this *FTPClientPool) client(backend *teaconfigs.BackendConfig) *FTPClient { +func (this *FTPClientPool) client(backend *teaconfigs.BackendConfig, location *teaconfigs.LocationConfig) *FTPClient { key := backend.UniqueKey() + if location != nil { + key = location.Id + "_" + key + } this.locker.Lock() defer this.locker.Unlock() diff --git a/teaproxy/http_client_pool.go b/teaproxy/http_client_pool.go index f4675a6..02e7d7d 100644 --- a/teaproxy/http_client_pool.go +++ b/teaproxy/http_client_pool.go @@ -5,9 +5,11 @@ import ( "crypto/tls" "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teautils" + "github.com/iwind/TeaGo/logs" "net" "net/http" "runtime" + "strconv" "strings" "sync" "time" @@ -30,8 +32,11 @@ func NewHTTPClientPool() *HTTPClientPool { } // 根据地址获取客户端 -func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig) *http.Client { +func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig, location *teaconfigs.LocationConfig) *http.Client { key := backend.UniqueKey() + if location != nil { + key = location.Id + "_" + key + } this.locker.RLock() client, found := this.clientsMap[key] @@ -70,6 +75,8 @@ func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig) *http.Clie idleConns = numberCPU } + logs.Println("[proxy]setup backend '" + key + "', max connections:" + strconv.Itoa(maxConnections) + ", max idles:" + strconv.Itoa(idleConns)) + // TLS通讯 tlsConfig := &tls.Config{ InsecureSkipVerify: true, @@ -129,6 +136,7 @@ func (this *HTTPClientPool) closeOldClient(key string) { if backendId == backendId2 && key != key2 { teautils.CloseHTTPClient(client) delete(this.clientsMap, key2) + logs.Println("[proxy]close backend '" + key2) break } } diff --git a/teaproxy/request_backend.go b/teaproxy/request_backend.go index 825eaac..1db9c3b 100644 --- a/teaproxy/request_backend.go +++ b/teaproxy/request_backend.go @@ -113,10 +113,10 @@ func (this *Request) callBackend(writer *ResponseWriter) error { var resp *http.Response = nil var err error = nil if this.backend.IsFTP() { - client := SharedFTPClientPool.client(this.backend) + client := SharedFTPClientPool.client(this.backend, this.location) resp, err = client.Do(this.raw) } else { - client := SharedHTTPClientPool.client(this.backend) + client := SharedHTTPClientPool.client(this.backend, this.location) resp, err = client.Do(this.raw) } if err != nil { From ee2bf559f1d51a42fdb3f24470bbb5012e8fd4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月10日 09:40:35 +0800 Subject: [PATCH 059/121] =?UTF-8?q?[proxy]websocket=E5=8F=91=E7=94=9F?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E7=9A=84=E6=97=B6=E5=80=99=EF=BC=8C=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=E6=9B=B4=E8=AF=A6=E7=BB=86=E7=9A=84=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_websocket.go | 14 +++++++++++--- teautils/map_test.go | 12 ++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 teautils/map_test.go diff --git a/teaproxy/request_websocket.go b/teaproxy/request_websocket.go index 5465b59..4d0db6b 100644 --- a/teaproxy/request_websocket.go +++ b/teaproxy/request_websocket.go @@ -13,6 +13,7 @@ import ( "net" "net/http" "net/url" + "strconv" "strings" "time" ) @@ -140,10 +141,16 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { } } - server, _, err := dialer.Dial(wsURL.String(), header) + server, resp, err := dialer.Dial(wsURL.String(), header) if err != nil { - logs.Error(err) - this.addError(err) + errString := "" + if resp != nil && resp.Body != nil { + data, _ := ioutil.ReadAll(resp.Body) + errString = strconv.Itoa(resp.StatusCode) + " " + string(bytes.TrimSpace(data)) + } + err1 := errors.New(err.Error() + ": " + errString) + logs.Error(err1) + this.addError(err1) currentFails := this.backend.IncreaseFails() if this.backend.MaxFails> 0 && currentFails>= this.backend.MaxFails { this.backend.IsDown = true @@ -159,6 +166,7 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { this.websocket.SetupScheduling(false) } + return err } defer func() { diff --git a/teautils/map_test.go b/teautils/map_test.go new file mode 100644 index 0000000..7fac404 --- /dev/null +++ b/teautils/map_test.go @@ -0,0 +1,12 @@ +package teautils + +import ( + "testing" +) + +func TestObjectToMapJSON(t *testing.T) { + var a interface{} = nil + m := map[string]interface{}{} + t.Log(ObjectToMapJSON(a, &m)) + t.Log(m) +} From e36119198e20a97c6896aedec1591eb677a0ad34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月14日 19:02:58 +0800 Subject: [PATCH 060/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8DWebSocket?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF=E5=BC=82=E5=B8=B8=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=9C=8D=E5=8A=A1=E5=99=A8=E7=AB=AF=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E5=85=B3=E9=97=AD=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/agents/source_socket_connectivity.go | 2 +- teaproxy/request_websocket.go | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/teaconfigs/agents/source_socket_connectivity.go b/teaconfigs/agents/source_socket_connectivity.go index f3ef2a6..edae01e 100644 --- a/teaconfigs/agents/source_socket_connectivity.go +++ b/teaconfigs/agents/source_socket_connectivity.go @@ -63,7 +63,7 @@ func (this *SocketConnectivitySource) Execute(params map[string]string) (value i return value, err } - conn.Close() + _ = conn.Close() value = maps.Map{ "cost": time.Since(before).Seconds(), diff --git a/teaproxy/request_websocket.go b/teaproxy/request_websocket.go index 4d0db6b..d7b4b85 100644 --- a/teaproxy/request_websocket.go +++ b/teaproxy/request_websocket.go @@ -146,7 +146,7 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { errString := "" if resp != nil && resp.Body != nil { data, _ := ioutil.ReadAll(resp.Body) - errString = strconv.Itoa(resp.StatusCode) + " " + string(bytes.TrimSpace(data)) + errString = strconv.Itoa(resp.StatusCode) + ": " + string(bytes.TrimSpace(data)) } err1 := errors.New(err.Error() + ": " + errString) logs.Error(err1) @@ -195,6 +195,13 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { this.addError(err) } clientIsClosed = true + _ = client.Close() + + // 关闭Server + if !serverIsClosed { + serverIsClosed = true + _ = server.Close() + } break } _ = server.WriteMessage(messageType, message) @@ -212,7 +219,10 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { } serverIsClosed = true _ = server.Close() + + // 关闭客户端 if !clientIsClosed { + clientIsClosed = true _ = client.Close() } break From 2731f9f8000e2ce6b171e4ef9a78c2e83e457e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月17日 16:52:38 +0800 Subject: [PATCH 061/121] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E8=AE=BF=E9=97=AE=E6=97=A5=E5=BF=97=E5=9B=A0=E4=B8=BA=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA=E8=80=8C=E6=97=A0=E6=B3=95=E4=BF=9D=E5=AD=98=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconst/const.go | 2 +- tealogs/accesslogs/access_log.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/teaconst/const.go b/teaconst/const.go index 65223fb..d31c698 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.9.3" + TeaVersion = "0.1.9.4" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 diff --git a/tealogs/accesslogs/access_log.go b/tealogs/accesslogs/access_log.go index 0992e7f..9389911 100644 --- a/tealogs/accesslogs/access_log.go +++ b/tealogs/accesslogs/access_log.go @@ -476,7 +476,7 @@ func (this *AccessLog) DBColumns() maps.Map { func (this *AccessLog) jsonEncode(v interface{}) []byte { if types.IsNil(v) { - return nil + return []byte("null") } data, _ := ffjson.Marshal(v) return data From 0479e5966ceb90d7d772a141a02b73c7f2870c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月25日 10:31:55 +0800 Subject: [PATCH 062/121] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E5=AE=88=E6=8A=A4?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=EF=BC=8C=E9=98=B2=E6=AD=A2=E6=84=8F=E5=A4=96?= =?UTF-8?q?=E9=80=80=E5=87=BA=E8=BF=9B=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconst/const.go | 2 +- teautils/process.go | 23 +++++++++++ teaweb/cmd/web_shell.go | 86 ++++++++++++++++++++++++++++++++++------- 3 files changed, 95 insertions(+), 16 deletions(-) diff --git a/teaconst/const.go b/teaconst/const.go index d31c698..879a84c 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.9.4" + TeaVersion = "0.1.10" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 diff --git a/teautils/process.go b/teautils/process.go index b314302..850552b 100644 --- a/teautils/process.go +++ b/teautils/process.go @@ -72,6 +72,29 @@ func WritePid(path string) error { return nil } +// 写入Ppid +func WritePpid(path string) error { + fp, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_RDONLY, 0666) + if err != nil { + return err + } + + if runtime.GOOS != "windows" { + err = LockFile(fp) + if err != nil { + return err + } + } + pidFileList = append(pidFileList, fp) // hold the file pointers + + _, err = fp.WriteString(fmt.Sprintf("%d", os.Getppid())) + if err != nil { + return err + } + + return nil +} + // 删除Pid func DeletePid(path string) error { _, err := os.Stat(path) diff --git a/teaweb/cmd/web_shell.go b/teaweb/cmd/web_shell.go index 855eb47..b5feda9 100644 --- a/teaweb/cmd/web_shell.go +++ b/teaweb/cmd/web_shell.go @@ -50,7 +50,7 @@ func (this *WebShell) Start(server *TeaGo.Server) { } // 当前PID文件句柄 - err := this.writePid() + err := this.writePpid() if err != nil { logs.Println("[error]write pid file failed: '" + err.Error() + "'") return @@ -133,8 +133,10 @@ func (this *WebShell) execArgs(writer io.Writer) bool { this.write(writer, teaconst.TeaProductName+" is already running, pid:", proc.Pid) return true } + return false } + args := os.Args[1:] arg0 := "" if len(args)> 0 { @@ -142,27 +144,43 @@ func (this *WebShell) execArgs(writer io.Writer) bool { } if this.hasArg(arg0, "?", "help", "-help", "h", "-h") { // 帮助 return this.ExecHelp(writer) - } else if this.hasArg(arg0, "-v", "version", "-version") { // 版本号 + } + if this.hasArg(arg0, "-v", "version", "-version") { // 版本号 return this.ExecVersion(writer) - } else if this.hasArg(arg0, "start") { // 启动 + } + if this.hasArg(arg0, "start") { // 启动 return this.ExecStart(writer) - } else if this.hasArg(arg0, "stop") { // 停止 + } + if this.hasArg(arg0, "stop") { // 停止 return this.ExecStop(os.Stdout) - } else if this.hasArg(arg0, "reload") { // 重新加载代理配置 + } + if this.hasArg(arg0, "reload") { // 重新加载代理配置 return this.ExecReload(writer) - } else if this.hasArg(arg0, "restart") { // 重启 + } + if this.hasArg(arg0, "restart") { // 重启 return this.ExecRestart(writer) - } else if this.hasArg(arg0, "reset") { // 重置 + } + if this.hasArg(arg0, "reset") { // 重置 return this.ExecReset(writer) - } else if this.hasArg(arg0, "status") { // 状态 + } + if this.hasArg(arg0, "status") { // 状态 return this.ExecStatus(writer) - } else if this.hasArg(arg0, "sync") { // 同步 + } + if this.hasArg(arg0, "sync") { // 同步 return this.ExecSync(writer) - } else if this.hasArg(arg0, "service") && runtime.GOOS == "windows" { // Windows服务 + } + if this.hasArg(arg0, "service") && runtime.GOOS == "windows" { // Windows服务 return this.ExecService(writer) - } else if this.hasArg(arg0, "pprof") { + } + if this.hasArg(arg0, "pprof") { return this.ExecPprof(writer) } + if this.hasArg(arg0, "master") { + return this.ExecMaster(writer) + } + if this.hasArg(arg0, "worker") { + return this.ExecWorker(writer) + } if len(args)> 0 { this.write(writer, "Unknown command option '"+strings.Join(args, " ")+"', run './bin/"+teaconst.TeaProcessName+" -h' to lookup the usage.") @@ -207,7 +225,7 @@ func (this *WebShell) ExecStart(writer io.Writer) bool { return true } - cmd := exec.Command(os.Args[0]) + cmd := exec.Command(os.Args[0], "master") err := cmd.Start() if err != nil { this.write(writer, teaconst.TeaProductName+" start failed:", err.Error()) @@ -226,13 +244,14 @@ func (this *WebShell) ExecStop(writer io.Writer) bool { return true } + // 停止子进程 err := proc.Kill() if err != nil { this.write(writer, teaconst.TeaProductName+" stop error:", err.Error()) return true } - // 在Windows上经常不能即使释放资源 + // 在Windows上经常不能及时释放资源 _ = teautils.DeletePid(Tea.Root + "/bin/pid") this.write(writer, teaconst.TeaProductName+" stopped ok, pid:", proc.Pid) @@ -340,9 +359,46 @@ func (this *WebShell) ExecPprof(writer io.Writer) bool { return false } +// 守护进程 +func (this *WebShell) ExecMaster(writer io.Writer) bool { + // 生成子进程,当前进程变成守护进程 + for { + cmd := exec.Command(os.Args[0], "worker") + err := cmd.Start() + if err != nil { + this.write(writer, teaconst.TeaProductName+" start failed:", err.Error()) + break + } + _ = cmd.Wait() + } + return true +} + +// 在后台执行 +func (this *WebShell) ExecWorker(writer io.Writer) bool { + ppid := os.Getppid() + + go func() { + for { + // 检查父进程是否已消失 + if os.Getppid() != ppid { + os.Exit(0) + return + } + parentProcess, _ := os.FindProcess(ppid) + if parentProcess == nil { + os.Exit(0) + return + } + time.Sleep(5 * time.Second) + } + }() + return false +} + // 写入PID -func (this *WebShell) writePid() error { - return teautils.WritePid(Tea.Root + Tea.DS + "bin" + Tea.DS + "pid") +func (this *WebShell) writePpid() error { + return teautils.WritePpid(Tea.Root + Tea.DS + "bin" + Tea.DS + "pid") } // 检查PID From f8520740e5f0dae5bff125297defffc9584db3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月25日 15:51:38 +0800 Subject: [PATCH 063/121] =?UTF-8?q?[proxy]=E6=B7=BB=E5=8A=A0=E5=9F=9F?= =?UTF-8?q?=E5=90=8D=E6=98=AF=E5=90=A6=E4=B8=A5=E6=A0=BC=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E9=80=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/proxy_setting.go | 62 +++++++++++++++++++ teaconfigs/proxy_setting_test.go | 26 ++++++++ teaproxy/listener.go | 9 ++- .../actions/default/proxy/proxyutils/menu.go | 3 +- .../actions/default/proxy/settings/helper.go | 15 +++++ .../actions/default/proxy/settings/index.go | 14 +++++ teaweb/actions/default/proxy/settings/init.go | 23 +++++++ .../actions/default/proxy/settings/update.go | 32 ++++++++++ teaweb/actions/default/ui/components.go | 39 ++++++++++++ teaweb/actions/default/ui/init.go | 16 +++++ teaweb/web.go | 2 + 11 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 teaconfigs/proxy_setting.go create mode 100644 teaconfigs/proxy_setting_test.go create mode 100644 teaweb/actions/default/proxy/settings/helper.go create mode 100644 teaweb/actions/default/proxy/settings/index.go create mode 100644 teaweb/actions/default/proxy/settings/init.go create mode 100644 teaweb/actions/default/proxy/settings/update.go create mode 100644 teaweb/actions/default/ui/components.go create mode 100644 teaweb/actions/default/ui/init.go diff --git a/teaconfigs/proxy_setting.go b/teaconfigs/proxy_setting.go new file mode 100644 index 0000000..f573fee --- /dev/null +++ b/teaconfigs/proxy_setting.go @@ -0,0 +1,62 @@ +package teaconfigs + +import ( + "github.com/TeaWeb/code/teaconfigs/shared" + "github.com/go-yaml/yaml" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/logs" + "io/ioutil" +) + +const proxySettingFile = "proxy_setting.conf" + +var sharedProxySetting *ProxySetting = nil + +// 代理全局设置 +type ProxySetting struct { + MatchDomainStrictly bool `yaml:"matchDomainStrictly" json:"matchDomainStrictly"` // 是否严格匹配域名 +} + +// 取得共享的代理设置 +func SharedProxySetting() *ProxySetting { + shared.Locker.Lock() + defer shared.Locker.ReadUnlock() + if sharedProxySetting == nil { + sharedProxySetting = LoadProxySetting() + } + return sharedProxySetting +} + +// 从配置文件中加载配置 +func LoadProxySetting() *ProxySetting { + setting := &ProxySetting{ + MatchDomainStrictly: false, + } + + data, err := ioutil.ReadFile(Tea.ConfigFile(proxySettingFile)) + if err != nil { + return setting + } + + err = yaml.Unmarshal(data, setting) + if err != nil { + logs.Error(err) + } + + return setting +} + +// 保存 +func (this *ProxySetting) Save() error { + data, err := yaml.Marshal(this) + if err != nil { + return err + } + err = ioutil.WriteFile(Tea.ConfigFile(proxySettingFile), data, 0666) + if err == nil { + shared.Locker.Lock() + sharedProxySetting = this + shared.Locker.ReadUnlock() + } + return err +} diff --git a/teaconfigs/proxy_setting_test.go b/teaconfigs/proxy_setting_test.go new file mode 100644 index 0000000..25e37bd --- /dev/null +++ b/teaconfigs/proxy_setting_test.go @@ -0,0 +1,26 @@ +package teaconfigs + +import ( + "github.com/iwind/TeaGo/logs" + "sync" + "testing" +) + +func TestSharedProxySetting(t *testing.T) { + logs.PrintAsJSON(SharedProxySetting(), t) +} + +func TestSharedProxySettingConcurrent(t *testing.T) { + wg := sync.WaitGroup{} + wg.Add(1000) + for i := 0; i < 1000; i++ { + go func() { + defer wg.Done() + setting := SharedProxySetting() + if setting == nil { + t.Log("nil") + } + }() + } + wg.Wait() +} diff --git a/teaproxy/listener.go b/teaproxy/listener.go index 4ac0323..78cb7ff 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -482,8 +482,11 @@ func (this *Listener) findNamedServer(name string) (serverConfig *teaconfigs.Ser // 只记录N个记录,防止内存耗尽 maxNamedServers := 10240 + // 是否严格匹配域名 + matchDomainStrictly := teaconfigs.SharedProxySetting().MatchDomainStrictly + // 如果只有一个server,则默认为这个 - if countServers == 1 { + if countServers == 1 && !matchDomainStrictly { server := this.currentServers[0] matchedName, matched := server.MatchName(name) if matched { @@ -541,6 +544,10 @@ func (this *Listener) findNamedServer(name string) (serverConfig *teaconfigs.Ser } // 如果没有找到,则匹配到第一个 + if matchDomainStrictly { + return nil, name + } + server := this.currentServers[0] firstName := server.FirstName() if len(firstName)> 0 { diff --git a/teaweb/actions/default/proxy/proxyutils/menu.go b/teaweb/actions/default/proxy/proxyutils/menu.go index cf30ec8..47db091 100644 --- a/teaweb/actions/default/proxy/proxyutils/menu.go +++ b/teaweb/actions/default/proxy/proxyutils/menu.go @@ -23,7 +23,7 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { var hasServer = false var isTCP = false - isIndex := !action.HasPrefix("/proxy/add", "/cache", "/proxy/waf", "/proxy/log/policies", "/proxy/certs") + isIndex := !action.HasPrefix("/proxy/add", "/cache", "/proxy/waf", "/proxy/log/policies", "/proxy/certs", "/proxy/settings") serverId := action.ParamString("serverId") serverList, err := teaconfigs.SharedServerList() @@ -112,6 +112,7 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { menu.Add("WAF策略", "", "/proxy/waf", action.HasPrefix("/proxy/waf")) menu.Add("日志策略", "", "/proxy/log/policies", action.HasPrefix("/proxy/log/policies")) menu.Add("SSL证书管理", "", "/proxy/certs", action.HasPrefix("/proxy/certs")) + menu.Add("通用设置", "", "/proxy/settings", action.HasPrefix("/proxy/settings")) } utils.SetSubMenu(action, menuGroup) diff --git a/teaweb/actions/default/proxy/settings/helper.go b/teaweb/actions/default/proxy/settings/helper.go new file mode 100644 index 0000000..378b519 --- /dev/null +++ b/teaweb/actions/default/proxy/settings/helper.go @@ -0,0 +1,15 @@ +package settings + +import ( + "github.com/iwind/TeaGo/actions" + "net/http" +) + +type Helper struct { +} + +func (this *Helper) BeforeAction(action *actions.ActionObject) { + if action.Request.Method != http.MethodGet { + return + } +} diff --git a/teaweb/actions/default/proxy/settings/index.go b/teaweb/actions/default/proxy/settings/index.go new file mode 100644 index 0000000..85a536c --- /dev/null +++ b/teaweb/actions/default/proxy/settings/index.go @@ -0,0 +1,14 @@ +package settings + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/iwind/TeaGo/actions" +) + +type IndexAction actions.Action + +func (this *IndexAction) RunGet(params struct{}) { + this.Data["setting"] = teaconfigs.LoadProxySetting() + + this.Show() +} diff --git a/teaweb/actions/default/proxy/settings/init.go b/teaweb/actions/default/proxy/settings/init.go new file mode 100644 index 0000000..28c56d4 --- /dev/null +++ b/teaweb/actions/default/proxy/settings/init.go @@ -0,0 +1,23 @@ +package settings + +import ( + "github.com/TeaWeb/code/teaweb/actions/default/proxy" + "github.com/TeaWeb/code/teaweb/configs" + "github.com/TeaWeb/code/teaweb/helpers" + "github.com/iwind/TeaGo" +) + +func init() { + TeaGo.BeforeStart(func(server *TeaGo.Server) { + server. + Prefix("/proxy/settings"). + Helper(&helpers.UserMustAuth{ + Grant: configs.AdminGrantProxy, + }). + Helper(new(Helper)). + Helper(new(proxy.Helper)). + Get("", new(IndexAction)). + GetPost("/update", new(UpdateAction)). + EndAll() + }) +} diff --git a/teaweb/actions/default/proxy/settings/update.go b/teaweb/actions/default/proxy/settings/update.go new file mode 100644 index 0000000..1fcb664 --- /dev/null +++ b/teaweb/actions/default/proxy/settings/update.go @@ -0,0 +1,32 @@ +package settings + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" + "github.com/iwind/TeaGo/actions" +) + +type UpdateAction actions.Action + +// 修改全局代理设置 +func (this *UpdateAction) RunGet(params struct{}) { + this.Data["setting"] = teaconfigs.LoadProxySetting() + this.Show() +} + +func (this *UpdateAction) RunPost(params struct { + MatchDomainStrictly bool + + Must *actions.Must +}) { + setting := teaconfigs.LoadProxySetting() + setting.MatchDomainStrictly = params.MatchDomainStrictly + err := setting.Save() + if err != nil { + this.Fail("保存失败:" + err.Error()) + } + + proxyutils.NotifyChange() + + this.Success() +} diff --git a/teaweb/actions/default/ui/components.go b/teaweb/actions/default/ui/components.go new file mode 100644 index 0000000..90d7b84 --- /dev/null +++ b/teaweb/actions/default/ui/components.go @@ -0,0 +1,39 @@ +package ui + +import ( + "bytes" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/files" + "github.com/iwind/TeaGo/logs" +) + +type ComponentsAction actions.Action + +func (this *ComponentsAction) RunGet(params struct{}) { + this.AddHeader("Content-Type", "text/javascript; charset=utf-8") + + // TODO 增加缓存 + + webRoot := Tea.Root + "/web/public/js/components/" + f := files.NewFile(webRoot) + + buf := bytes.NewBuffer([]byte{}) + f.Range(func(file *files.File) { + if !file.IsFile() { + return + } + if file.Ext() != ".js" { + return + } + data, err := file.ReadAll() + if err != nil { + logs.Error(err) + return + } + buf.Write(data) + buf.WriteByte('\n') + buf.WriteByte('\n') + }) + this.Write(buf.Bytes()) +} diff --git a/teaweb/actions/default/ui/init.go b/teaweb/actions/default/ui/init.go new file mode 100644 index 0000000..06b4a0d --- /dev/null +++ b/teaweb/actions/default/ui/init.go @@ -0,0 +1,16 @@ +package ui + +import ( + "github.com/iwind/TeaGo" + "github.com/iwind/TeaGo/actions" +) + +func init() { + TeaGo.BeforeStart(func(server *TeaGo.Server) { + server. + Prefix("/ui"). + Helper(new(actions.Gzip)). + Get("/components.js", new(ComponentsAction)). + EndAll() + }) +} diff --git a/teaweb/web.go b/teaweb/web.go index 1408552..08c3488 100644 --- a/teaweb/web.go +++ b/teaweb/web.go @@ -44,6 +44,7 @@ import ( _ "github.com/TeaWeb/code/teaweb/actions/default/proxy/notices" _ "github.com/TeaWeb/code/teaweb/actions/default/proxy/rewrite" _ "github.com/TeaWeb/code/teaweb/actions/default/proxy/servers" + _ "github.com/TeaWeb/code/teaweb/actions/default/proxy/settings" _ "github.com/TeaWeb/code/teaweb/actions/default/proxy/ssl" _ "github.com/TeaWeb/code/teaweb/actions/default/proxy/stat" _ "github.com/TeaWeb/code/teaweb/actions/default/proxy/tunnel" @@ -60,6 +61,7 @@ import ( _ "github.com/TeaWeb/code/teaweb/actions/default/settings/profile" _ "github.com/TeaWeb/code/teaweb/actions/default/settings/server" _ "github.com/TeaWeb/code/teaweb/actions/default/settings/update" + _ "github.com/TeaWeb/code/teaweb/actions/default/ui" "github.com/TeaWeb/code/teaweb/cmd" "github.com/TeaWeb/code/teaweb/utils" "github.com/iwind/TeaGo" From 2367c43a0929362e53a9dc7fa702f627c61a8780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月25日 17:09:39 +0800 Subject: [PATCH 064/121] =?UTF-8?q?[proxy]=E4=BB=A3=E7=90=86=E7=BD=91?= =?UTF-8?q?=E7=BB=9C=E5=9C=B0=E5=9D=80=E6=94=AF=E6=8C=81=E7=AB=AF=E5=8F=A3?= =?UTF-8?q?=E8=8C=83=E5=9B=B4=EF=BC=8C=E6=AF=94=E5=A6=82"0.0.0.0:[8100-820?= =?UTF-8?q?0]"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/server.go | 26 ++++++++++++++++++++++++ teaconfigs/server_test.go | 42 +++++++++++++++++++++++++++++++++++++++ teaproxy/manager.go | 2 +- 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/teaconfigs/server.go b/teaconfigs/server.go index da19a0b..8e623ca 100644 --- a/teaconfigs/server.go +++ b/teaconfigs/server.go @@ -15,8 +15,11 @@ import ( "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" + "github.com/iwind/TeaGo/types" "github.com/iwind/TeaGo/utils/string" "net/http" + "regexp" + "strconv" "strings" ) @@ -498,6 +501,29 @@ func (this *ServerConfig) AddListen(address string) { this.Listen = append(this.Listen, address) } +// 分解所有监听地址 +func (this *ServerConfig) ParseListenAddresses() []string { + result := []string{} + var reg = regexp.MustCompile(`:\s*\[\s*(\d+)\s*[,:-]\s*(\d+)\s*]$`) + for _, addr := range this.Listen { + match := reg.FindStringSubmatch(addr) + if len(match) == 0 { + result = append(result, addr) + } else { + min := types.Int(match[1]) + max := types.Int(match[2]) + if min> max { + min, max = max, min + } + for i := min; i <= max; i++ { + newAddr := reg.ReplaceAllString(addr, ":"+strconv.Itoa(i)) + result = append(result, newAddr) + } + } + } + return result +} + // 获取某个位置上的配置 func (this *ServerConfig) LocationAtIndex(index int) *LocationConfig { if index < 0 { diff --git a/teaconfigs/server_test.go b/teaconfigs/server_test.go index 9981be4..898782d 100644 --- a/teaconfigs/server_test.go +++ b/teaconfigs/server_test.go @@ -68,3 +68,45 @@ func TestServerConfig_Encode(t *testing.T) { t.Log(string(data)) } + +func TestServerConfig_ParseListenAddresses(t *testing.T) { + a := assert.NewAssertion(t) + + { + server := NewServerConfig() + server.AddListen("127.0.0.1:1234") + result := server.ParseListenAddresses() + t.Log(result) + a.IsTrue(len(result) == 1) + } + + { + server := NewServerConfig() + server.AddListen("127.0.0.1:[100-200]") + result := server.ParseListenAddresses() + t.Log(result) + a.IsTrue(len(result) == 101) + } + + { + server := NewServerConfig() + server.AddListen("127.0.0.1: [ 80 - 88 ]") + result := server.ParseListenAddresses() + t.Log(result) + a.IsTrue(len(result) == 9) + } + { + server := NewServerConfig() + server.AddListen("127.0.0.1:[80,88]") + result := server.ParseListenAddresses() + t.Log(result) + a.IsTrue(len(result) == 9) + } + { + server := NewServerConfig() + server.AddListen("127.0.0.1:[80:88]") + result := server.ParseListenAddresses() + t.Log(result) + a.IsTrue(len(result) == 9) + } +} diff --git a/teaproxy/manager.go b/teaproxy/manager.go index b24a788..c0fbb8d 100644 --- a/teaproxy/manager.go +++ b/teaproxy/manager.go @@ -131,7 +131,7 @@ func (this *Manager) ApplyServer(server *teaconfigs.ServerConfig) { if server.IsHTTP() { // HTTP // HTTP if server.Http { - for _, address := range server.Listen { + for _, address := range server.ParseListenAddresses() { // 是否有端口 if shared.RegexpDigitNumber.MatchString(address) { address = ":" + address From 16a08b392d94e4b67ec4fbf07c22cab948617f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月25日 17:46:02 +0800 Subject: [PATCH 065/121] =?UTF-8?q?[proxy]=E5=90=8E=E7=AB=AF=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E5=99=A8=E5=9C=B0=E5=9D=80=E4=B8=AD=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/backend.go | 10 ++++++++++ teaproxy/ftp_client_pool.go | 14 +++++++++++--- teaproxy/http_client_pool.go | 15 +++++++++++---- teaproxy/request_backend.go | 4 ++-- teaproxy/request_websocket.go | 6 +++++- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/teaconfigs/backend.go b/teaconfigs/backend.go index 3c4a9d3..12525d8 100644 --- a/teaconfigs/backend.go +++ b/teaconfigs/backend.go @@ -83,6 +83,8 @@ type BackendConfig struct { hasHost bool uniqueKey string + + hasAddrVariables bool // 地址中是否含有变量 } // 获取新对象 @@ -168,6 +170,9 @@ func (this *BackendConfig) Validate() error { // host this.hasHost = len(this.Host)> 0 + // variables + this.hasAddrVariables = shared.RegexpNamedVariable.MatchString(this.Address) + return nil } @@ -477,3 +482,8 @@ func (this *BackendConfig) IsTCP() bool { func (this *BackendConfig) IsFTP() bool { return this.Scheme == "ftp" } + +// 地址中是否含有变量 +func (this *BackendConfig) HasAddrVariables() bool { + return this.hasAddrVariables +} diff --git a/teaproxy/ftp_client_pool.go b/teaproxy/ftp_client_pool.go index bf3a21b..9e0fd0d 100644 --- a/teaproxy/ftp_client_pool.go +++ b/teaproxy/ftp_client_pool.go @@ -25,12 +25,18 @@ func NewFTPClientPool() *FTPClientPool { } // 通过Backend配置FTP客户端 -func (this *FTPClientPool) client(backend *teaconfigs.BackendConfig, location *teaconfigs.LocationConfig) *FTPClient { +func (this *FTPClientPool) client(req *Request, backend *teaconfigs.BackendConfig, location *teaconfigs.LocationConfig) *FTPClient { key := backend.UniqueKey() if location != nil { key = location.Id + "_" + key } + backendAddr := backend.Address + if backend.HasAddrVariables() { + backendAddr = req.Format(backend.Address) + key += "@" + backendAddr + } + this.locker.Lock() defer this.locker.Unlock() client, ok := this.clientMap[key] @@ -39,7 +45,9 @@ func (this *FTPClientPool) client(backend *teaconfigs.BackendConfig, location *t } // 关闭以前的连接 - this.closeOldClients(key) + if !backend.HasAddrVariables() { + this.closeOldClients(key) + } if backend.FTP == nil { backend.FTP = &teaconfigs.FTPBackendConfig{} @@ -53,7 +61,7 @@ func (this *FTPClientPool) client(backend *teaconfigs.BackendConfig, location *t } client = &FTPClient{ pool: &FTPConnectionPool{ - addr: backend.Address, + addr: backendAddr, username: backend.FTP.Username, password: backend.FTP.Password, dir: backend.FTP.Dir, diff --git a/teaproxy/http_client_pool.go b/teaproxy/http_client_pool.go index 02e7d7d..370bdaf 100644 --- a/teaproxy/http_client_pool.go +++ b/teaproxy/http_client_pool.go @@ -32,12 +32,18 @@ func NewHTTPClientPool() *HTTPClientPool { } // 根据地址获取客户端 -func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig, location *teaconfigs.LocationConfig) *http.Client { +func (this *HTTPClientPool) client(req *Request, backend *teaconfigs.BackendConfig, location *teaconfigs.LocationConfig) *http.Client { key := backend.UniqueKey() if location != nil { key = location.Id + "_" + key } + backendAddr := backend.Address + if backend.HasAddrVariables() { + backendAddr = req.Format(backend.Address) + key += "@" + backendAddr + } + this.locker.RLock() client, found := this.clientsMap[key] if found { @@ -49,7 +55,6 @@ func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig, location * maxConnections := int(backend.MaxConns) connectionTimeout := backend.FailTimeoutDuration() - address := backend.Address readTimeout := backend.ReadTimeoutDuration() idleTimeout := backend.IdleTimeoutDuration() idleConns := int(backend.IdleConns) @@ -98,7 +103,7 @@ func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig, location * return (&net.Dialer{ Timeout: connectionTimeout, KeepAlive: 2 * time.Minute, - }).DialContext(ctx, network, address) + }).DialContext(ctx, network, backendAddr) }, MaxIdleConns: 0, MaxIdleConnsPerHost: idleConns, @@ -121,7 +126,9 @@ func (this *HTTPClientPool) client(backend *teaconfigs.BackendConfig, location * this.clientsMap[key] = client // 关闭老的 - this.closeOldClient(key) + if !backend.HasAddrVariables() { + this.closeOldClient(key) + } this.locker.Unlock() diff --git a/teaproxy/request_backend.go b/teaproxy/request_backend.go index 1db9c3b..66f277e 100644 --- a/teaproxy/request_backend.go +++ b/teaproxy/request_backend.go @@ -113,10 +113,10 @@ func (this *Request) callBackend(writer *ResponseWriter) error { var resp *http.Response = nil var err error = nil if this.backend.IsFTP() { - client := SharedFTPClientPool.client(this.backend, this.location) + client := SharedFTPClientPool.client(this, this.backend, this.location) resp, err = client.Do(this.raw) } else { - client := SharedHTTPClientPool.client(this.backend, this.location) + client := SharedHTTPClientPool.client(this, this.backend, this.location) resp, err = client.Do(this.raw) } if err != nil { diff --git a/teaproxy/request_websocket.go b/teaproxy/request_websocket.go index d7b4b85..167b77c 100644 --- a/teaproxy/request_websocket.go +++ b/teaproxy/request_websocket.go @@ -112,9 +112,13 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { connectionTimeout = 15 * time.Second } + backendAddr := this.backend.Address + if this.backend.HasAddrVariables() { + backendAddr = this.Format(backendAddr) + } dialer := websocket.Dialer{ NetDial: func(network, addr string) (conn net.Conn, err error) { - return net.DialTimeout(network, this.backend.Address, connectionTimeout) + return net.DialTimeout(network, backendAddr, connectionTimeout) }, TLSClientConfig: tlsConfig, HandshakeTimeout: this.backend.FailTimeoutDuration(), From 0b3bde5e3518bfac325d8a3090cd197239ad2063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月25日 19:16:20 +0800 Subject: [PATCH 066/121] =?UTF-8?q?[proxy]=E5=A2=9E=E5=8A=A0${host.first}?= =?UTF-8?q?=E3=80=81${host.last}=E3=80=81${host.0}=E3=80=81${host.1}?= =?UTF-8?q?=E3=80=82=E3=80=82=E3=80=82=E7=AD=89=E8=AF=B7=E6=B1=82=E5=8F=98?= =?UTF-8?q?=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/ftp_client_test.go | 4 +-- teaproxy/request.go | 67 +++++++++++++++++++++++++++++++++++++ teaproxy/request_test.go | 36 ++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-) diff --git a/teaproxy/ftp_client_test.go b/teaproxy/ftp_client_test.go index 4294aa4..2214ec8 100644 --- a/teaproxy/ftp_client_test.go +++ b/teaproxy/ftp_client_test.go @@ -21,7 +21,7 @@ func TestFTPClient_Do(t *testing.T) { Dir: "", }, } - client := SharedFTPClientPool.client(backend) + client := SharedFTPClientPool.client(nil, backend, nil) for _, file := range []string{"/index.html", "index.a", "/dir1/dir2/hello.txt"} { func() { @@ -62,7 +62,7 @@ func TestFTPClient_Do_ChangeDir(t *testing.T) { Dir: "/dir1/dir2", }, } - client := SharedFTPClientPool.client(backend) + client := SharedFTPClientPool.client(nil, backend, nil) for _, file := range []string{"hello.txt", "hello1.txt", "hello2.txt"} { func() { diff --git a/teaproxy/request.go b/teaproxy/request.go index 9ef5c95..e36727a 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -1309,6 +1309,73 @@ func (this *Request) Format(source string) string { } } + // host + if prefix == "host" { + pieces := strings.Split(this.host, ".") + switch suffix { + case "first": + if len(pieces)> 0 { + return pieces[0] + } + return "" + case "last": + if len(pieces)> 0 { + return pieces[len(pieces)-1] + } + return "" + case "0": + if len(pieces)> 0 { + return pieces[0] + } + return "" + case "1": + if len(pieces)> 1 { + return pieces[1] + } + return "" + case "2": + if len(pieces)> 2 { + return pieces[2] + } + return "" + case "3": + if len(pieces)> 3 { + return pieces[3] + } + return "" + case "4": + if len(pieces)> 4 { + return pieces[4] + } + return "" + case "-1": + if len(pieces)> 0 { + return pieces[len(pieces)-1] + } + return "" + case "-2": + if len(pieces)> 1 { + return pieces[len(pieces)-2] + } + return "" + case "-3": + if len(pieces)> 2 { + return pieces[len(pieces)-3] + } + return "" + case "-4": + if len(pieces)> 3 { + return pieces[len(pieces)-4] + } + return "" + case "-5": + if len(pieces)> 4 { + return pieces[len(pieces)-5] + } + return "" + } + } + return "${" + varName + "}" }) } diff --git a/teaproxy/request_test.go b/teaproxy/request_test.go index e44d01f..870a8d1 100644 --- a/teaproxy/request_test.go +++ b/teaproxy/request_test.go @@ -222,6 +222,7 @@ func TestRequest_Format(t *testing.T) { req.method = "GET" req.filePath = "hello.go" req.scheme = "http" + req.host = "www.example.com" a.IsTrue(req.requestRemoteAddr() == "127.0.0.1") t.Log(req.requestRemotePort()) @@ -251,6 +252,41 @@ func TestRequest_Format(t *testing.T) { a.IsTrue(req.requestRemoteAddr() == "192.168.1.103") t.Log(req.Format("hello ${teaVersion} remoteAddr:${remoteAddr} name:${arg.name} header:${header.Content-Type} test:${test}")) + + { + req.host = "a.b.c.example.com" + t.Log("===") + t.Log(req.Format("host:${host} first:${host.first}, last:${host.last}, 0:${host.0}, 1:${host.1}, 2:${host.2}, 3:${host.3}, 4:${host.4}")) + t.Log(req.Format("-1:${host.-1} -2:${host.-2} -3:${host.-3} -4:${host.-4}")) + } + + { + req.host = "a.b.example.com" + t.Log("===") + t.Log(req.Format("host:${host} first:${host.first}, last:${host.last}, 0:${host.0}, 1:${host.1}, 2:${host.2}, 3:${host.3}, 4:${host.4}")) + t.Log(req.Format("-1:${host.-1} -2:${host.-2} -3:${host.-3} -4:${host.-4}")) + } + + { + req.host = "a.example.com" + t.Log("===") + t.Log(req.Format("host:${host} first:${host.first}, last:${host.last}, 0:${host.0}, 1:${host.1}, 2:${host.2}, 3:${host.3}, 4:${host.4}")) + t.Log(req.Format("-1:${host.-1} -2:${host.-2} -3:${host.-3} -4:${host.-4}")) + } + + { + req.host = "a.example.com" + t.Log("===") + t.Log(req.Format("host:${host} first:${host.first}, last:${host.last}, 0:${host.0}, 1:${host.1}, 2:${host.2}, 3:${host.3}, 4:${host.4}")) + t.Log(req.Format("-1:${host.-1} -2:${host.-2} -3:${host.-3} -4:${host.-4}")) + } + + { + req.host = "example.com" + t.Log("===") + t.Log(req.Format("host:${host} first:${host.first}, last:${host.last}, 0:${host.0}, 1:${host.1}, 2:${host.2}, 3:${host.3}, 4:${host.4}")) + t.Log(req.Format("-1:${host.-1} -2:${host.-2} -3:${host.-3} -4:${host.-4}")) + } } func TestRequest_FormatPerformance(t *testing.T) { From 952bc9ea5cfa02fd5d873536e5e75ca45e76dfda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月25日 20:53:49 +0800 Subject: [PATCH 067/121] =?UTF-8?q?[proxy]=E8=B7=AF=E5=BE=84=E8=A7=84?= =?UTF-8?q?=E5=88=99=E5=A2=9E=E5=8A=A0=E7=A6=81=E7=94=A8=E6=9D=A1=E4=BB=B6?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E6=98=AF=E5=90=A6=E7=A6=81=E6=AD=A2=E6=89=80?= =?UTF-8?q?=E6=9C=89=E8=AE=BF=E9=97=AE=E9=80=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/location.go | 29 ++++++++++++++++++- teaproxy/request.go | 17 +++++++++++ teaweb/actions/default/proxy/locations/add.go | 9 ++++++ .../actions/default/proxy/locations/detail.go | 2 ++ .../actions/default/proxy/locations/update.go | 11 +++++++ 5 files changed, 67 insertions(+), 1 deletion(-) diff --git a/teaconfigs/location.go b/teaconfigs/location.go index b1cdf07..e111f21 100644 --- a/teaconfigs/location.go +++ b/teaconfigs/location.go @@ -70,6 +70,10 @@ type LocationConfig struct { // - cond ${requestPath} regexp .*\.png Cond []*shared.RequestCond `yaml:"cond" json:"cond"` + // 禁止访问的条件 + DenyCond []*shared.RequestCond `yaml:"denyCond" json:"denyCond"` + DenyAll bool `yaml:"denyAll" json:"denyAll"` + IsBreak bool `yaml:"isBreak" json:"isBreak"` // 终止向下解析 Pages []*PageConfig `yaml:"pages" json:"pages"` // 特殊页 @@ -275,6 +279,13 @@ func (this *LocationConfig) Validate() error { } } + for _, cond := range this.DenyCond { + err := cond.Validate() + if err != nil { + return err + } + } + // request groups for _, group := range this.requestGroups { group.Backends = []*BackendConfig{} @@ -399,7 +410,7 @@ func (this *LocationConfig) IsCaseInsensitive() bool { } // 判断是否匹配路径 -func (this *LocationConfig) Match(path string, formatter func(source string) string) (map[string]string, bool) { +func (this *LocationConfig) Match(path string, formatter func(source string) string) (vars map[string]string, isMatched bool) { // 判断条件 if len(this.Cond)> 0 { for _, cond := range this.Cond { @@ -470,6 +481,22 @@ func (this *LocationConfig) Match(path string, formatter func(source string) str return nil, false } +// 判断是否屏蔽 +func (this *LocationConfig) IsDenied(formatter func(source string) string) bool { + if this.DenyAll { + return true + } + + if len(this.DenyCond)> 0 { + for _, cond := range this.DenyCond { + if cond.Match(formatter) { + return true + } + } + } + return false +} + // 组合参数为一个字符串 func (this *LocationConfig) SetPattern(pattern string, patternType int, caseInsensitive bool, reverse bool) { op := "" diff --git a/teaproxy/request.go b/teaproxy/request.go index e36727a..a70b330 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -133,6 +133,7 @@ type Request struct { locationContext *teaconfigs.LocationConfig // 当前变量的上下文 *Location ... hasForwardHeader bool + isDenied bool } // 获取新的请求 @@ -223,6 +224,7 @@ func (this *Request) reset(rawRequest *http.Request) { this.accessLog = nil this.locationContext = nil + this.isDenied = false this.gzip = nil this.debug = false @@ -338,6 +340,11 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b } this.locationContext = location if locationMatches, ok := location.Match(rawPath, this.Format); ok { + if location.IsDenied(this.Format) { + this.isDenied = true + return nil + } + this.addVarMapping(locationMatches) if len(location.Root)> 0 { @@ -815,6 +822,15 @@ func (this *Request) callBegin(writer *ResponseWriter) error { } } + // 禁止访问页面 + if this.isDenied { + if !this.callPage(writer, http.StatusForbidden) { + writer.WriteHeader(http.StatusForbidden) + _, _ = writer.Write([]byte("Request Forbidden")) + } + return nil + } + // 临时关闭页面 if this.shutdown != nil { return this.callShutdown(writer) @@ -847,6 +863,7 @@ func (this *Request) callBegin(writer *ResponseWriter) error { if len(this.root)> 0 { return this.callRoot(writer) } + return errors.New("unable to handle the request") } diff --git a/teaweb/actions/default/proxy/locations/add.go b/teaweb/actions/default/proxy/locations/add.go index 96a647e..25c98df 100644 --- a/teaweb/actions/default/proxy/locations/add.go +++ b/teaweb/actions/default/proxy/locations/add.go @@ -81,6 +81,7 @@ func (this *AddAction) RunPost(params struct { MaxBodyUnit string AccessLogIsInherited bool EnableStat bool + DenyAll bool // gzip GzipLevel int8 @@ -130,6 +131,14 @@ func (this *AddAction) RunPost(params struct { } location.Cond = conds + // 禁止条件 + denyConds, breakCond, err := proxyutils.ParseRequestConds(this.Request, "deny") + if err != nil { + this.Fail("禁止访问条件\"" + breakCond.Param + " " + breakCond.Operator + " " + breakCond.Value + "\"校验失败:" + err.Error()) + } + location.DenyCond = denyConds + location.DenyAll = params.DenyAll + location.SetPattern(params.Pattern, params.PatternType, params.IsCaseInsensitive, params.IsReverse) location.On = params.On location.IsBreak = params.IsBreak diff --git a/teaweb/actions/default/proxy/locations/detail.go b/teaweb/actions/default/proxy/locations/detail.go index 24f4533..d9e7f76 100644 --- a/teaweb/actions/default/proxy/locations/detail.go +++ b/teaweb/actions/default/proxy/locations/detail.go @@ -40,6 +40,8 @@ func (this *DetailAction) Run(params struct { "gzip": location.Gzip, "redirectToHttps": location.RedirectToHttps, "conds": location.Cond, + "denyConds": location.DenyCond, + "denyAll": location.DenyAll, "fastcgi": location.Fastcgi, "headers": location.Headers, diff --git a/teaweb/actions/default/proxy/locations/update.go b/teaweb/actions/default/proxy/locations/update.go index 71d415b..0196752 100644 --- a/teaweb/actions/default/proxy/locations/update.go +++ b/teaweb/actions/default/proxy/locations/update.go @@ -71,6 +71,8 @@ func (this *UpdateAction) Run(params struct { "enableStat": !location.DisableStat, "redirectToHttps": location.RedirectToHttps, "conds": location.Cond, + "denyConds": location.DenyCond, + "denyAll": location.DenyAll, // 菜单用 "rewrite": location.Rewrite, @@ -121,6 +123,7 @@ func (this *UpdateAction) RunPost(params struct { MaxBodyUnit string AccessLogIsInherited bool EnableStat bool + DenyAll bool GzipLevel int8 GzipMinLength float64 @@ -172,6 +175,14 @@ func (this *UpdateAction) RunPost(params struct { location.Cond = conds + // 禁止条件 + denyConds, breakCond, err := proxyutils.ParseRequestConds(this.Request, "deny") + if err != nil { + this.Fail("禁止访问条件\"" + breakCond.Param + " " + breakCond.Operator + " " + breakCond.Value + "\"校验失败:" + err.Error()) + } + location.DenyCond = denyConds + location.DenyAll = params.DenyAll + location.SetPattern(params.Pattern, params.PatternType, params.IsCaseInsensitive, params.IsReverse) location.On = params.On location.IsBreak = params.IsBreak From 834736b71d4532418cb9072fb07c8ed5b4bc996b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月25日 21:01:19 +0800 Subject: [PATCH 068/121] =?UTF-8?q?[proxy]=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E8=AF=B7=E6=B1=82Header=E4=BF=9D=E7=95=99=E5=8E=9F=E6=9C=89?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_backend.go | 4 ++-- teaproxy/request_url.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/teaproxy/request_backend.go b/teaproxy/request_backend.go index 66f277e..8fdfdb0 100644 --- a/teaproxy/request_backend.go +++ b/teaproxy/request_backend.go @@ -96,9 +96,9 @@ func (this *Request) callBackend(writer *ResponseWriter) error { continue } if header.HasVariables() { - this.raw.Header.Set(header.Name, this.Format(header.Value)) + this.raw.Header[header.Name] = []string{this.Format(header.Value)} } else { - this.raw.Header.Set(header.Name, header.Value) + this.raw.Header[header.Name] = []string{header.Value} } // 支持修改Host diff --git a/teaproxy/request_url.go b/teaproxy/request_url.go index 311df1f..82aef85 100644 --- a/teaproxy/request_url.go +++ b/teaproxy/request_url.go @@ -30,9 +30,9 @@ func (this *Request) callURL(writer *ResponseWriter, method string, url string, continue } if header.HasVariables() { - req.Header.Set(header.Name, this.Format(header.Value)) + req.Header[header.Name] = []string{this.Format(header.Value)} } else { - req.Header.Set(header.Name, header.Value) + req.Header[header.Name] = []string{header.Value} } } } From aca9e1d12bca327bd02f76af983b90fa07ebca2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月25日 21:33:05 +0800 Subject: [PATCH 069/121] =?UTF-8?q?[proxy]=E4=BF=AE=E6=94=B9=E7=BB=86?= =?UTF-8?q?=E8=8A=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/shared/header_list.go | 2 +- teaproxy/request_url.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/teaconfigs/shared/header_list.go b/teaconfigs/shared/header_list.go index 2de2605..20ce56a 100644 --- a/teaconfigs/shared/header_list.go +++ b/teaconfigs/shared/header_list.go @@ -61,7 +61,7 @@ type HeaderList struct { // 忽略的响应Headers IgnoreHeaders []string `yaml:"ignoreHeaders" json:"ignoreHeaders"` - // 请求Headers + // 自定义请求Headers RequestHeaders []*HeaderConfig `yaml:"requestHeaders" json:"requestHeaders"` hasResponseHeaders bool diff --git a/teaproxy/request_url.go b/teaproxy/request_url.go index 82aef85..cdf65dd 100644 --- a/teaproxy/request_url.go +++ b/teaproxy/request_url.go @@ -23,6 +23,9 @@ func (this *Request) callURL(writer *ResponseWriter, method string, url string, // 添加当前Header req.Header = this.raw.Header + // 代理头部 + this.setProxyHeaders(req.Header) + // 自定义请求Header if len(this.requestHeaders)> 0 { for _, header := range this.requestHeaders { @@ -37,9 +40,6 @@ func (this *Request) callURL(writer *ResponseWriter, method string, url string, } } - // 代理头部 - this.setProxyHeaders(req.Header) - var client = teautils.SharedHttpClient(60 * time.Second) resp, err := client.Do(req) if err != nil { From 8e2a4173620aef3a210d157280785d4249a44dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月26日 15:40:32 +0800 Subject: [PATCH 070/121] =?UTF-8?q?[waf]=E5=A2=9E=E5=8A=A0=E6=8B=A6?= =?UTF-8?q?=E6=88=AA=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teadb/dao_access_log.go | 9 ++ teadb/dao_access_log_mongo.go | 70 ++++++++++ teadb/dao_access_log_test.go | 18 +++ teadb/dao_acess_log_sql.go | 74 ++++++++++ teaproxy/request.go | 2 + teaproxy/request_waf.go | 20 +-- teawaf/waf.go | 32 ++--- teaweb/actions/default/proxy/waf/day.go | 142 ++++++++++++++++++++ teaweb/actions/default/proxy/waf/helper.go | 2 + teaweb/actions/default/proxy/waf/history.go | 70 ++++++++++ teaweb/actions/default/proxy/waf/init.go | 2 + 11 files changed, 417 insertions(+), 24 deletions(-) create mode 100644 teaweb/actions/default/proxy/waf/day.go create mode 100644 teaweb/actions/default/proxy/waf/history.go diff --git a/teadb/dao_access_log.go b/teadb/dao_access_log.go index 9713cc0..19631d2 100644 --- a/teadb/dao_access_log.go +++ b/teadb/dao_access_log.go @@ -39,6 +39,15 @@ type AccessLogDAOInterface interface { // 判断某个代理服务是否有日志 HasAccessLog(day string, serverId string) (bool, error) + // 列出WAF日志 + ListAccessLogsWithWAF(day string, wafId string, fromId string, onlyErrors bool, searchIP string, offset int, size int) ([]*accesslogs.AccessLog, error) + + // 检查WAF是否有下一条日志 + HasNextAccessLogWithWAF(day string, wafId string, fromId string, onlyErrors bool, searchIP string) (bool, error) + + // 判断某日是否有WAF日志 + HasAccessLogWithWAF(day string, wafId string) (bool, error) + // 列出最近的某些日志 ListLatestAccessLogs(day string, serverId string, fromId string, onlyErrors bool, size int) ([]*accesslogs.AccessLog, error) diff --git a/teadb/dao_access_log_mongo.go b/teadb/dao_access_log_mongo.go index f88abbe..123eeac 100644 --- a/teadb/dao_access_log_mongo.go +++ b/teadb/dao_access_log_mongo.go @@ -178,6 +178,76 @@ func (this *MongoAccessLogDAO) HasAccessLog(day string, serverId string) (bool, return one != nil, err } +func (this *MongoAccessLogDAO) ListAccessLogsWithWAF(day string, wafId string, fromId string, onlyErrors bool, searchIP string, offset int, size int) ([]*accesslogs.AccessLog, error) { + query := NewQuery(this.TableName(day)) + query.Attr("attrs.waf_id", wafId) + if len(fromId)> 0 { + fromIdObject, err := shared.ObjectIdFromHex(fromId) + if err != nil { + return nil, err + } + query.Lt("_id", fromIdObject) + } + if onlyErrors { + query.Or([]*OperandList{ + NewOperandList().Add("hasErrors", NewOperand(OperandEq, true)), + NewOperandList().Add("status", NewOperand(OperandGte, 400)), + }) + } + if len(searchIP)> 0 { + query.Attr("remoteAddr", searchIP) + } + query.Offset(offset) + query.Limit(size) + query.Desc("_id") + ones, err := query.FindOnes(new(accesslogs.AccessLog)) + if err != nil { + return nil, err + } + + result := []*accesslogs.AccessLog{} + for _, one := range ones { + result = append(result, one.(*accesslogs.AccessLog)) + } + return result, nil +} + +func (this *MongoAccessLogDAO) HasNextAccessLogWithWAF(day string, wafId string, fromId string, onlyErrors bool, searchIP string) (bool, error) { + query := NewQuery(this.TableName(day)) + query.Attr("attrs.waf_id", wafId). + Result("_id") + if len(fromId)> 0 { + fromIdObject, err := shared.ObjectIdFromHex(fromId) + if err != nil { + return false, err + } + query.Lt("_id", fromIdObject) + } + if onlyErrors { + query.Or([]*OperandList{ + NewOperandList().Add("hasErrors", NewOperand(OperandEq, true)), + NewOperandList().Add("status", NewOperand(OperandGte, 400)), + }) + } + if len(searchIP)> 0 { + query.Attr("remoteAddr", searchIP) + } + + one, err := query.FindOne(new(accesslogs.AccessLog)) + if err != nil { + return false, err + } + return one != nil, nil +} + +func (this *MongoAccessLogDAO) HasAccessLogWithWAF(day string, wafId string) (bool, error) { + query := NewQuery(this.TableName(day)) + one, err := query.Attr("attrs.waf_id", wafId). + Result("_id"). + FindOne(new(accesslogs.AccessLog)) + return one != nil, err +} + func (this *MongoAccessLogDAO) ListLatestAccessLogs(day string, serverId string, fromId string, onlyErrors bool, size int) ([]*accesslogs.AccessLog, error) { query := NewQuery(this.TableName(day)) diff --git a/teadb/dao_access_log_test.go b/teadb/dao_access_log_test.go index e082ff7..8c1f535 100644 --- a/teadb/dao_access_log_test.go +++ b/teadb/dao_access_log_test.go @@ -195,6 +195,24 @@ func TestAccessLogDAO_HasAccessLog(t *testing.T) { } } +func TestAccessLogDAO_HasAccessLogWithWAF(t *testing.T) { + { + b, err := AccessLogDAO().HasAccessLogWithWAF(timeutil.Format("Ymd"), "pq6HzRfIjcGsUqNe") + if err != nil { + t.Fatal(err) + } + t.Log(b) + } + + { + b, err := AccessLogDAO().HasAccessLogWithWAF(timeutil.Format("Ymd"), "pq6HzRfIjcGsU123") + if err != nil { + t.Fatal(err) + } + t.Log(b) + } +} + func TestAccessLogDAO_ListLatestAccessLogs(t *testing.T) { dao := AccessLogDAO() { diff --git a/teadb/dao_acess_log_sql.go b/teadb/dao_acess_log_sql.go index ee59944..5348bf3 100644 --- a/teadb/dao_acess_log_sql.go +++ b/teadb/dao_acess_log_sql.go @@ -175,6 +175,80 @@ func (this *SQLAccessLogDAO) HasAccessLog(day string, serverId string) (bool, er return one != nil, err } +// 列出WAF日志 +func (this *SQLAccessLogDAO) ListAccessLogsWithWAF(day string, wafId string, fromId string, onlyErrors bool, searchIP string, offset int, size int) ([]*accesslogs.AccessLog, error) { + query := NewQuery(this.TableName(day)) + query.Attr(this.driver.(SQLDriverInterface).JSONExtract("attrs", "waf_id"), wafId) + if len(fromId)> 0 { + query.Lt("_id", fromId) + } + if onlyErrors { + query.Or([]*OperandList{ + NewOperandList().Add("hasErrors", NewOperand(OperandEq, 1)), + NewOperandList().Add("status", NewOperand(OperandGte, 400)), + }) + } + if len(searchIP)> 0 { + query.Attr("remoteAddr", searchIP) + } + query.Offset(offset) + query.Limit(size) + query.Desc("_id") + ones, err := query.FindOnes(new(accesslogs.AccessLog)) + if err != nil { + if this.tableNotFound(err) { + return nil, nil + } + return nil, err + } + + result := []*accesslogs.AccessLog{} + for _, one := range ones { + result = append(result, one.(*accesslogs.AccessLog)) + } + return result, nil +} + +// 检查是否有下一条日志 +func (this *SQLAccessLogDAO) HasNextAccessLogWithWAF(day string, wafId string, fromId string, onlyErrors bool, searchIP string) (bool, error) { + query := NewQuery(this.TableName(day)) + query.Attr(this.driver.(SQLDriverInterface).JSONExtract("attrs", "waf_id"), wafId). + Result("_id") + if len(fromId)> 0 { + query.Lt("_id", fromId) + } + if onlyErrors { + query.Or([]*OperandList{ + NewOperandList().Add("hasErrors", NewOperand(OperandEq, 1)), + NewOperandList().Add("status", NewOperand(OperandGte, 400)), + }) + } + if len(searchIP)> 0 { + query.Attr("remoteAddr", searchIP) + } + + one, err := query.FindOne(new(accesslogs.AccessLog)) + if err != nil { + if this.tableNotFound(err) { + return false, nil + } + return false, err + } + return one != nil, nil +} + +// 判断某个WAF是否有日志 +func (this *SQLAccessLogDAO) HasAccessLogWithWAF(day string, wafId string) (bool, error) { + query := NewQuery(this.TableName(day)) + one, err := query.Attr(this.driver.(SQLDriverInterface).JSONExtract("attrs", "waf_id"), wafId). + Result("_id"). + FindOne(new(accesslogs.AccessLog)) + if err != nil && this.tableNotFound(err) { + return false, nil + } + return one != nil, err +} + // 列出最近的某些日志 func (this *SQLAccessLogDAO) ListLatestAccessLogs(day string, serverId string, fromId string, onlyErrors bool, size int) ([]*accesslogs.AccessLog, error) { query := NewQuery(this.TableName(day)) diff --git a/teaproxy/request.go b/teaproxy/request.go index a70b330..6c5ca7d 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -1399,6 +1399,8 @@ func (this *Request) Format(source string) string { // 设置属性 func (this *Request) SetAttr(key string, value string) { + // 需要处理key中的点(.)符号,因为很多数据库不支持在key中含有点 + key = strings.Replace(key, ".", "_", -1) this.attrs[key] = value } diff --git a/teaproxy/request_waf.go b/teaproxy/request_waf.go index 4563599..e506008 100644 --- a/teaproxy/request_waf.go +++ b/teaproxy/request_waf.go @@ -11,7 +11,7 @@ func (this *Request) callWAFRequest(writer *ResponseWriter) (blocked bool) { if this.waf == nil { return } - goNext, ruleSet, err := this.waf.MatchRequest(this.raw, writer) + goNext, group, ruleSet, err := this.waf.MatchRequest(this.raw, writer) if err != nil { logs.Error(err) return @@ -19,9 +19,11 @@ func (this *Request) callWAFRequest(writer *ResponseWriter) (blocked bool) { if ruleSet != nil { if ruleSet.Action != actions.ActionAllow { - this.SetAttr("waf.action", ruleSet.Action) - this.SetAttr("waf.ruleset", ruleSet.Name) - this.SetAttr("waf.id", this.waf.Id) + this.SetAttr("waf_action", ruleSet.Action) + this.SetAttr("waf_group", group.Id) + this.SetAttr("waf_ruleset", ruleSet.Id) + this.SetAttr("waf_ruleset_name", ruleSet.Name) + this.SetAttr("waf_id", this.waf.Id) } } @@ -34,7 +36,7 @@ func (this *Request) callWAFResponse(resp *http.Response, writer *ResponseWriter return } - goNext, ruleSet, err := this.waf.MatchResponse(this.raw, resp, writer) + goNext, group, ruleSet, err := this.waf.MatchResponse(this.raw, resp, writer) if err != nil { logs.Error(err) return @@ -42,9 +44,11 @@ func (this *Request) callWAFResponse(resp *http.Response, writer *ResponseWriter if ruleSet != nil { if ruleSet.Action != actions.ActionAllow { - this.SetAttr("waf.action", ruleSet.Action) - this.SetAttr("waf.ruleset", ruleSet.Name) - this.SetAttr("waf.id", this.waf.Id) + this.SetAttr("waf_action", ruleSet.Action) + this.SetAttr("waf_group", group.Id) + this.SetAttr("waf_ruleset", ruleSet.Id) + this.SetAttr("waf_ruleset_name", ruleSet.Name) + this.SetAttr("waf_id", this.waf.Id) } } diff --git a/teawaf/waf.go b/teawaf/waf.go index c5e46fb..991eab5 100644 --- a/teawaf/waf.go +++ b/teawaf/waf.go @@ -252,9 +252,9 @@ func (this *WAF) MoveOutboundRuleGroup(fromIndex int, toIndex int) { this.Outbound = result } -func (this *WAF) MatchRequest(rawReq *http.Request, writer http.ResponseWriter) (goNext bool, set *rules.RuleSet, err error) { +func (this *WAF) MatchRequest(rawReq *http.Request, writer http.ResponseWriter) (goNext bool, group *rules.RuleGroup, set *rules.RuleSet, err error) { if !this.hasInboundRules { - return true, nil, nil + return true, nil, nil, nil } req := requests.NewRequest(rawReq) for _, group := range this.Inbound { @@ -263,32 +263,32 @@ func (this *WAF) MatchRequest(rawReq *http.Request, writer http.ResponseWriter) } b, set, err := group.MatchRequest(req) if err != nil { - return true, nil, err + return true, nil, nil, err } if b { if this.onActionCallback == nil { if set.Action == actions.ActionBlock && this.ActionBlock != nil { - return this.ActionBlock.Perform(rawReq, writer), set, nil + return this.ActionBlock.Perform(rawReq, writer), group, set, nil } else { actionObject := actions.FindActionInstance(set.Action) if actionObject == nil { - return true, set, errors.New("no action called '" + set.Action + "'") + return true, group, set, errors.New("no action called '" + set.Action + "'") } goNext := actionObject.Perform(rawReq, writer) - return goNext, set, nil + return goNext, group, set, nil } } else { goNext = this.onActionCallback(set.Action) } - return goNext, set, nil + return goNext, group, set, nil } } - return true, nil, nil + return true, nil, nil, nil } -func (this *WAF) MatchResponse(rawReq *http.Request, rawResp *http.Response, writer http.ResponseWriter) (goNext bool, set *rules.RuleSet, err error) { +func (this *WAF) MatchResponse(rawReq *http.Request, rawResp *http.Response, writer http.ResponseWriter) (goNext bool, group *rules.RuleGroup, set *rules.RuleSet, err error) { if !this.hasOutboundRules { - return true, nil, nil + return true, nil, nil, nil } req := requests.NewRequest(rawReq) resp := requests.NewResponse(rawResp) @@ -298,27 +298,27 @@ func (this *WAF) MatchResponse(rawReq *http.Request, rawResp *http.Response, wri } b, set, err := group.MatchResponse(req, resp) if err != nil { - return true, nil, err + return true, nil, nil, err } if b { if this.onActionCallback == nil { if set.Action == actions.ActionBlock && this.ActionBlock != nil { - return this.ActionBlock.Perform(rawReq, writer), set, nil + return this.ActionBlock.Perform(rawReq, writer), group, set, nil } else { actionObject := actions.FindActionInstance(set.Action) if actionObject == nil { - return true, set, errors.New("no action called '" + set.Action + "'") + return true, group, set, errors.New("no action called '" + set.Action + "'") } goNext := actionObject.Perform(rawReq, writer) - return goNext, set, nil + return goNext, group, set, nil } } else { goNext = this.onActionCallback(set.Action) } - return goNext, set, nil + return goNext, group, set, nil } } - return true, nil, nil + return true, nil, nil, nil } // save to file path diff --git a/teaweb/actions/default/proxy/waf/day.go b/teaweb/actions/default/proxy/waf/day.go new file mode 100644 index 0000000..5c794b2 --- /dev/null +++ b/teaweb/actions/default/proxy/waf/day.go @@ -0,0 +1,142 @@ +package waf + +import ( + "fmt" + "github.com/TeaWeb/code/teaconfigs" + "github.com/TeaWeb/code/teadb" + "github.com/TeaWeb/code/tealogs/accesslogs" + "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/lists" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" + "github.com/iwind/TeaGo/utils/time" + "net/http" + "regexp" + "time" +) + +type DayAction actions.Action + +// 某天的日志 +func (this *DayAction) Run(params struct { + WafId string + Day string + LogType string + FromId string + Page int + Size int + SearchIP string +}) { + waf := teaconfigs.SharedWAFList().FindWAF(params.WafId) + if waf == nil { + this.Fail("找不到WAF") + } + + this.Data["config"] = maps.Map{ + "id": waf.Id, + "name": waf.Name, + "countInbound": waf.CountInboundRuleSets(), + "countOutbound": waf.CountOutboundRuleSets(), + "on": waf.On, + "actionBlock": waf.ActionBlock, + "cond": waf.Cond, + } + + if params.Page < 1 { + params.Page = 1 + } + if params.Size < 1 { + params.Size = 20 + } + + this.Data["searchIP"] = params.SearchIP + + // 检查数据库连接 + this.Data["mongoError"] = "" + err := teadb.SharedDB().Test() + mongoAvailable := true + if err != nil { + this.Data["mongoError"] = "此功能需要连接数据库" + mongoAvailable = false + } + + this.Data["day"] = params.Day + this.Data["isHistory"] = regexp.MustCompile("^\\d+$").MatchString(params.Day) + this.Data["logType"] = params.LogType + this.Data["logs"] = []interface{}{} + this.Data["fromId"] = params.FromId + this.Data["hasNext"] = false + this.Data["page"] = params.Page + + // 日志列表 + if mongoAvailable { + realDay := "" + if regexp.MustCompile("^\\d+$").MatchString(params.Day) { + realDay = params.Day + } else if params.Day == "today" { + realDay = timeutil.Format("Ymd") + } else if params.Day == "yesterday" { + realDay = timeutil.Format("Ymd", time.Now().Add(-24*time.Hour)) + } else { + realDay = timeutil.Format("Ymd") + } + + accessLogList, err := teadb.AccessLogDAO().ListAccessLogsWithWAF(realDay, waf.Id, params.FromId, params.LogType == "errorLog", params.SearchIP, params.Size*(params.Page-1), params.Size) + + if err != nil { + this.Data["mongoError"] = "数据库查询错误:" + err.Error() + } else { + result := lists.Map(accessLogList, func(k int, v interface{}) interface{} { + accessLog := v.(*accesslogs.AccessLog) + return map[string]interface{}{ + "id": accessLog.Id.Hex(), + "requestTime": accessLog.RequestTime, + "request": accessLog.Request, + "requestURI": accessLog.RequestURI, + "requestMethod": accessLog.RequestMethod, + "remoteAddr": accessLog.RemoteAddr, + "remotePort": accessLog.RemotePort, + "userAgent": accessLog.UserAgent, + "host": accessLog.Host, + "status": accessLog.Status, + "statusMessage": fmt.Sprintf("%d", accessLog.Status) + " " + http.StatusText(accessLog.Status), + "timeISO8601": accessLog.TimeISO8601, + "timeLocal": accessLog.TimeLocal, + "requestScheme": accessLog.Scheme, + "proto": accessLog.Proto, + "contentType": accessLog.SentContentType(), + "bytesSent": accessLog.BytesSent, + "backendAddress": accessLog.BackendAddress, + "fastcgiAddress": accessLog.FastcgiAddress, + "extend": accessLog.Extend, + "referer": accessLog.Referer, + "upgrade": accessLog.GetHeader("Upgrade"), + "day": timeutil.Format("Ymd", accessLog.Time()), + "errors": accessLog.Errors, + "attrs": accessLog.Attrs, + } + }) + + this.Data["logs"] = result + + if len(result)> 0 { + if len(params.FromId) == 0 { + fromId := accessLogList[0].Id.Hex() + this.Data["fromId"] = fromId + } + + { + nextId := accessLogList[len(accessLogList)-1].Id.Hex() + b, err := teadb.AccessLogDAO().HasNextAccessLogWithWAF(realDay, waf.Id, nextId, params.LogType == "errorLog", params.SearchIP) + if err != nil { + logs.Error(err) + } else { + this.Data["hasNext"] = b + } + } + } + } + } + + this.Show() +} diff --git a/teaweb/actions/default/proxy/waf/helper.go b/teaweb/actions/default/proxy/waf/helper.go index 429b99a..4ce7623 100644 --- a/teaweb/actions/default/proxy/waf/helper.go +++ b/teaweb/actions/default/proxy/waf/helper.go @@ -28,6 +28,8 @@ func (this *Helper) BeforeAction(action *actions.ActionObject) { action.Data["selectedSubMenu"] = "export" } else if action.HasPrefix("/proxy/waf/import") { action.Data["selectedSubMenu"] = "import" + } else if action.HasPrefix("/proxy/waf/history") || action.HasPrefix("/proxy/waf/day") { + action.Data["selectedSubMenu"] = "history" } action.Data["inbound"] = false diff --git a/teaweb/actions/default/proxy/waf/history.go b/teaweb/actions/default/proxy/waf/history.go new file mode 100644 index 0000000..e94f61e --- /dev/null +++ b/teaweb/actions/default/proxy/waf/history.go @@ -0,0 +1,70 @@ +package waf + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/TeaWeb/code/teadb" + "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" + timeutil "github.com/iwind/TeaGo/utils/time" + "time" +) + +type HistoryAction actions.Action + +func (this *HistoryAction) RunGet(params struct { + WafId string +}) { + waf := teaconfigs.SharedWAFList().FindWAF(params.WafId) + if waf == nil { + this.Fail("找不到WAF") + } + + this.Data["config"] = maps.Map{ + "id": waf.Id, + "name": waf.Name, + "countInbound": waf.CountInboundRuleSets(), + "countOutbound": waf.CountOutboundRuleSets(), + "on": waf.On, + "actionBlock": waf.ActionBlock, + "cond": waf.Cond, + } + + // 检查MongoDB连接 + this.Data["mongoError"] = "" + err := teadb.SharedDB().Test() + mongoAvailable := true + if err != nil { + this.Data["mongoError"] = "此功能需要连接数据库" + mongoAvailable = false + } + + // 列出最近30天的日志 + days := []maps.Map{} + if mongoAvailable { + for i := 0; i < 60; i++ { + day := timeutil.Format("Ymd", time.Now().Add(time.Duration(-i*24)*time.Hour)) + + b, err := teadb.AccessLogDAO().HasAccessLogWithWAF(day, waf.Id) + if err != nil { + logs.Error(err) + } + if b { + days = append(days, maps.Map{ + "day": day, + "has": true, + }) + } else { + days = append(days, maps.Map{ + "day": day, + "has": false, + }) + } + } + } + + this.Data["days"] = days + this.Data["today"] = timeutil.Format("Ymd") + + this.Show() +} diff --git a/teaweb/actions/default/proxy/waf/init.go b/teaweb/actions/default/proxy/waf/init.go index 4f5c60c..ee18b23 100644 --- a/teaweb/actions/default/proxy/waf/init.go +++ b/teaweb/actions/default/proxy/waf/init.go @@ -35,6 +35,8 @@ func init() { Post("/group/rule/delete", new(RuleDeleteAction)). Post("/group/rule/move", new(RuleMoveAction)). GetPost("/group/rule/update", new(RuleUpdateAction)). + Get("/history", new(HistoryAction)). + Get("/day", new(DayAction)). EndAll() }) } From ec1b5e85f3c81ab0031dd05ad88045635db21069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月26日 16:37:38 +0800 Subject: [PATCH 071/121] =?UTF-8?q?[waf]=E5=A2=9E=E5=8A=A0=E6=8B=A6?= =?UTF-8?q?=E6=88=AA=E7=B1=BB=E5=9E=8B=E7=BB=9F=E8=AE=A1=E9=A1=B9=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teastats/init.go | 1 + teastats/waf_block_all_period.go | 105 +++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 teastats/waf_block_all_period.go diff --git a/teastats/init.go b/teastats/init.go index 3979dcc..b0bb090 100644 --- a/teastats/init.go +++ b/teastats/init.go @@ -39,6 +39,7 @@ func init() { new(RegionAllPeriodFilter), new(ProvinceAllPeriodFilter), new(CityAllPeriodFilter), + new(WAFBlockAllPeriodFilter), ) // 注册AccessLogHook diff --git a/teastats/waf_block_all_period.go b/teastats/waf_block_all_period.go new file mode 100644 index 0000000..7a78846 --- /dev/null +++ b/teastats/waf_block_all_period.go @@ -0,0 +1,105 @@ +package teastats + +import ( + "github.com/TeaWeb/code/tealogs/accesslogs" + "github.com/TeaWeb/code/teawaf/actions" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" + "strings" +) + +// WAF拦截统计 +type WAFBlockAllPeriodFilter struct { + CounterFilter +} + +func (this *WAFBlockAllPeriodFilter) Name() string { + return "WAF拦截类型统计" +} + +func (this *WAFBlockAllPeriodFilter) Description() string { + return "所有WAF拦截类型统计统计" +} + +// 参数说明 +func (this *WAFBlockAllPeriodFilter) ParamVariables() []*Variable { + return []*Variable{ + NewVariable("wafId", "WAF ID"), + NewVariable("ruleSetId", "规则集ID"), + NewVariable("ruleSetName", "规则集名称"), + } +} + +// 统计数据说明 +func (this *WAFBlockAllPeriodFilter) ValueVariables() []*Variable { + return []*Variable{ + NewVariable("count", "匹配的数量"), + } +} + +func (this *WAFBlockAllPeriodFilter) Codes() []string { + return []string{ + "waf.block.all.second", + "waf.block.all.minute", + "waf.block.all.hour", + "waf.block.all.day", + "waf.block.all.week", + "waf.block.all.month", + "waf.block.all.year", + } +} + +func (this *WAFBlockAllPeriodFilter) Indexes() []string { + return []string{"wafId", "ruleSetId", "ruleSetName"} +} + +func (this *WAFBlockAllPeriodFilter) Start(queue *Queue, code string) { + if queue == nil { + logs.Println("stat queue should be specified for '" + code + "'") + return + } + this.queue = queue + this.queue.Index(this.Indexes()) + this.StartFilter(code, code[strings.LastIndex(code, ".")+1:]) +} + +func (this *WAFBlockAllPeriodFilter) Filter(accessLog *accesslogs.AccessLog) { + if accessLog.Attrs == nil { + return + } + + wafAction, ok := accessLog.Attrs["waf_action"] + if !ok { + return + } + if wafAction != actions.ActionBlock { + return + } + + wafId, ok := accessLog.Attrs["waf_id"] + if !ok { + return + } + + ruleSetId, ok := accessLog.Attrs["waf_ruleset"] + if !ok { + return + } + + ruleSetName, ok := accessLog.Attrs["waf_ruleset_name"] + if !ok { + return + } + + this.ApplyFilter(accessLog, map[string]string{ + "wafId": wafId, + "ruleSetId": ruleSetId, + "ruleSetName": ruleSetName, + }, maps.Map{ + "count": 1, + }) +} + +func (this *WAFBlockAllPeriodFilter) Stop() { + this.StopFilter() +} From 5844ae119aa3e7be356cd776e45245982d753317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月26日 19:48:43 +0800 Subject: [PATCH 072/121] =?UTF-8?q?[waf]=E6=8B=A6=E6=88=AA=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E4=B8=AD=E5=A2=9E=E5=8A=A0=E6=8B=A6=E6=88=AA=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teadb/dao_access_log.go | 4 +++ teadb/dao_access_log_mongo.go | 38 +++++++++++++++++++++++++ teadb/dao_access_log_test.go | 8 ++++++ teadb/dao_acess_log_sql.go | 38 +++++++++++++++++++++++++ teadb/driver_mongo.go | 2 ++ teaweb/actions/default/proxy/waf/day.go | 9 ++++++ 6 files changed, 99 insertions(+) diff --git a/teadb/dao_access_log.go b/teadb/dao_access_log.go index 19631d2..71fe911 100644 --- a/teadb/dao_access_log.go +++ b/teadb/dao_access_log.go @@ -2,6 +2,7 @@ package teadb import ( "github.com/TeaWeb/code/tealogs/accesslogs" + "github.com/iwind/TeaGo/maps" ) // 访问日志DAO @@ -48,6 +49,9 @@ type AccessLogDAOInterface interface { // 判断某日是否有WAF日志 HasAccessLogWithWAF(day string, wafId string) (bool, error) + // 统计当前WAF拦截的规则分组 + GroupWAFRuleGroups(day string, wafId string) ([]maps.Map, error) + // 列出最近的某些日志 ListLatestAccessLogs(day string, serverId string, fromId string, onlyErrors bool, size int) ([]*accesslogs.AccessLog, error) diff --git a/teadb/dao_access_log_mongo.go b/teadb/dao_access_log_mongo.go index 123eeac..04e1ce3 100644 --- a/teadb/dao_access_log_mongo.go +++ b/teadb/dao_access_log_mongo.go @@ -2,10 +2,12 @@ package teadb import ( "context" + "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teadb/shared" "github.com/TeaWeb/code/tealogs/accesslogs" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" timeutil "github.com/iwind/TeaGo/utils/time" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -248,6 +250,42 @@ func (this *MongoAccessLogDAO) HasAccessLogWithWAF(day string, wafId string) (bo return one != nil, err } +func (this *MongoAccessLogDAO) GroupWAFRuleGroups(day string, wafId string) ([]maps.Map, error) { + waf := teaconfigs.SharedWAFList().FindWAF(wafId) + if waf == nil { + return []maps.Map{}, nil + } + + query := NewQuery(this.TableName(day)) + ones, err := query. + Attr("attrs.waf_id", wafId). + Group("attrs.waf_group", map[string]Expr{ + "groupId": "attrs.waf_group", + "count": maps.Map{ + "$sum": 1, + }, + }) + if err != nil { + return nil, err + } + + result := []maps.Map{} + for _, one := range ones { + groupId := one.GetString("groupId") + group := waf.FindRuleGroup(groupId) + if group == nil { + continue + } + + result = append(result, maps.Map{ + "name": group.Name, + "count": one.GetInt("count"), + }) + } + + return result, err +} + func (this *MongoAccessLogDAO) ListLatestAccessLogs(day string, serverId string, fromId string, onlyErrors bool, size int) ([]*accesslogs.AccessLog, error) { query := NewQuery(this.TableName(day)) diff --git a/teadb/dao_access_log_test.go b/teadb/dao_access_log_test.go index 8c1f535..02705a5 100644 --- a/teadb/dao_access_log_test.go +++ b/teadb/dao_access_log_test.go @@ -213,6 +213,14 @@ func TestAccessLogDAO_HasAccessLogWithWAF(t *testing.T) { } } +func TestAccessLogDAO_GroupWAFRuleGroups(t *testing.T) { + ruleSets, err := AccessLogDAO().GroupWAFRuleGroups(timeutil.Format("Ymd"), "pq6HzRfIjcGsUqNe") + if err != nil { + t.Fatal(err) + } + logs.PrintAsJSON(ruleSets, t) +} + func TestAccessLogDAO_ListLatestAccessLogs(t *testing.T) { dao := AccessLogDAO() { diff --git a/teadb/dao_acess_log_sql.go b/teadb/dao_acess_log_sql.go index 5348bf3..8e7971b 100644 --- a/teadb/dao_acess_log_sql.go +++ b/teadb/dao_acess_log_sql.go @@ -1,13 +1,16 @@ package teadb import ( + "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teadb/shared" "github.com/TeaWeb/code/tealogs/accesslogs" "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" timeutil "github.com/iwind/TeaGo/utils/time" "github.com/lib/pq" + "strings" ) type SQLAccessLogDAO struct { @@ -249,6 +252,41 @@ func (this *SQLAccessLogDAO) HasAccessLogWithWAF(day string, wafId string) (bool return one != nil, err } +func (this *SQLAccessLogDAO) GroupWAFRuleGroups(day string, wafId string) ([]maps.Map, error) { + waf := teaconfigs.SharedWAFList().FindWAF(wafId) + if waf == nil { + return []maps.Map{}, nil + } + + driver := this.driver.(SQLDriverInterface) + query := NewQuery(this.TableName(day)) + ones, err := query. + Attr(driver.JSONExtract("attrs", "waf_id"), wafId). + Group(driver.JSONExtract("attrs", "waf_group"), map[string]Expr{ + "groupId": "attrs.waf_group", + "count": "COUNT(_id)", + }) + if err != nil { + return []maps.Map{}, err + } + + result := []maps.Map{} + for _, one := range ones { + groupId := strings.Trim(one.GetString("groupId"), "\"") + group := waf.FindRuleGroup(groupId) + if group == nil { + continue + } + + result = append(result, maps.Map{ + "name": group.Name, + "count": one.GetInt("count"), + }) + } + + return result, err +} + // 列出最近的某些日志 func (this *SQLAccessLogDAO) ListLatestAccessLogs(day string, serverId string, fromId string, onlyErrors bool, size int) ([]*accesslogs.AccessLog, error) { query := NewQuery(this.TableName(day)) diff --git a/teadb/driver_mongo.go b/teadb/driver_mongo.go index 6010428..a6ece02 100644 --- a/teadb/driver_mongo.go +++ b/teadb/driver_mongo.go @@ -364,6 +364,8 @@ func (this *MongoDriver) Group(query *Query, field string, result map[string]Exp group[name] = map[string]interface{}{ "$first": this.convertArrayElement(e), } + case maps.Map: + group[name] = e } } diff --git a/teaweb/actions/default/proxy/waf/day.go b/teaweb/actions/default/proxy/waf/day.go index 5c794b2..4074972 100644 --- a/teaweb/actions/default/proxy/waf/day.go +++ b/teaweb/actions/default/proxy/waf/day.go @@ -136,6 +136,15 @@ func (this *DayAction) Run(params struct { } } } + + // 统计 + stat, err := teadb.AccessLogDAO().GroupWAFRuleGroups(realDay, waf.Id) + if err != nil { + logs.Error(err) + this.Data["stat"] = []maps.Map{} + } else { + this.Data["stat"] = stat + } } this.Show() From 9a2fce579f12e409baf6898df3dd74e417b26ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月26日 21:13:03 +0800 Subject: [PATCH 073/121] =?UTF-8?q?[waf]CC=E7=BB=9F=E8=AE=A1=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E7=94=A8=E6=88=B7=E8=AF=86=E5=88=AB=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E6=9D=A5=E6=BA=90=E3=80=81=E7=94=A8=E6=88=B7=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E3=80=81=E5=AD=97=E6=AE=B5=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E7=AD=89=EF=BC=8C=E4=BB=A5=E4=BE=BF=E4=BA=8E?= =?UTF-8?q?=E6=9B=B4=E7=81=B5=E6=B4=BB=E5=9C=B0=E5=8C=BA=E5=88=86=E7=94=A8?= =?UTF-8?q?=E6=88=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teawaf/checkpoints/cc.go | 97 ++++++++++++++++++- teawaf/checkpoints/checkpoint.go | 2 +- teawaf/checkpoints/checkpoint_interface.go | 2 +- teawaf/checkpoints/option.go | 21 +--- teawaf/checkpoints/option_field.go | 26 +++++ teawaf/checkpoints/option_options.go | 30 ++++++ teaweb/actions/default/proxy/waf/ruleAdd.go | 46 ++++++--- .../actions/default/proxy/waf/ruleUpdate.go | 46 ++++++--- 8 files changed, 223 insertions(+), 47 deletions(-) create mode 100644 teawaf/checkpoints/option_field.go create mode 100644 teawaf/checkpoints/option_options.go diff --git a/teawaf/checkpoints/cc.go b/teawaf/checkpoints/cc.go index 79510f7..7241995 100644 --- a/teawaf/checkpoints/cc.go +++ b/teawaf/checkpoints/cc.go @@ -3,6 +3,7 @@ package checkpoints import ( "github.com/TeaWeb/code/teamemory" "github.com/TeaWeb/code/teawaf/requests" + "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/types" "net" "regexp" @@ -51,8 +52,55 @@ func (this *CCCheckpoint) RequestValue(req *requests.Request, param string, opti return } + userType, _ := options["userType"] + userField, _ := options["userField"] + userIndex, _ := options["userIndex"] + userIndexInt := types.Int(userIndex) + if param == "requests" { // requests - key := this.ip(req) + var key = "" + switch userType { + case "ip": + key = this.ip(req) + case "cookie": + if len(userField) == 0 { + key = this.ip(req) + } else { + cookie, _ := req.Cookie(userField) + if cookie != nil { + v := cookie.Value + if userIndexInt> 0 && len(v)> userIndexInt { + v = v[userIndexInt:] + } + key = "USER@" + userType + "@" + userField + "@" + v + } + } + case "get": + if len(userField) == 0 { + key = this.ip(req) + } else { + v := req.URL.Query().Get(userField) + if userIndexInt> 0 && len(v)> userIndexInt { + v = v[userIndexInt:] + } + key = "USER@" + userType + "@" + userField + "@" + v + } + case "post": + if len(userField) == 0 { + key = this.ip(req) + } else { + v := req.PostFormValue(userField) + if userIndexInt> 0 && len(v)> userIndexInt { + v = v[userIndexInt:] + } + key = "USER@" + userType + "@" + userField + "@" + v + } + default: + key = this.ip(req) + } + if len(key) == 0 { + key = this.ip(req) + } value = this.grid.IncreaseInt64([]byte(key), 1, period) } @@ -72,12 +120,12 @@ func (this *CCCheckpoint) ParamOptions() *ParamOptions { return option } -func (this *CCCheckpoint) Options() []*Option { - options := []*Option{} +func (this *CCCheckpoint) Options() []OptionInterface { + options := []OptionInterface{} // period { - option := NewOption("统计周期", "period") + option := NewFieldOption("统计周期", "period") option.Value = "60" option.RightLabel = "秒" option.Size = 8 @@ -93,6 +141,47 @@ func (this *CCCheckpoint) Options() []*Option { options = append(options, option) } + // type + { + option := NewOptionsOption("用户识别读取来源", "userType") + option.Size = 10 + option.SetOptions([]maps.Map{ + { + "name": "IP", + "value": "ip", + }, + { + "name": "Cookie", + "value": "cookie", + }, + { + "name": "URL参数", + "value": "get", + }, + { + "name": "POST参数", + "value": "post", + }, + }) + options = append(options, option) + } + + // user field + { + option := NewFieldOption("用户识别字段", "userField") + option.Comment = "识别用户的唯一性字段,在用户读取来源不是IP时使用" + options = append(options, option) + } + + // user value index + { + option := NewFieldOption("字段读取位置", "userIndex") + option.Size = 5 + option.MaxLength = 5 + option.Comment = "读取用户识别字段的位置,从0开始,比如user12345的数字ID 12345的位置就是5,在用户读取来源不是IP时使用" + options = append(options, option) + } + return options } diff --git a/teawaf/checkpoints/checkpoint.go b/teawaf/checkpoints/checkpoint.go index d663856..3b1475b 100644 --- a/teawaf/checkpoints/checkpoint.go +++ b/teawaf/checkpoints/checkpoint.go @@ -15,7 +15,7 @@ func (this *Checkpoint) ParamOptions() *ParamOptions { return nil } -func (this *Checkpoint) Options() []*Option { +func (this *Checkpoint) Options() []OptionInterface { return nil } diff --git a/teawaf/checkpoints/checkpoint_interface.go b/teawaf/checkpoints/checkpoint_interface.go index 956967c..71ff376 100644 --- a/teawaf/checkpoints/checkpoint_interface.go +++ b/teawaf/checkpoints/checkpoint_interface.go @@ -22,7 +22,7 @@ type CheckpointInterface interface { ParamOptions() *ParamOptions // options - Options() []*Option + Options() []OptionInterface // start Start() diff --git a/teawaf/checkpoints/option.go b/teawaf/checkpoints/option.go index fb76eaa..04850a8 100644 --- a/teawaf/checkpoints/option.go +++ b/teawaf/checkpoints/option.go @@ -1,22 +1,5 @@ package checkpoints -// attach option -type Option struct { - Name string - Code string - Value string // default value - IsRequired bool - Size int - Comment string - Placeholder string - RightLabel string - MaxLength int - Validate func(value string) (ok bool, message string) -} - -func NewOption(name string, code string) *Option { - return &Option{ - Name: name, - Code: code, - } +type OptionInterface interface { + Type() string } diff --git a/teawaf/checkpoints/option_field.go b/teawaf/checkpoints/option_field.go new file mode 100644 index 0000000..c476b20 --- /dev/null +++ b/teawaf/checkpoints/option_field.go @@ -0,0 +1,26 @@ +package checkpoints + +// attach option +type FieldOption struct { + Name string + Code string + Value string // default value + IsRequired bool + Size int + Comment string + Placeholder string + RightLabel string + MaxLength int + Validate func(value string) (ok bool, message string) +} + +func NewFieldOption(name string, code string) *FieldOption { + return &FieldOption{ + Name: name, + Code: code, + } +} + +func (this *FieldOption) Type() string { + return "field" +} diff --git a/teawaf/checkpoints/option_options.go b/teawaf/checkpoints/option_options.go new file mode 100644 index 0000000..dcfa6dc --- /dev/null +++ b/teawaf/checkpoints/option_options.go @@ -0,0 +1,30 @@ +package checkpoints + +import "github.com/iwind/TeaGo/maps" + +type OptionsOption struct { + Name string + Code string + Value string // default value + IsRequired bool + Size int + Comment string + RightLabel string + Validate func(value string) (ok bool, message string) + Options []maps.Map +} + +func NewOptionsOption(name string, code string) *OptionsOption { + return &OptionsOption{ + Name: name, + Code: code, + } +} + +func (this *OptionsOption) Type() string { + return "options" +} + +func (this *OptionsOption) SetOptions(options []maps.Map) { + this.Options = options +} diff --git a/teaweb/actions/default/proxy/waf/ruleAdd.go b/teaweb/actions/default/proxy/waf/ruleAdd.go index 19d1db0..2acc6a1 100644 --- a/teaweb/actions/default/proxy/waf/ruleAdd.go +++ b/teaweb/actions/default/proxy/waf/ruleAdd.go @@ -64,18 +64,42 @@ func (this *RuleAddAction) RunGet(params struct { "hasParams": def.HasParams, "paramOptions": def.Instance.ParamOptions(), "options": lists.Map(def.Instance.Options(), func(k int, v interface{}) interface{} { - option := v.(*checkpoints.Option) - return maps.Map{ - "name": option.Name, - "maxLength": option.MaxLength, - "code": option.Code, - "rightLabel": option.RightLabel, - "value": option.Value, - "isRequired": option.IsRequired, - "size": option.Size, - "comment": option.Comment, - "placeholder": option.Placeholder, + { + option, ok := v.(*checkpoints.FieldOption) + if ok { + return maps.Map{ + "type": option.Type(), + "name": option.Name, + "maxLength": option.MaxLength, + "code": option.Code, + "rightLabel": option.RightLabel, + "value": option.Value, + "isRequired": option.IsRequired, + "size": option.Size, + "comment": option.Comment, + "placeholder": option.Placeholder, + } + } } + + { + option, ok := v.(*checkpoints.OptionsOption) + if ok { + return maps.Map{ + "type": option.Type(), + "name": option.Name, + "code": option.Code, + "rightLabel": option.RightLabel, + "value": option.Value, + "isRequired": option.IsRequired, + "size": option.Size, + "comment": option.Comment, + "options": option.Options, + } + } + } + + return maps.Map{} }), }) } diff --git a/teaweb/actions/default/proxy/waf/ruleUpdate.go b/teaweb/actions/default/proxy/waf/ruleUpdate.go index 89c6967..19a5852 100644 --- a/teaweb/actions/default/proxy/waf/ruleUpdate.go +++ b/teaweb/actions/default/proxy/waf/ruleUpdate.go @@ -104,18 +104,42 @@ func (this *RuleUpdateAction) RunGet(params struct { "hasParams": def.HasParams, "paramOptions": def.Instance.ParamOptions(), "options": lists.Map(def.Instance.Options(), func(k int, v interface{}) interface{} { - option := v.(*checkpoints.Option) - return maps.Map{ - "name": option.Name, - "maxLength": option.MaxLength, - "code": option.Code, - "rightLabel": option.RightLabel, - "value": option.Value, - "isRequired": option.IsRequired, - "size": option.Size, - "comment": option.Comment, - "placeholder": option.Placeholder, + { + option, ok := v.(*checkpoints.FieldOption) + if ok { + return maps.Map{ + "type": option.Type(), + "name": option.Name, + "maxLength": option.MaxLength, + "code": option.Code, + "rightLabel": option.RightLabel, + "value": option.Value, + "isRequired": option.IsRequired, + "size": option.Size, + "comment": option.Comment, + "placeholder": option.Placeholder, + } + } } + + { + option, ok := v.(*checkpoints.OptionsOption) + if ok { + return maps.Map{ + "type": option.Type(), + "name": option.Name, + "code": option.Code, + "rightLabel": option.RightLabel, + "value": option.Value, + "isRequired": option.IsRequired, + "size": option.Size, + "comment": option.Comment, + "options": option.Options, + } + } + } + + return maps.Map{} }), }) } From a2cc88c848ab55e17eae019ac3288c2f529f6626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月27日 16:26:18 +0800 Subject: [PATCH 074/121] =?UTF-8?q?[proxy]=E5=AE=9E=E7=8E=B0=E5=9F=BA?= =?UTF-8?q?=E6=9C=AC=E7=9A=84=E6=AD=A3=E5=90=91=E4=BB=A3=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/forward_http.go | 10 ++ teaconfigs/server.go | 8 ++ teaproxy/listener.go | 13 +++ teaproxy/request.go | 5 + teaproxy/request_forward.go | 93 +++++++++++++++++++ teaweb/actions/default/proxy/add.go | 2 + teaweb/actions/default/proxy/detail.go | 1 + .../actions/default/proxy/proxyutils/menu.go | 2 + teaweb/actions/default/proxy/update.go | 1 + 9 files changed, 135 insertions(+) create mode 100644 teaconfigs/forward_http.go create mode 100644 teaproxy/request_forward.go diff --git a/teaconfigs/forward_http.go b/teaconfigs/forward_http.go new file mode 100644 index 0000000..80167fc --- /dev/null +++ b/teaconfigs/forward_http.go @@ -0,0 +1,10 @@ +package teaconfigs + +// 正向代理设置 +type ForwardHTTPConfig struct { +} + +// 获取新对象 +func NewForwardHTTPConfig() *ForwardHTTPConfig { + return &ForwardHTTPConfig{} +} \ No newline at end of file diff --git a/teaconfigs/server.go b/teaconfigs/server.go index 8e623ca..9fc4792 100644 --- a/teaconfigs/server.go +++ b/teaconfigs/server.go @@ -68,6 +68,9 @@ type ServerConfig struct { // TCP,如果有此配置的说明为TCP代理 TCP *TCPConfig `yaml:"tcp" json:"tcp"` + // ForwardHTTP,如果有此配置的说明为正向HTTP代理 + ForwardHTTP *ForwardHTTPConfig `yaml:"forwardHTTP" json:"forwardHTTP"` + // 参考:http://nginx.org/en/docs/http/ngx_http_access_module.html Allow []string `yaml:"allow" json:"allow"` //TODO Deny []string `yaml:"deny" json:"deny"` //TODO @@ -1058,6 +1061,11 @@ func (this *ServerConfig) IsHTTP() bool { return this.TCP == nil } +// 是否为正向代理 +func (this *ServerConfig) IsForwardHTTP() bool { + return this.ForwardHTTP != nil +} + // 克隆运行时状态 func (this *ServerConfig) CloneState(oldServer *ServerConfig) { if oldServer == nil { diff --git a/teaproxy/listener.go b/teaproxy/listener.go index 78cb7ff..48de39f 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -450,6 +450,19 @@ func (this *Listener) handleHTTP(writer http.ResponseWriter, rawRequest *http.Re return } + // 正向代理 + if server.ForwardHTTP != nil { + err = req.Forward(req.responseWriter) + if err != nil { + logs.Error(errors.New(reqHost + rawRequest.URL.String() + ": " + err.Error())) + } + + // 返还request + requestPool.Put(req) + + return + } + // 处理请求 err = req.call(req.responseWriter) if err != nil { diff --git a/teaproxy/request.go b/teaproxy/request.go index 6c5ca7d..29d91f9 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -331,6 +331,11 @@ func (this *Request) configure(server *teaconfigs.ServerConfig, redirects int, b } } + // 如果是正向代理,则直接返回 + if server.ForwardHTTP != nil { + return nil + } + if !breakRewrite { // location的相关配置 var locationConfigured = false diff --git a/teaproxy/request_forward.go b/teaproxy/request_forward.go new file mode 100644 index 0000000..06b93de --- /dev/null +++ b/teaproxy/request_forward.go @@ -0,0 +1,93 @@ +package teaproxy + +import ( + "github.com/TeaWeb/code/teautils" + "github.com/iwind/TeaGo/lists" + "io" + "net" + "net/http" + "time" +) + +// 正向代理 +func (this *Request) Forward(writer *ResponseWriter) error { + defer this.log() + + if len(this.raw.URL.Scheme) == 0 { + this.rawScheme = "https" + } + + this.setProxyHeaders(this.raw.Header) + + if this.method == http.MethodConnect { // connect + hostConn, err := net.DialTimeout("tcp", this.host, 30*time.Second) + if err != nil { + this.serverError(writer) + this.addError(err) + return nil + } + + hijacker, ok := writer.writer.(http.Hijacker) + if !ok { + this.serverError(writer) + this.addError(err) + return nil + } + + writer.WriteHeader(http.StatusOK) + + clientConn, _, err := hijacker.Hijack() + if err != nil { + this.serverError(writer) + this.addError(err) + return nil + } + + go func() { + _, _ = io.Copy(clientConn, hostConn) + _ = clientConn.Close() + _ = hostConn.Close() + }() + go func() { + _, _ = io.Copy(hostConn, clientConn) + _ = clientConn.Close() + _ = hostConn.Close() + }() + } else { // http + this.raw.RequestURI = "" + + // 删除代理相关Header + for n, _ := range this.raw.Header { + if lists.ContainsString([]string{"Proxy-Connection", "Connection", "Proxy-Authorization"}, n) { + this.raw.Header.Del(n) + } + } + + client := teautils.SharedHttpClient(30 * time.Second) + resp, err := client.Do(this.raw) + if err != nil { + this.serverError(writer) + this.addError(err) + return nil + } + defer func() { + _ = resp.Body.Close() + }() + + for k, v := range resp.Header { + if k == "Connection" { + continue + } + for _, subV := range v { + writer.Header().Add(k, subV) + } + } + + writer.Prepare(resp.ContentLength) + writer.WriteHeader(resp.StatusCode) + + _, _ = io.Copy(writer, resp.Body) + } + + return nil +} diff --git a/teaweb/actions/default/proxy/add.go b/teaweb/actions/default/proxy/add.go index 93e8ed0..3ddebcd 100644 --- a/teaweb/actions/default/proxy/add.go +++ b/teaweb/actions/default/proxy/add.go @@ -88,6 +88,8 @@ func (this *AddAction) RunPost(params struct { server.AddBackend(backendObject) } } + } else if params.ServerType == "forwardProxy" { // 正向HTTP代理 + server.ForwardHTTP = teaconfigs.NewForwardHTTPConfig() } else if params.ServerType == "tcp" { // TCP代理服务 // DEMO版防止通过代理获取服务器数据库数据 if teaconst.DemoEnabled { diff --git a/teaweb/actions/default/proxy/detail.go b/teaweb/actions/default/proxy/detail.go index a0952a7..7726d37 100644 --- a/teaweb/actions/default/proxy/detail.go +++ b/teaweb/actions/default/proxy/detail.go @@ -25,6 +25,7 @@ func (this *DetailAction) Run(params struct { this.Data["selectedTab"] = "basic" this.Data["server"] = server this.Data["isTCP"] = server.IsTCP() + this.Data["isForwardHTTP"] = server.IsForwardHTTP() this.Data["errs"] = teaproxy.SharedManager.FindServerErrors(params.ServerId) this.Data["accessLogs"] = proxyutils.FormatAccessLog(server.AccessLog) diff --git a/teaweb/actions/default/proxy/proxyutils/menu.go b/teaweb/actions/default/proxy/proxyutils/menu.go index 47db091..c5451d7 100644 --- a/teaweb/actions/default/proxy/proxyutils/menu.go +++ b/teaweb/actions/default/proxy/proxyutils/menu.go @@ -52,6 +52,8 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { if server.IsTCP() { item.SupName = "tcp" + } else if server.ForwardHTTP != nil { + item.SupName = "forward" } // port diff --git a/teaweb/actions/default/proxy/update.go b/teaweb/actions/default/proxy/update.go index 92fc61d..75a7250 100644 --- a/teaweb/actions/default/proxy/update.go +++ b/teaweb/actions/default/proxy/update.go @@ -24,6 +24,7 @@ func (this *UpdateAction) Run(params struct { this.Data["server"] = server this.Data["selectedTab"] = "basic" this.Data["isTCP"] = server.IsTCP() + this.Data["isForwardHTTP"] = server.ForwardHTTP if server.Gzip == nil { server.Gzip = &teaconfigs.GzipConfig{ From 70d9d2bf83cf9dc00ebc8c7c25198b51a2a585f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月27日 16:31:38 +0800 Subject: [PATCH 075/121] =?UTF-8?q?[proxy]=E6=AD=A3=E5=90=91=E4=BB=A3?= =?UTF-8?q?=E7=90=86=EF=BC=9A=E5=8F=AF=E4=BB=A5=E8=AE=B0=E5=BD=95=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E5=92=8C=E5=93=8D=E5=BA=94Body?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_forward.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/teaproxy/request_forward.go b/teaproxy/request_forward.go index 06b93de..d19d219 100644 --- a/teaproxy/request_forward.go +++ b/teaproxy/request_forward.go @@ -1,11 +1,15 @@ package teaproxy import ( + "bytes" + "github.com/TeaWeb/code/tealogs/accesslogs" "github.com/TeaWeb/code/teautils" "github.com/iwind/TeaGo/lists" "io" + "io/ioutil" "net" "net/http" + "net/http/httputil" "time" ) @@ -13,6 +17,36 @@ import ( func (this *Request) Forward(writer *ResponseWriter) error { defer this.log() + // watch + if this.isWatching { + // 判断如果Content-Length过长,则截断 + reqData, err := httputil.DumpRequest(this.raw, true) + if err == nil { + if len(reqData)> 100240 { + reqData = reqData[:100240] + } + this.requestData = reqData + } + + writer.SetBodyCopying(true) + } else { + max := 512 * 1024 // 512K + if this.accessLog != nil && lists.ContainsInt(this.accessLog.Fields, accesslogs.AccessLogFieldRequestBody) { + body, err := ioutil.ReadAll(this.raw.Body) + if err == nil { + if len(body)> max { + this.requestData = body[:max] + } else { + this.requestData = body + } + } + this.raw.Body = ioutil.NopCloser(bytes.NewReader(body)) + } + if this.accessLog != nil && lists.ContainsInt(this.accessLog.Fields, accesslogs.AccessLogFieldResponseBody) { + writer.SetBodyCopying(true) + } + } + if len(this.raw.URL.Scheme) == 0 { this.rawScheme = "https" } From 2c52ef644664a3992e33fc44ee9d2bbcb4cdcb8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月28日 16:16:07 +0800 Subject: [PATCH 076/121] =?UTF-8?q?[proxy]=E4=B8=BB=E5=8A=A8=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E5=AE=9E=E7=8E=B0MITM=E6=A8=A1=E5=BC=8F=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E8=83=BD=E8=AE=B0=E5=BD=95=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/forward_http.go | 3 +- teaproxy/mitm/README.md | 2 + teaproxy/mitm/mitm.go | 287 ++++++++++++++++++++ teaproxy/mitm/mitm_test.go | 205 +++++++++++++++ teaproxy/proxy_forward.go | 347 +++++++++++++++++++++++++ teaproxy/request_forward.go | 124 ++------- teaproxy/request_forward_test.go | 89 +++++++ teaproxy/response_empty.go | 26 ++ teaweb/actions/default/proxy/update.go | 5 + 9 files changed, 980 insertions(+), 108 deletions(-) create mode 100644 teaproxy/mitm/README.md create mode 100644 teaproxy/mitm/mitm.go create mode 100644 teaproxy/mitm/mitm_test.go create mode 100644 teaproxy/proxy_forward.go create mode 100644 teaproxy/request_forward_test.go create mode 100644 teaproxy/response_empty.go diff --git a/teaconfigs/forward_http.go b/teaconfigs/forward_http.go index 80167fc..6cc802e 100644 --- a/teaconfigs/forward_http.go +++ b/teaconfigs/forward_http.go @@ -2,9 +2,10 @@ package teaconfigs // 正向代理设置 type ForwardHTTPConfig struct { + EnableMITM bool `yaml:"enableMITM" json:"enableMITM"` } // 获取新对象 func NewForwardHTTPConfig() *ForwardHTTPConfig { return &ForwardHTTPConfig{} -} \ No newline at end of file +} diff --git a/teaproxy/mitm/README.md b/teaproxy/mitm/README.md new file mode 100644 index 0000000..6a1e5d7 --- /dev/null +++ b/teaproxy/mitm/README.md @@ -0,0 +1,2 @@ +The code is from: +https://github.com/google/martian/blob/master/mitm/ \ No newline at end of file diff --git a/teaproxy/mitm/mitm.go b/teaproxy/mitm/mitm.go new file mode 100644 index 0000000..74b3a5d --- /dev/null +++ b/teaproxy/mitm/mitm.go @@ -0,0 +1,287 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mitm provides tooling for MITMing TLS connections. It provides +// tooling to create CA certs and generate TLS configs that can be used to MITM +// a TLS connection with a provided CA certificate. +package mitm + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "math/big" + "net" + "net/http" + "sync" + "time" +) + +// MaxSerialNumber is the upper boundary that is used to create unique serial +// numbers for the certificate. This can be any unsigned integer up to 20 +// bytes (2^(8*20)-1). +var MaxSerialNumber = big.NewInt(0).SetBytes(bytes.Repeat([]byte{255}, 20)) + +// Config is a set of configuration values that are used to build TLS configs +// capable of MITM. +type Config struct { + ca *x509.Certificate + capriv interface{} + priv *rsa.PrivateKey + keyID []byte + validity time.Duration + org string + getCertificate func(*tls.ClientHelloInfo) (*tls.Certificate, error) + roots *x509.CertPool + skipVerify bool + handshakeErrorCallback func(*http.Request, error) + + certmu sync.RWMutex + certs map[string]*tls.Certificate +} + +// NewAuthority creates a new CA certificate and associated +// private key. +func NewAuthority(name, organization string, validity time.Duration) (*x509.Certificate, *rsa.PrivateKey, error) { + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + pub := priv.Public() + + // Subject Key Identifier support for end entity certificate. + // https://www.ietf.org/rfc/rfc3280.txt (section 4.2.1.2) + pkixpub, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return nil, nil, err + } + h := sha1.New() + h.Write(pkixpub) + keyID := h.Sum(nil) + + // TODO: keep a map of used serial numbers to avoid potentially reusing a + // serial multiple times. + serial, err := rand.Int(rand.Reader, MaxSerialNumber) + if err != nil { + return nil, nil, err + } + + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: name, + Organization: []string{organization}, + }, + SubjectKeyId: keyID, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + NotBefore: time.Now().Add(-validity), + NotAfter: time.Now().Add(validity), + DNSNames: []string{name}, + IsCA: true, + } + + raw, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, priv) + if err != nil { + return nil, nil, err + } + + // Parse certificate bytes so that we have a leaf certificate. + x509c, err := x509.ParseCertificate(raw) + if err != nil { + return nil, nil, err + } + + return x509c, priv, nil +} + +// NewConfig creates a MITM config using the CA certificate and +// private key to generate on-the-fly certificates. +func NewConfig(ca *x509.Certificate, privateKey interface{}) (*Config, error) { + roots := x509.NewCertPool() + roots.AddCert(ca) + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + pub := priv.Public() + + // Subject Key Identifier support for end entity certificate. + // https://www.ietf.org/rfc/rfc3280.txt (section 4.2.1.2) + pkixpub, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return nil, err + } + h := sha1.New() + h.Write(pkixpub) + keyID := h.Sum(nil) + + return &Config{ + ca: ca, + capriv: privateKey, + priv: priv, + keyID: keyID, + validity: time.Hour, + org: "Martian Proxy", + certs: make(map[string]*tls.Certificate), + roots: roots, + }, nil +} + +// SetValidity sets the validity window around the current time that the +// certificate is valid for. +func (c *Config) SetValidity(validity time.Duration) { + c.validity = validity +} + +// SkipTLSVerify skips the TLS certification verification check. +func (c *Config) SkipTLSVerify(skip bool) { + c.skipVerify = skip +} + +// SetOrganization sets the organization of the certificate. +func (c *Config) SetOrganization(org string) { + c.org = org +} + +// SetHandshakeErrorCallback sets the handshakeErrorCallback function. +func (c *Config) SetHandshakeErrorCallback(cb func(*http.Request, error)) { + c.handshakeErrorCallback = cb +} + +// HandshakeErrorCallback calls the handshakeErrorCallback function in this +// Config, if it is non-nil. Request is the connect request that this handshake +// is being executed through. +func (c *Config) HandshakeErrorCallback(r *http.Request, err error) { + if c.handshakeErrorCallback != nil { + c.handshakeErrorCallback(r, err) + } +} + +// TLS returns a *tls.Config that will generate certificates on-the-fly using +// the SNI extension in the TLS ClientHello. +func (c *Config) TLS() *tls.Config { + return &tls.Config{ + InsecureSkipVerify: c.skipVerify, + GetCertificate: func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if clientHello.ServerName == "" { + return nil, errors.New("mitm: SNI not provided, failed to build certificate") + } + + return c.cert(clientHello.ServerName) + }, + NextProtos: []string{"http/1.1"}, + } +} + +// TLSForHost returns a *tls.Config that will generate certificates on-the-fly +// using SNI from the connection, or fall back to the provided hostname. +func (c *Config) TLSForHost(hostname string) *tls.Config { + return &tls.Config{ + InsecureSkipVerify: c.skipVerify, + GetCertificate: func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + host := clientHello.ServerName + if host == "" { + host = hostname + } + + return c.cert(host) + }, + NextProtos: []string{"http/1.1"}, + } +} + +func (c *Config) cert(hostname string) (*tls.Certificate, error) { + // Remove the port if it exists. + host, _, err := net.SplitHostPort(hostname) + if err == nil { + hostname = host + } + + c.certmu.RLock() + tlsc, ok := c.certs[hostname] + c.certmu.RUnlock() + + if ok { + //log.Printf("mitm: cache hit for %s", hostname) + + // Check validity of the certificate for hostname match, expiry, etc. In + // particular, if the cached certificate has expired, create a new one. + if _, err := tlsc.Leaf.Verify(x509.VerifyOptions{ + DNSName: hostname, + Roots: c.roots, + }); err == nil { + return tlsc, nil + } + + //log.Printf("mitm: invalid certificate in cache for %s", hostname) + } + + //log.Printf("mitm: cache miss for %s", hostname) + + serial, err := rand.Int(rand.Reader, MaxSerialNumber) + if err != nil { + return nil, err + } + + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: hostname, + Organization: []string{c.org}, + }, + SubjectKeyId: c.keyID, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + NotBefore: time.Now().Add(-c.validity), + NotAfter: time.Now().Add(c.validity), + } + + if ip := net.ParseIP(hostname); ip != nil { + tmpl.IPAddresses = []net.IP{ip} + } else { + tmpl.DNSNames = []string{hostname} + } + + raw, err := x509.CreateCertificate(rand.Reader, tmpl, c.ca, c.priv.Public(), c.capriv) + if err != nil { + return nil, err + } + + // Parse certificate bytes so that we have a leaf certificate. + x509c, err := x509.ParseCertificate(raw) + if err != nil { + return nil, err + } + + tlsc = &tls.Certificate{ + Certificate: [][]byte{raw, c.ca.Raw}, + PrivateKey: c.priv, + Leaf: x509c, + } + + c.certmu.Lock() + c.certs[hostname] = tlsc + c.certmu.Unlock() + + return tlsc, nil +} diff --git a/teaproxy/mitm/mitm_test.go b/teaproxy/mitm/mitm_test.go new file mode 100644 index 0000000..d9f584e --- /dev/null +++ b/teaproxy/mitm/mitm_test.go @@ -0,0 +1,205 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mitm + +import ( + "crypto/tls" + "crypto/x509" + "net" + "reflect" + "testing" + "time" +) + +func TestMITM(t *testing.T) { + ca, priv, err := NewAuthority("martian.proxy", "Martian Authority", 24*time.Hour) + if err != nil { + t.Fatalf("NewAuthority(): got %v, want no error", err) + } + + c, err := NewConfig(ca, priv) + if err != nil { + t.Fatalf("NewConfig(): got %v, want no error", err) + } + + c.SetValidity(20 * time.Hour) + c.SetOrganization("Test Organization") + + protos := []string{"http/1.1"} + + conf := c.TLS() + if got := conf.NextProtos; !reflect.DeepEqual(got, protos) { + t.Errorf("conf.NextProtos: got %v, want %v", got, protos) + } + if conf.InsecureSkipVerify { + t.Error("conf.InsecureSkipVerify: got true, want false") + } + + // Simulate a TLS connection without SNI. + clientHello := &tls.ClientHelloInfo{ + ServerName: "", + } + + if _, err := conf.GetCertificate(clientHello); err == nil { + t.Fatal("conf.GetCertificate(): got nil, want error") + } + + // Simulate a TLS connection with SNI. + clientHello.ServerName = "example.com" + + tlsc, err := conf.GetCertificate(clientHello) + if err != nil { + t.Fatalf("conf.GetCertificate(): got %v, want no error", err) + } + + x509c := tlsc.Leaf + if got, want := x509c.Subject.CommonName, "example.com"; got != want { + t.Errorf("x509c.Subject.CommonName: got %q, want %q", got, want) + } + + c.SkipTLSVerify(true) + + conf = c.TLSForHost("example.com") + if got := conf.NextProtos; !reflect.DeepEqual(got, protos) { + t.Errorf("conf.NextProtos: got %v, want %v", got, protos) + } + if !conf.InsecureSkipVerify { + t.Error("conf.InsecureSkipVerify: got false, want true") + } + + // Set SNI, takes precedence over host. + clientHello.ServerName = "google.com" + tlsc, err = conf.GetCertificate(clientHello) + if err != nil { + t.Fatalf("conf.GetCertificate(): got %v, want no error", err) + } + + x509c = tlsc.Leaf + if got, want := x509c.Subject.CommonName, "google.com"; got != want { + t.Errorf("x509c.Subject.CommonName: got %q, want %q", got, want) + } + + // Reset SNI to fallback to hostname. + clientHello.ServerName = "" + tlsc, err = conf.GetCertificate(clientHello) + if err != nil { + t.Fatalf("conf.GetCertificate(): got %v, want no error", err) + } + + x509c = tlsc.Leaf + if got, want := x509c.Subject.CommonName, "example.com"; got != want { + t.Errorf("x509c.Subject.CommonName: got %q, want %q", got, want) + } +} + +func TestCert(t *testing.T) { + ca, priv, err := NewAuthority("martian.proxy", "Martian Authority", 24*time.Hour) + if err != nil { + t.Fatalf("NewAuthority(): got %v, want no error", err) + } + + c, err := NewConfig(ca, priv) + if err != nil { + t.Fatalf("NewConfig(): got %v, want no error", err) + } + + tlsc, err := c.cert("example.com") + if err != nil { + t.Fatalf("c.cert(%q): got %v, want no error", "example.com:8080", err) + } + + if tlsc.Certificate == nil { + t.Error("tlsc.Certificate: got nil, want certificate bytes") + } + if tlsc.PrivateKey == nil { + t.Error("tlsc.PrivateKey: got nil, want private key") + } + + x509c := tlsc.Leaf + if x509c == nil { + t.Fatal("x509c: got nil, want *x509.Certificate") + } + + if got := x509c.SerialNumber; got.Cmp(MaxSerialNumber)>= 0 { + t.Errorf("x509c.SerialNumber: got %v, want <= MaxSerialNumber", got) + } + if got, want := x509c.Subject.CommonName, "example.com"; got != want { + t.Errorf("X509c.Subject.CommonName: got %q, want %q", got, want) + } + if err := x509c.VerifyHostname("example.com"); err != nil { + t.Errorf("x509c.VerifyHostname(%q): got %v, want no error", "example.com", err) + } + + if got, want := x509c.Subject.Organization, []string{"Martian Proxy"}; !reflect.DeepEqual(got, want) { + t.Errorf("x509c.Subject.Organization: got %v, want %v", got, want) + } + + if got := x509c.SubjectKeyId; got == nil { + t.Error("x509c.SubjectKeyId: got nothing, want key ID") + } + if !x509c.BasicConstraintsValid { + t.Error("x509c.BasicConstraintsValid: got false, want true") + } + + if got, want := x509c.KeyUsage, x509.KeyUsageKeyEncipherment; got&want == 0 { + t.Error("x509c.KeyUsage: got nothing, want to include x509.KeyUsageKeyEncipherment") + } + if got, want := x509c.KeyUsage, x509.KeyUsageDigitalSignature; got&want == 0 { + t.Error("x509c.KeyUsage: got nothing, want to include x509.KeyUsageDigitalSignature") + } + + want := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + if got := x509c.ExtKeyUsage; !reflect.DeepEqual(got, want) { + t.Errorf("x509c.ExtKeyUsage: got %v, want %v", got, want) + } + + if got, want := x509c.DNSNames, []string{"example.com"}; !reflect.DeepEqual(got, want) { + t.Errorf("x509c.DNSNames: got %v, want %v", got, want) + } + + before := time.Now().Add(-2 * time.Hour) + if got := x509c.NotBefore; before.After(got) { + t.Errorf("x509c.NotBefore: got %v, want after %v", got, before) + } + + after := time.Now().Add(2 * time.Hour) + if got := x509c.NotAfter; !after.After(got) { + t.Errorf("x509c.NotAfter: got %v, want before %v", got, want) + } + + // Retrieve cached certificate. + tlsc2, err := c.cert("example.com") + if err != nil { + t.Fatalf("c.cert(%q): got %v, want no error", "example.com", err) + } + if tlsc != tlsc2 { + t.Error("tlsc2: got new certificate, want cached certificate") + } + + // TLS certificate for IP. + tlsc, err = c.cert("10.0.0.1:8227") + if err != nil { + t.Fatalf("c.cert(%q): got %v, want no error", "10.0.0.1:8227", err) + } + x509c = tlsc.Leaf + + if got, want := len(x509c.IPAddresses), 1; got != want { + t.Fatalf("len(x509c.IPAddresses): got %d, want %d", got, want) + } + + if got, want := x509c.IPAddresses[0], net.ParseIP("10.0.0.1"); !got.Equal(want) { + t.Fatalf("x509c.IPAddresses: got %v, want %v", got, want) + } +} diff --git a/teaproxy/proxy_forward.go b/teaproxy/proxy_forward.go new file mode 100644 index 0000000..534e043 --- /dev/null +++ b/teaproxy/proxy_forward.go @@ -0,0 +1,347 @@ +package teaproxy + +import ( + "bufio" + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "github.com/TeaWeb/code/tealogs/accesslogs" + "github.com/TeaWeb/code/teaproxy/mitm" + "github.com/TeaWeb/code/teautils" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/lists" + "github.com/iwind/TeaGo/logs" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httputil" + "time" +) + +type ForwardProxy struct { + req *Request + writer *ResponseWriter +} + +func (this *ForwardProxy) forwardHTTP() error { + defer this.req.log() + + // watch + if this.req.isWatching { + // 判断如果Content-Length过长,则截断 + reqData, err := httputil.DumpRequest(this.req.raw, true) + if err == nil { + if len(reqData)> 100240 { + reqData = reqData[:100240] + } + this.req.requestData = reqData + } + + this.writer.SetBodyCopying(true) + } else { + max := 512 * 1024 // 512K + if this.req.accessLog != nil && lists.ContainsInt(this.req.accessLog.Fields, accesslogs.AccessLogFieldRequestBody) { + body, err := ioutil.ReadAll(this.req.raw.Body) + if err == nil { + if len(body)> max { + this.req.requestData = body[:max] + } else { + this.req.requestData = body + } + } + this.req.raw.Body = ioutil.NopCloser(bytes.NewReader(body)) + } + if this.req.accessLog != nil && lists.ContainsInt(this.req.accessLog.Fields, accesslogs.AccessLogFieldResponseBody) { + this.writer.SetBodyCopying(true) + } + } + + this.req.raw.RequestURI = "" + + // 删除代理相关Header + for n, _ := range this.req.raw.Header { + if lists.ContainsString([]string{ + "Connection", + "Accept-Encoding", + "Proxy-Connection", + "Proxy-Authenticate", + "Proxy-Authorization", + }, n) { + this.req.raw.Header.Del(n) + } + } + + client := teautils.SharedHttpClient(30 * time.Second) + resp, err := client.Do(this.req.raw) + if err != nil { + this.req.serverError(this.writer) + this.req.addError(err) + return nil + } + defer func() { + _ = resp.Body.Close() + }() + + for k, v := range resp.Header { + if k == "Connection" { + continue + } + for _, subV := range v { + this.writer.Header().Add(k, subV) + } + } + + this.writer.Prepare(resp.ContentLength) + this.writer.WriteHeader(resp.StatusCode) + + _, _ = io.Copy(this.writer, resp.Body) + return nil +} + +func (this *ForwardProxy) forwardConnect() error { + defer this.req.log() + + hijacker, ok := this.writer.writer.(http.Hijacker) + if !ok { + this.req.serverError(this.writer) + return nil + } + + clientConn, _, err := hijacker.Hijack() + if err != nil { + this.req.serverError(this.writer) + this.req.addError(err) + return nil + } + + _, _ = clientConn.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) + + hostConn, err := net.DialTimeout("tcp", this.req.host, 30*time.Second) + if err != nil { + this.req.serverError(this.writer) + this.req.addError(err) + return nil + } + + go func() { + _, _ = io.Copy(clientConn, hostConn) + _ = clientConn.Close() + _ = hostConn.Close() + }() + go func() { + _, _ = io.Copy(hostConn, clientConn) + _ = clientConn.Close() + _ = hostConn.Close() + }() + return nil +} + +func (this *ForwardProxy) forwardMitm() error { + hijacker, ok := this.writer.writer.(http.Hijacker) + this.req.log() + if !ok { + this.req.serverError(this.writer) + return nil + } + + clientConn, _, err := hijacker.Hijack() + if err != nil { + this.req.serverError(this.writer) + this.req.addError(err) + return nil + } + + _, _ = clientConn.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) + + hostName, _, _ := net.SplitHostPort(this.req.host) + if len(hostName) == 0 { + hostName = this.req.host + } + + mitmLocker.Lock() + clientConfig, ok := mitmCache[hostName] + mitmLocker.Unlock() + + if !ok { + certData, err := ioutil.ReadFile(Tea.Root + "/web/certs/teaweb.proxy.pem") + if err != nil { + logs.Error(err) + return nil + } + + certBlock, _ := pem.Decode(certData) + if err != nil { + logs.Error(err) + return nil + } + + rootCert, err := x509.ParseCertificate(certBlock.Bytes) + if err != nil { + logs.Error(err) + return nil + } + + keyData, err := ioutil.ReadFile(Tea.Root + "/web/certs/teaweb.proxy.key") + if err != nil { + logs.Error(err) + return nil + } + + block, _ := pem.Decode(keyData) + if err != nil { + logs.Error(err) + return nil + } + + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + logs.Error(err) + return nil + } + + config, err := mitm.NewConfig(rootCert, privateKey) + if err != nil { + this.req.serverError(this.writer) + this.req.addError(err) + return nil + } + + mitmLocker.Lock() + clientConfig = config.TLSForHost(hostName) + clientConfig.ServerName = hostName + mitmCache[hostName] = clientConfig + + // 清理 + if len(mitmCache)>= 100000 { + mitmCache = map[string]*tls.Config{} + } + mitmLocker.Unlock() + } + + hostConn, err := tls.Dial("tcp", this.req.host, nil) + if err != nil { + this.req.serverError(this.writer) + this.req.addError(err) + return nil + } + + client := tls.Server(clientConn, clientConfig) + + var closer = func() { + _ = hostConn.Close() + _ = clientConn.Close() + } + defer closer() + + clientReader := bufio.NewReader(client) + hostReader := bufio.NewReader(hostConn) + + accessLogOn := this.req.accessLog != nil && this.req.accessLog.On + + for { + rawReq, err := http.ReadRequest(clientReader) + if err != nil { + closer() + break + } + + var req *Request = nil + logResponse := false + if accessLogOn { + req = NewRequest(rawReq) + + req.responseWriter = NewResponseWriter(NewEmptyResponseWriter()) + req.accessLog = this.req.accessLog + req.enableStat = this.req.enableStat + req.host = this.req.host + req.method = rawReq.Method + req.uri = rawReq.URL.RequestURI() + req.rawScheme = "https" + req.scheme = "https" // 转发后的scheme + req.serverName = this.req.serverName + req.serverAddr = this.req.serverAddr + + _ = req.configure(this.req.server, 0, false) + + // watch + if this.req.isWatching { + // 判断如果Content-Length过长,则截断 + reqData, err := httputil.DumpRequest(req.raw, true) + if err == nil { + if len(reqData)> 100240 { + reqData = reqData[:100240] + } + req.requestData = reqData + } + + logResponse = true + } else { + max := 512 * 1024 // 512K + if req.accessLog != nil && lists.ContainsInt(req.accessLog.Fields, accesslogs.AccessLogFieldRequestBody) { + body, err := ioutil.ReadAll(req.raw.Body) + if err == nil { + if len(body)> max { + req.requestData = body[:max] + } else { + req.requestData = body + } + } + req.raw.Body = ioutil.NopCloser(bytes.NewReader(body)) + } + if req.accessLog != nil && lists.ContainsInt(this.req.accessLog.Fields, accesslogs.AccessLogFieldResponseBody) { + logResponse = true + } + } + } + + err = rawReq.Write(hostConn) + if err != nil { + if accessLogOn { + req.addError(err) + } + closer() + break + } + + resp, err := http.ReadResponse(hostReader, nil) + if err != nil { + if accessLogOn { + req.addError(err) + req.log() + } + + closer() + break + } + + if accessLogOn { + req.responseWriter.Prepare(resp.ContentLength) + req.responseWriter.WriteHeader(resp.StatusCode) + req.responseWriter.AddHeaders(resp.Header) + + if logResponse { + req.responseWriter.SetBodyCopying(true) + + bodyData, _ := ioutil.ReadAll(resp.Body) + req.responseWriter.body = bodyData + resp.Body = ioutil.NopCloser(bytes.NewReader(bodyData)) + } + } + + err = resp.Write(client) + if err != nil { + if accessLogOn { + req.addError(err) + req.log() + } + closer() + break + } + + req.log() + } + + return nil +} diff --git a/teaproxy/request_forward.go b/teaproxy/request_forward.go index d19d219..b08dd37 100644 --- a/teaproxy/request_forward.go +++ b/teaproxy/request_forward.go @@ -1,127 +1,37 @@ package teaproxy import ( - "bytes" - "github.com/TeaWeb/code/tealogs/accesslogs" - "github.com/TeaWeb/code/teautils" - "github.com/iwind/TeaGo/lists" - "io" - "io/ioutil" - "net" + "crypto/tls" "net/http" - "net/http/httputil" - "time" + "sync" ) +var mitmCache = map[string]*tls.Config{} // host => config +var mitmLocker = &sync.Mutex{} + // 正向代理 +// TODO 支持WebSocket func (this *Request) Forward(writer *ResponseWriter) error { - defer this.log() - - // watch - if this.isWatching { - // 判断如果Content-Length过长,则截断 - reqData, err := httputil.DumpRequest(this.raw, true) - if err == nil { - if len(reqData)> 100240 { - reqData = reqData[:100240] - } - this.requestData = reqData - } - - writer.SetBodyCopying(true) - } else { - max := 512 * 1024 // 512K - if this.accessLog != nil && lists.ContainsInt(this.accessLog.Fields, accesslogs.AccessLogFieldRequestBody) { - body, err := ioutil.ReadAll(this.raw.Body) - if err == nil { - if len(body)> max { - this.requestData = body[:max] - } else { - this.requestData = body - } - } - this.raw.Body = ioutil.NopCloser(bytes.NewReader(body)) - } - if this.accessLog != nil && lists.ContainsInt(this.accessLog.Fields, accesslogs.AccessLogFieldResponseBody) { - writer.SetBodyCopying(true) - } - } - if len(this.raw.URL.Scheme) == 0 { this.rawScheme = "https" } this.setProxyHeaders(this.raw.Header) - if this.method == http.MethodConnect { // connect - hostConn, err := net.DialTimeout("tcp", this.host, 30*time.Second) - if err != nil { - this.serverError(writer) - this.addError(err) - return nil - } - - hijacker, ok := writer.writer.(http.Hijacker) - if !ok { - this.serverError(writer) - this.addError(err) - return nil - } - - writer.WriteHeader(http.StatusOK) + enableMitm := this.server.ForwardHTTP.EnableMITM - clientConn, _, err := hijacker.Hijack() - if err != nil { - this.serverError(writer) - this.addError(err) - return nil + proxy := &ForwardProxy{ + req: this, + writer: writer, + } + if this.method == http.MethodConnect { // connect + if enableMitm { + return proxy.forwardMitm() + } else { + return proxy.forwardConnect() } - go func() { - _, _ = io.Copy(clientConn, hostConn) - _ = clientConn.Close() - _ = hostConn.Close() - }() - go func() { - _, _ = io.Copy(hostConn, clientConn) - _ = clientConn.Close() - _ = hostConn.Close() - }() } else { // http - this.raw.RequestURI = "" - - // 删除代理相关Header - for n, _ := range this.raw.Header { - if lists.ContainsString([]string{"Proxy-Connection", "Connection", "Proxy-Authorization"}, n) { - this.raw.Header.Del(n) - } - } - - client := teautils.SharedHttpClient(30 * time.Second) - resp, err := client.Do(this.raw) - if err != nil { - this.serverError(writer) - this.addError(err) - return nil - } - defer func() { - _ = resp.Body.Close() - }() - - for k, v := range resp.Header { - if k == "Connection" { - continue - } - for _, subV := range v { - writer.Header().Add(k, subV) - } - } - - writer.Prepare(resp.ContentLength) - writer.WriteHeader(resp.StatusCode) - - _, _ = io.Copy(writer, resp.Body) + return proxy.forwardHTTP() } - - return nil } diff --git a/teaproxy/request_forward_test.go b/teaproxy/request_forward_test.go new file mode 100644 index 0000000..8a52ab9 --- /dev/null +++ b/teaproxy/request_forward_test.go @@ -0,0 +1,89 @@ +package teaproxy + +import ( + "crypto/x509" + "encoding/pem" + "github.com/TeaWeb/code/teaproxy/mitm" + "github.com/iwind/TeaGo/Tea" + "io/ioutil" + "os" + "testing" + "time" +) + +func TestRequest_Forward_CA(t *testing.T) { + ca, privateKey, err := mitm.NewAuthority("teaweb.teaos.cn", "TeaWeb Authority", 10*365*24*time.Hour) + if err != nil { + t.Fatal(err) + } + t.Logf("%#v", ca.PublicKey) + + { + writer, err := os.OpenFile("teaweb.proxy.pem", os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + t.Fatal(err) + } + + err = pem.Encode(writer, &pem.Block{ + Type: "CERTIFICATE", + Bytes: ca.Raw, + }) + if err != nil { + t.Fatal(err) + } + } + + { + writer, err := os.OpenFile("teaweb.proxy.key", os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + t.Fatal(err) + } + + err = pem.Encode(writer, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + }) + if err != nil { + t.Fatal(err) + } + } + + t.Log("OK") +} + +func TestRequest_Forward_Decode(t *testing.T) { + { + data, err := ioutil.ReadFile(Tea.Root + "/web/certs/teaweb.proxy.pem") + if err != nil { + t.Fatal(err) + } + + for { + block, rest := pem.Decode(data) + if err != nil { + t.Fatal(err) + } + + t.Log(x509.ParseCertificate(block.Bytes)) + + if len(rest) == 0 { + break + } + data = rest + } + } + + { + data, err := ioutil.ReadFile(Tea.Root + "/web/certs/teaweb.proxy.key") + if err != nil { + t.Fatal(err) + } + + block, _ := pem.Decode(data) + if err != nil { + t.Fatal(err) + } + + t.Log(x509.ParsePKCS1PrivateKey(block.Bytes)) + } +} diff --git a/teaproxy/response_empty.go b/teaproxy/response_empty.go new file mode 100644 index 0000000..f28b5cf --- /dev/null +++ b/teaproxy/response_empty.go @@ -0,0 +1,26 @@ +package teaproxy + +import "net/http" + +// 空的响应Writer +type EmptyResponseWriter struct { + header http.Header +} + +func NewEmptyResponseWriter() *EmptyResponseWriter { + return &EmptyResponseWriter{ + header: http.Header{}, + } +} + +func (this *EmptyResponseWriter) Header() http.Header { + return this.header +} + +func (this *EmptyResponseWriter) Write([]byte) (int, error) { + return 0, nil +} + +func (this *EmptyResponseWriter) WriteHeader(statusCode int) { + +} diff --git a/teaweb/actions/default/proxy/update.go b/teaweb/actions/default/proxy/update.go index 75a7250..a1e6c26 100644 --- a/teaweb/actions/default/proxy/update.go +++ b/teaweb/actions/default/proxy/update.go @@ -97,6 +97,9 @@ func (this *UpdateAction) RunPost(params struct { TcpReadBufferSize int TcpWriteBufferSize int + // ForwardHTTP + ForwardHTTPEnableMITM bool + Must *actions.Must }) { server := teaconfigs.NewServerConfigFromId(params.ServerId) @@ -120,6 +123,8 @@ func (this *UpdateAction) RunPost(params struct { if params.TcpWriteBufferSize>= 0 { server.TCP.WriteBufferSize = params.TcpWriteBufferSize } + } else if server.ForwardHTTP != nil { // ForwardHTTP + server.ForwardHTTP.EnableMITM = params.ForwardHTTPEnableMITM } else { // HTTP server.Http = params.HttpOn server.Root = params.Root From 346775246f2c6f9ae7d8753d127e71e6410f9b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月28日 17:09:48 +0800 Subject: [PATCH 077/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8D=E6=AD=A3?= =?UTF-8?q?=E5=90=91=E4=BB=A3=E7=90=86=E6=97=A0=E6=B3=95=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E8=AE=BF=E9=97=AE=E6=97=A5=E5=BF=97=E8=AE=BE=E7=BD=AE=E7=9A=84?= =?UTF-8?q?Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/proxy/update.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/teaweb/actions/default/proxy/update.go b/teaweb/actions/default/proxy/update.go index a1e6c26..e52c086 100644 --- a/teaweb/actions/default/proxy/update.go +++ b/teaweb/actions/default/proxy/update.go @@ -124,7 +124,14 @@ func (this *UpdateAction) RunPost(params struct { server.TCP.WriteBufferSize = params.TcpWriteBufferSize } } else if server.ForwardHTTP != nil { // ForwardHTTP + server.Http = params.HttpOn + server.ForwardHTTP.EnableMITM = params.ForwardHTTPEnableMITM + + // 访问日志 + server.AccessLog = proxyutils.ParseAccessLogForm(this.Request) + + server.DisableStat = !params.EnableStat } else { // HTTP server.Http = params.HttpOn server.Root = params.Root From 44e999ec0804b13d13a13a7e0911c1ea9d84d9d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月28日 17:29:44 +0800 Subject: [PATCH 078/121] =?UTF-8?q?[proxy]MITM=E4=B9=9F=E6=94=AF=E6=8C=81X?= =?UTF-8?q?-Forward-*=E3=80=81X-Real-IP=E7=AD=89Header?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/proxy_forward.go | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/teaproxy/proxy_forward.go b/teaproxy/proxy_forward.go index 534e043..9ebfc1d 100644 --- a/teaproxy/proxy_forward.go +++ b/teaproxy/proxy_forward.go @@ -17,6 +17,7 @@ import ( "net" "net/http" "net/http/httputil" + "strings" "time" ) @@ -246,6 +247,7 @@ func (this *ForwardProxy) forwardMitm() error { closer() break } + this.setProxyHeaders(rawReq) var req *Request = nil logResponse := false @@ -345,3 +347,43 @@ func (this *ForwardProxy) forwardMitm() error { return nil } + +// 设置代理相关头部信息 +// 参考:https://tools.ietf.org/html/rfc7239 +func (this *ForwardProxy) setProxyHeaders(rawRequest *http.Request) { + remoteAddr := this.req.raw.RemoteAddr + host, _, err := net.SplitHostPort(remoteAddr) + if err == nil { + remoteAddr = host + } + + // x-real-ip + { + _, ok1 := rawRequest.Header["X-Real-IP"] + _, ok2 := rawRequest.Header["X-Real-Ip"] + if !ok1 && !ok2 { + rawRequest.Header["X-Real-IP"] = []string{remoteAddr} + } + } + + // X-Forwarded-For + { + forwardedFor, ok := rawRequest.Header["X-Forwarded-For"] + if ok { + rawRequest.Header["X-Forwarded-For"] = []string{strings.Join(forwardedFor, ", ") + ", " + remoteAddr} + } else { + rawRequest.Header["X-Forwarded-For"] = []string{remoteAddr} + } + } + + // others + rawRequest.Header.Set("X-Forwarded-By", this.req.serverAddr) + + if _, ok := rawRequest.Header["X-Forwarded-Host"]; !ok { + rawRequest.Header.Set("X-Forwarded-Host", this.req.host) + } + + if _, ok := rawRequest.Header["X-Forwarded-Proto"]; !ok { + rawRequest.Header.Set("X-Forwarded-Proto", this.req.rawScheme) + } +} From c261677fe071bd4f417ebc057de57f3b53a9f8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月28日 19:58:16 +0800 Subject: [PATCH 079/121] =?UTF-8?q?=E4=BF=AE=E5=A4=8DMaster=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E6=97=A0=E6=B3=95=E6=AD=A3=E5=B8=B8=E9=87=8D=E5=90=AF?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teautils/signal_windows.go | 5 ++- teaweb/cmd/web_shell.go | 92 +++++++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/teautils/signal_windows.go b/teautils/signal_windows.go index f48c22c..0b46e42 100644 --- a/teautils/signal_windows.go +++ b/teautils/signal_windows.go @@ -11,6 +11,7 @@ import ( "github.com/iwind/TeaGo/types" "net" "os" + "strconv" "syscall" ) @@ -20,7 +21,7 @@ const signalPipePath = `\\.\pipe\` + teaconst.TeaProcessName + `.signal.pipe` // 监听Signal func ListenSignal(f func(sig os.Signal), sig ...os.Signal) { - pipe, err := winio.ListenPipe(signalPipePath, nil) + pipe, err := winio.ListenPipe(signalPipePath+"."+strconv.Itoa(os.Getpid()), nil) if err != nil { logs.Error(err) return @@ -69,7 +70,7 @@ func ListenSignal(f func(sig os.Signal), sig ...os.Signal) { // 通知Signal func NotifySignal(proc *os.Process, sig os.Signal) error { - conn, err := winio.DialPipe(signalPipePath, nil) + conn, err := winio.DialPipe(signalPipePath+"."+strconv.Itoa(proc.Pid), nil) if err != nil { return errors.New("can not connect to signal pipe: " + err.Error()) } diff --git a/teaweb/cmd/web_shell.go b/teaweb/cmd/web_shell.go index b5feda9..4e4b2dc 100644 --- a/teaweb/cmd/web_shell.go +++ b/teaweb/cmd/web_shell.go @@ -14,6 +14,7 @@ import ( "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" "io" + logger "log" "net/http" _ "net/http/pprof" "os" @@ -26,6 +27,7 @@ import ( ) var sharedShell *WebShell = nil +var workerProcess *os.Process // 命令行相关封装 type WebShell struct { @@ -49,13 +51,6 @@ func (this *WebShell) Start(server *TeaGo.Server) { return } - // 当前PID文件句柄 - err := this.writePpid() - if err != nil { - logs.Println("[error]write pid file failed: '" + err.Error() + "'") - return - } - // 信号 teautils.ListenSignal(func(sig os.Signal) { logs.Println("[signal]" + sig.String()) @@ -93,11 +88,6 @@ func (this *WebShell) Start(server *TeaGo.Server) { } } - // 删除PID - err = teautils.DeletePid(Tea.Root + "/bin/pid") - if err != nil { - logs.Error(err) - } os.Exit(0) } }, syscall.SIGINT, syscall.SIGHUP, syscall.Signal(0x1e /**syscall.SIGUSR1**/), syscall.Signal(0x1f /**syscall.SIGUSR2**/), syscall.SIGTERM) @@ -176,7 +166,7 @@ func (this *WebShell) execArgs(writer io.Writer) bool { return this.ExecPprof(writer) } if this.hasArg(arg0, "master") { - return this.ExecMaster(writer) + return this.ExecMasterProcess(writer) } if this.hasArg(arg0, "worker") { return this.ExecWorker(writer) @@ -244,12 +234,12 @@ func (this *WebShell) ExecStop(writer io.Writer) bool { return true } - // 停止子进程 - err := proc.Kill() - if err != nil { - this.write(writer, teaconst.TeaProductName+" stop error:", err.Error()) - return true - } + // 通知关闭 + _ = teautils.NotifySignal(proc, syscall.SIGINT) + time.Sleep(1 * time.Second) + + // 停止进程 + _ = proc.Kill() // 在Windows上经常不能及时释放资源 _ = teautils.DeletePid(Tea.Root + "/bin/pid") @@ -278,17 +268,17 @@ func (this *WebShell) ExecReload(writer io.Writer) bool { func (this *WebShell) ExecRestart(writer io.Writer) bool { proc := this.checkPid() if proc != nil { - err := proc.Kill() - if err != nil { - this.write(writer, teaconst.TeaProductName+" stop error:", err.Error()) - return true - } + // 通知关闭 + _ = teautils.NotifySignal(proc, syscall.SIGINT) + time.Sleep(1 * time.Second) + + _ = proc.Kill() // 等待进程结束 - time.Sleep(1 * time.Second) + time.Sleep(100 * time.Millisecond) } - cmd := exec.Command(os.Args[0]) + cmd := exec.Command(os.Args[0], "master") err := cmd.Start() if err != nil { this.write(writer, teaconst.TeaProductName+" restart failed:", err.Error()) @@ -359,8 +349,49 @@ func (this *WebShell) ExecPprof(writer io.Writer) bool { return false } -// 守护进程 -func (this *WebShell) ExecMaster(writer io.Writer) bool { +// 启动守护进程 +func (this *WebShell) ExecMasterProcess(writer io.Writer) bool { + // 当前PID文件句柄 + err := this.writePid() + if err != nil { + logs.Println("[error]write pid file failed: '" + err.Error() + "'") + return true + } + + logWriter, _ := os.OpenFile(Tea.Root+"/logs/master.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if logWriter != nil { + logger.SetOutput(logWriter) + } + + teautils.ListenSignal(func(sig os.Signal) { + logger.Println("[signal]" + sig.String()) + + if sig == syscall.SIGHUP { // 重置 + if workerProcess != nil { + _ = teautils.NotifySignal(workerProcess, sig) + } + } else if sig == syscall.Signal(0x1e /**syscall.SIGUSR1**/) { // 刷新代理状态 + if workerProcess != nil { + _ = teautils.NotifySignal(workerProcess, sig) + } + } else if sig == syscall.Signal(0x1f /**syscall.SIGUSR2**/) { // 同步 + if workerProcess != nil { + _ = teautils.NotifySignal(workerProcess, sig) + } + } else { + if workerProcess != nil { + _ = workerProcess.Kill() + } + + // 删除PID + err := teautils.DeletePid(Tea.Root + "/bin/pid") + if err != nil { + logs.Error(err) + } + os.Exit(0) + } + }, syscall.SIGINT, syscall.SIGHUP, syscall.Signal(0x1e /**syscall.SIGUSR1**/), syscall.Signal(0x1f /**syscall.SIGUSR2**/), syscall.SIGTERM) + // 生成子进程,当前进程变成守护进程 for { cmd := exec.Command(os.Args[0], "worker") @@ -369,6 +400,7 @@ func (this *WebShell) ExecMaster(writer io.Writer) bool { this.write(writer, teaconst.TeaProductName+" start failed:", err.Error()) break } + workerProcess = cmd.Process _ = cmd.Wait() } return true @@ -397,8 +429,8 @@ func (this *WebShell) ExecWorker(writer io.Writer) bool { } // 写入PID -func (this *WebShell) writePpid() error { - return teautils.WritePpid(Tea.Root + Tea.DS + "bin" + Tea.DS + "pid") +func (this *WebShell) writePid() error { + return teautils.WritePid(Tea.Root + Tea.DS + "bin" + Tea.DS + "pid") } // 检查PID From 31872b332cf8ae5df9509a2e3b6f555f0cd01879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月29日 08:36:17 +0800 Subject: [PATCH 080/121] =?UTF-8?q?[proxy]=E4=BF=AE=E6=94=B9=E7=BB=86?= =?UTF-8?q?=E8=8A=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/server.go | 2 +- teaconfigs/ssl.go | 26 +++++++++++++++++++ teaproxy/manager.go | 6 ++--- .../actions/default/proxy/proxyutils/menu.go | 4 +++ 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/teaconfigs/server.go b/teaconfigs/server.go index 9fc4792..f38a0f2 100644 --- a/teaconfigs/server.go +++ b/teaconfigs/server.go @@ -507,7 +507,7 @@ func (this *ServerConfig) AddListen(address string) { // 分解所有监听地址 func (this *ServerConfig) ParseListenAddresses() []string { result := []string{} - var reg = regexp.MustCompile(`:\s*\[\s*(\d+)\s*[,:-]\s*(\d+)\s*]$`) + var reg = regexp.MustCompile(`\[\s*(\d+)\s*[,:-]\s*(\d+)\s*]$`) for _, addr := range this.Listen { match := reg.FindStringSubmatch(addr) if len(match) == 0 { diff --git a/teaconfigs/ssl.go b/teaconfigs/ssl.go index e2ac959..972b6f7 100644 --- a/teaconfigs/ssl.go +++ b/teaconfigs/ssl.go @@ -6,8 +6,11 @@ import ( "errors" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" + "github.com/iwind/TeaGo/types" "io/ioutil" "net" + "regexp" + "strconv" "strings" ) @@ -234,3 +237,26 @@ func (this *SSLConfig) FindCertTask(certTaskId string) *SSLCertTask { func (this *SSLConfig) CAPool() *x509.CertPool { return this.clientCAPool } + +// 分解所有监听地址 +func (this *SSLConfig) ParseListenAddresses() []string { + result := []string{} + var reg = regexp.MustCompile(`\[\s*(\d+)\s*[,:-]\s*(\d+)\s*]$`) + for _, addr := range this.Listen { + match := reg.FindStringSubmatch(addr) + if len(match) == 0 { + result = append(result, addr) + } else { + min := types.Int(match[1]) + max := types.Int(match[2]) + if min> max { + min, max = max, min + } + for i := min; i <= max; i++ { + newAddr := reg.ReplaceAllString(addr, ":"+strconv.Itoa(i)) + result = append(result, newAddr) + } + } + } + return result +} diff --git a/teaproxy/manager.go b/teaproxy/manager.go index c0fbb8d..b8ffe69 100644 --- a/teaproxy/manager.go +++ b/teaproxy/manager.go @@ -148,7 +148,7 @@ func (this *Manager) ApplyServer(server *teaconfigs.ServerConfig) { // HTTPS if server.SSL != nil && server.SSL.On { server.SSL.Validate() - for _, address := range server.SSL.Listen { + for _, address := range server.SSL.ParseListenAddresses() { // 是否有端口 if shared.RegexpDigitNumber.MatchString(address) { address = ":" + address @@ -164,7 +164,7 @@ func (this *Manager) ApplyServer(server *teaconfigs.ServerConfig) { } else if server.IsTCP() { // TCP // TCP if server.TCP.TCPOn { - for _, address := range server.Listen { + for _, address := range server.ParseListenAddresses() { // 是否有端口 if shared.RegexpDigitNumber.MatchString(address) { address = ":" + address @@ -182,7 +182,7 @@ func (this *Manager) ApplyServer(server *teaconfigs.ServerConfig) { // TCP+TLS if server.SSL != nil && server.SSL.On { server.SSL.Validate() - for _, address := range server.SSL.Listen { + for _, address := range server.SSL.ParseListenAddresses() { // 是否有端口 if shared.RegexpDigitNumber.MatchString(address) { address = ":" + address diff --git a/teaweb/actions/default/proxy/proxyutils/menu.go b/teaweb/actions/default/proxy/proxyutils/menu.go index c5451d7..827f0a0 100644 --- a/teaweb/actions/default/proxy/proxyutils/menu.go +++ b/teaweb/actions/default/proxy/proxyutils/menu.go @@ -63,6 +63,8 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { index := strings.LastIndex(listen, ":") if index> -1 { ports = append(ports, ":"+listen[index+1:]) + } else { + ports = append(ports, ":"+listen) } } } @@ -71,6 +73,8 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { index := strings.LastIndex(listen, ":") if index> -1 { ports = append(ports, ":"+listen[index+1:]) + } else { + ports = append(ports, ":"+listen) } } } From 27e2a1b57f482d886b048b8d26a541b748cecc99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月29日 09:08:20 +0800 Subject: [PATCH 081/121] =?UTF-8?q?[proxy]WebSocket=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E6=97=B6=E8=BF=94=E5=9B=9E=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E7=A0=81=E6=94=B9=E6=88=90500?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_websocket.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/teaproxy/request_websocket.go b/teaproxy/request_websocket.go index 167b77c..e9000fd 100644 --- a/teaproxy/request_websocket.go +++ b/teaproxy/request_websocket.go @@ -147,6 +147,13 @@ func (this *Request) callWebsocket(writer *ResponseWriter) error { server, resp, err := dialer.Dial(wsURL.String(), header) if err != nil { + writer.statusCode = http.StatusInternalServerError + _ = client.Close() + + if server != nil { + _ = server.Close() + } + errString := "" if resp != nil && resp.Body != nil { data, _ := ioutil.ReadAll(resp.Body) From 1c25ea05656ab1267fbf59fb0647848e34eeb9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月29日 09:14:13 +0800 Subject: [PATCH 082/121] =?UTF-8?q?[waf]=E6=8B=A6=E6=88=AA=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1=E5=A2=9E=E5=8A=A0=E7=99=BE=E5=88=86=E6=AF=94=E6=95=B0?= =?UTF-8?q?=E5=AD=97=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/proxy/waf/day.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/teaweb/actions/default/proxy/waf/day.go b/teaweb/actions/default/proxy/waf/day.go index 4074972..a180377 100644 --- a/teaweb/actions/default/proxy/waf/day.go +++ b/teaweb/actions/default/proxy/waf/day.go @@ -10,6 +10,7 @@ import ( "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/utils/time" + "math" "net/http" "regexp" "time" @@ -143,6 +144,18 @@ func (this *DayAction) Run(params struct { logs.Error(err) this.Data["stat"] = []maps.Map{} } else { + // 计算百分比 + total := 0 + for _, m := range stat { + total += m.GetInt("count") + } + if total> 0 { + for _, m := range stat { + percent := math.Ceil(float64(m.GetInt("count"))*10000/float64(total)) / 100 + m["name"] = m.GetString("name") + " " + fmt.Sprintf("%.2f", percent) + "%" + } + } + this.Data["stat"] = stat } } From 74c6287180933547d142c0f0547dc874f066fc2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月29日 09:17:34 +0800 Subject: [PATCH 083/121] =?UTF-8?q?[waf]=E6=94=AF=E6=8C=81=E4=BB=8EHeader?= =?UTF-8?q?=E4=B8=AD=E8=AF=BB=E5=8F=96=E7=94=A8=E6=88=B7=E6=A0=87=E8=AF=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teawaf/checkpoints/cc.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/teawaf/checkpoints/cc.go b/teawaf/checkpoints/cc.go index 7241995..d11d306 100644 --- a/teawaf/checkpoints/cc.go +++ b/teawaf/checkpoints/cc.go @@ -95,6 +95,16 @@ func (this *CCCheckpoint) RequestValue(req *requests.Request, param string, opti } key = "USER@" + userType + "@" + userField + "@" + v } + case "header": + if len(userField) == 0 { + key = this.ip(req) + } else { + v := req.Header.Get(userField) + if userIndexInt> 0 && len(v)> userIndexInt { + v = v[userIndexInt:] + } + key = "USER@" + userType + "@" + userField + "@" + v + } default: key = this.ip(req) } @@ -162,6 +172,10 @@ func (this *CCCheckpoint) Options() []OptionInterface { "name": "POST参数", "value": "post", }, + { + "name": "HTTP Header", + "value": "header", + }, }) options = append(options, option) } From cafff6a26f7b1b1ec00aa2fe30aa79f9e9a92d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月29日 16:10:25 +0800 Subject: [PATCH 084/121] =?UTF-8?q?=E6=93=8D=E4=BD=9C=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E4=B8=AD=E8=AE=B0=E5=BD=95=E5=A4=B1=E8=B4=A5=E7=9A=84=E7=99=BB?= =?UTF-8?q?=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/login/index.go | 34 +++++++++++++++++++-------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/teaweb/actions/default/login/index.go b/teaweb/actions/default/login/index.go index 90f71b0..db4d100 100644 --- a/teaweb/actions/default/login/index.go +++ b/teaweb/actions/default/login/index.go @@ -51,16 +51,6 @@ func (this *IndexAction) RunPost(params struct { Must *actions.Must Auth *helpers.UserShouldAuth }) { - // 记录登录 - go func() { - err := teadb.AuditLogDAO().InsertOne(audits.NewLog(params.Username, audits.ActionLogin, "登录", map[string]string{ - "ip": this.RequestRemoteIP(), - })) - if err != nil { - logs.Error(err) - } - }() - // 检查IP限制 if !configs.SharedAdminConfig().AllowIP(this.RequestRemoteIP()) { this.ResponseWriter.WriteHeader(http.StatusForbidden) @@ -85,14 +75,17 @@ func (this *IndexAction) RunPost(params struct { // 检查token if len(params.Token) <= 32 { + this.log(params.Username, false) this.Fail("请通过登录页面登录") } timestampString := params.Token[32:] if stringutil.Md5(TokenSalt+timestampString) != params.Token[:32] { + this.log(params.Username, false) this.FailField("refresh", "登录页面已过期,请刷新后重试") } timestamp := types.Int64(timestampString) if timestamp < time.Now().Unix()-1800 { + this.log(params.Username, false) this.FailField("refresh", "登录页面已过期,请刷新后重试") } @@ -102,12 +95,14 @@ func (this *IndexAction) RunPost(params struct { if user != nil { // 错误次数 if user.CountLoginTries()>= 3 { + this.log(params.Username, false) this.Fail("登录失败已超过3次,系统被锁定,需要重置服务后才能继续") } // 密码错误 if !adminConfig.ComparePassword(params.Password, user.Password) { user.IncreaseLoginTries() + this.log(params.Username, false) this.Fail("登录失败,请检查用户名密码") } @@ -128,9 +123,28 @@ func (this *IndexAction) RunPost(params struct { } } + this.log(params.Username, true) this.Next("/", nil, "").Success() return } + this.log(params.Username, false) this.Fail("登录失败,请检查用户名密码") } + +func (this *IndexAction) log(username string, success bool) { + go func() { + var message string + if success { + message = "登录成功" + } else { + message = "登录失败" + } + err := teadb.AuditLogDAO().InsertOne(audits.NewLog(username, audits.ActionLogin, message, map[string]string{ + "ip": this.RequestRemoteIP(), + })) + if err != nil { + logs.Error(err) + } + }() +} From 2d0b072aba97aa5e3114ef4e61b2ab12d555e37d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月29日 20:25:53 +0800 Subject: [PATCH 085/121] =?UTF-8?q?[waf]=E8=A7=84=E5=88=99=E9=9B=86?= =?UTF-8?q?=E5=8A=A8=E4=BD=9C=E6=94=AF=E6=8C=81=E8=B7=B3=E8=BD=AC=E5=88=B0?= =?UTF-8?q?=E4=B8=8B=E4=B8=80=E4=B8=AA=E8=A7=84=E5=88=99=E5=88=86=E7=BB=84?= =?UTF-8?q?=E6=88=96=E8=80=85=E4=B8=8B=E4=B8=80=E4=B8=AA=E8=A7=84=E5=88=99?= =?UTF-8?q?=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/request_waf.go | 6 +- teastats/waf_block_all_period.go | 4 +- teawaf/action_allow.go | 14 ++ teawaf/{actions => }/action_block.go | 5 +- teawaf/{actions => }/action_captcha.go | 7 +- teawaf/{actions => }/action_definition.go | 5 +- teawaf/action_go_group.go | 34 +++ teawaf/action_go_set.go | 37 +++ teawaf/{actions => }/action_instance.go | 2 +- teawaf/action_log.go | 13 + teawaf/action_type.go | 21 ++ teawaf/action_utils.go | 82 +++++++ teawaf/action_utils_test.go | 29 +++ teawaf/actions/action_allow.go | 11 - teawaf/actions/action_log.go | 12 - teawaf/actions/action_type.go | 16 -- teawaf/actions/utils.go | 42 ---- teawaf/requests/request.go | 4 + teawaf/{rules => }/rule.go | 2 +- teawaf/{rules => }/rule_group.go | 2 +- teawaf/{rules => }/rule_operator.go | 2 +- teawaf/{rules => }/rule_operator_test.go | 2 +- teawaf/{rules => }/rule_set.go | 7 +- teawaf/{rules => }/rule_set_test.go | 2 +- teawaf/{rules => }/rule_test.go | 2 +- teawaf/template.go | 231 +++++++++--------- teawaf/template_test.go | 33 ++- teawaf/waf.go | 48 ++-- teawaf/waf_test.go | 22 +- teaweb/actions/default/proxy/waf/add.go | 8 +- teaweb/actions/default/proxy/waf/detail.go | 3 +- teaweb/actions/default/proxy/waf/export.go | 6 +- teaweb/actions/default/proxy/waf/group.go | 47 +++- teaweb/actions/default/proxy/waf/groupAdd.go | 4 +- teaweb/actions/default/proxy/waf/ruleAdd.go | 39 ++- .../actions/default/proxy/waf/ruleUpdate.go | 41 ++-- teaweb/actions/default/proxy/waf/rules.go | 6 +- teaweb/actions/default/proxy/waf/test.go | 10 +- teaweb/actions/default/proxy/waf/update.go | 8 +- 39 files changed, 533 insertions(+), 336 deletions(-) create mode 100644 teawaf/action_allow.go rename teawaf/{actions => }/action_block.go (90%) rename teawaf/{actions => }/action_captcha.go (88%) rename teawaf/{actions => }/action_definition.go (71%) create mode 100644 teawaf/action_go_group.go create mode 100644 teawaf/action_go_set.go rename teawaf/{actions => }/action_instance.go (60%) create mode 100644 teawaf/action_log.go create mode 100644 teawaf/action_type.go create mode 100644 teawaf/action_utils.go create mode 100644 teawaf/action_utils_test.go delete mode 100644 teawaf/actions/action_allow.go delete mode 100644 teawaf/actions/action_log.go delete mode 100644 teawaf/actions/action_type.go delete mode 100644 teawaf/actions/utils.go rename teawaf/{rules => }/rule.go (99%) rename teawaf/{rules => }/rule_group.go (99%) rename teawaf/{rules => }/rule_operator.go (99%) rename teawaf/{rules => }/rule_operator_test.go (95%) rename teawaf/{rules => }/rule_set.go (92%) rename teawaf/{rules => }/rule_set_test.go (99%) rename teawaf/{rules => }/rule_test.go (99%) diff --git a/teaproxy/request_waf.go b/teaproxy/request_waf.go index e506008..f100db0 100644 --- a/teaproxy/request_waf.go +++ b/teaproxy/request_waf.go @@ -1,7 +1,7 @@ package teaproxy import ( - "github.com/TeaWeb/code/teawaf/actions" + "github.com/TeaWeb/code/teawaf" "github.com/iwind/TeaGo/logs" "net/http" ) @@ -18,7 +18,7 @@ func (this *Request) callWAFRequest(writer *ResponseWriter) (blocked bool) { } if ruleSet != nil { - if ruleSet.Action != actions.ActionAllow { + if ruleSet.Action != teawaf.ActionAllow { this.SetAttr("waf_action", ruleSet.Action) this.SetAttr("waf_group", group.Id) this.SetAttr("waf_ruleset", ruleSet.Id) @@ -43,7 +43,7 @@ func (this *Request) callWAFResponse(resp *http.Response, writer *ResponseWriter } if ruleSet != nil { - if ruleSet.Action != actions.ActionAllow { + if ruleSet.Action != teawaf.ActionAllow { this.SetAttr("waf_action", ruleSet.Action) this.SetAttr("waf_group", group.Id) this.SetAttr("waf_ruleset", ruleSet.Id) diff --git a/teastats/waf_block_all_period.go b/teastats/waf_block_all_period.go index 7a78846..caa7ee7 100644 --- a/teastats/waf_block_all_period.go +++ b/teastats/waf_block_all_period.go @@ -2,7 +2,7 @@ package teastats import ( "github.com/TeaWeb/code/tealogs/accesslogs" - "github.com/TeaWeb/code/teawaf/actions" + "github.com/TeaWeb/code/teawaf" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" "strings" @@ -72,7 +72,7 @@ func (this *WAFBlockAllPeriodFilter) Filter(accessLog *accesslogs.AccessLog) { if !ok { return } - if wafAction != actions.ActionBlock { + if wafAction != teawaf.ActionBlock { return } diff --git a/teawaf/action_allow.go b/teawaf/action_allow.go new file mode 100644 index 0000000..9929619 --- /dev/null +++ b/teawaf/action_allow.go @@ -0,0 +1,14 @@ +package teawaf + +import ( + "github.com/TeaWeb/code/teawaf/requests" + "net/http" +) + +type AllowAction struct { +} + +func (this *AllowAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) { + // do nothing + return true +} diff --git a/teawaf/actions/action_block.go b/teawaf/action_block.go similarity index 90% rename from teawaf/actions/action_block.go rename to teawaf/action_block.go index bebb359..d5c9f8f 100644 --- a/teawaf/actions/action_block.go +++ b/teawaf/action_block.go @@ -1,7 +1,8 @@ -package actions +package teawaf import ( "github.com/TeaWeb/code/teautils" + "github.com/TeaWeb/code/teawaf/requests" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/logs" "io" @@ -23,7 +24,7 @@ type BlockAction struct { URL string `yaml:"url" json:"url"` } -func (this *BlockAction) Perform(request *http.Request, writer http.ResponseWriter) (allow bool) { +func (this *BlockAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) { if writer != nil { if this.StatusCode> 0 { writer.WriteHeader(this.StatusCode) diff --git a/teawaf/actions/action_captcha.go b/teawaf/action_captcha.go similarity index 88% rename from teawaf/actions/action_captcha.go rename to teawaf/action_captcha.go index b74fe55..1c15b47 100644 --- a/teawaf/actions/action_captcha.go +++ b/teawaf/action_captcha.go @@ -1,9 +1,10 @@ -package actions +package teawaf import ( "bytes" "encoding/base64" "fmt" + "github.com/TeaWeb/code/teawaf/requests" "github.com/dchest/captcha" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/types" @@ -21,7 +22,7 @@ const ( type CaptchaAction struct { } -func (this *CaptchaAction) Perform(request *http.Request, writer http.ResponseWriter) (allow bool) { +func (this *CaptchaAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) { // TEAWEB_CAPTCHA: cookie, err := request.Cookie("TEAWEB_WAF_CAPTCHA") if err == nil && cookie != nil && len(cookie.Value)> 32 { @@ -48,7 +49,7 @@ func (this *CaptchaAction) Perform(request *http.Request, writer http.ResponseWr Path: "/", // all of dirs }) - http.Redirect(writer, request, request.URL.String(), http.StatusTemporaryRedirect) + http.Redirect(writer, request.Raw(), request.URL.String(), http.StatusTemporaryRedirect) return false } diff --git a/teawaf/actions/action_definition.go b/teawaf/action_definition.go similarity index 71% rename from teawaf/actions/action_definition.go rename to teawaf/action_definition.go index e61b995..e34aa72 100644 --- a/teawaf/actions/action_definition.go +++ b/teawaf/action_definition.go @@ -1,4 +1,6 @@ -package actions +package teawaf + +import "reflect" // action definition type ActionDefinition struct { @@ -6,4 +8,5 @@ type ActionDefinition struct { Code ActionString Description string Instance ActionInterface + Type reflect.Type } diff --git a/teawaf/action_go_group.go b/teawaf/action_go_group.go new file mode 100644 index 0000000..34fd8e0 --- /dev/null +++ b/teawaf/action_go_group.go @@ -0,0 +1,34 @@ +package teawaf + +import ( + "github.com/TeaWeb/code/teawaf/requests" + "github.com/iwind/TeaGo/logs" + "net/http" +) + +type GoGroupAction struct { + GroupId string `yaml:"groupId" json:"groupId"` +} + +func (this *GoGroupAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) { + group := waf.FindRuleGroup(this.GroupId) + if group == nil || !group.On { + return true + } + + b, set, err := group.MatchRequest(request) + if err != nil { + logs.Error(err) + return true + } + + if !b { + return true + } + + actionObject := FindActionInstance(set.Action, set.ActionOptions) + if actionObject == nil { + return true + } + return actionObject.Perform(waf, request, writer) +} diff --git a/teawaf/action_go_set.go b/teawaf/action_go_set.go new file mode 100644 index 0000000..3fcc8c9 --- /dev/null +++ b/teawaf/action_go_set.go @@ -0,0 +1,37 @@ +package teawaf + +import ( + "github.com/TeaWeb/code/teawaf/requests" + "github.com/iwind/TeaGo/logs" + "net/http" +) + +type GoSetAction struct { + GroupId string `yaml:"groupId" json:"groupId"` + SetId string `yaml:"setId" json:"setId"` +} + +func (this *GoSetAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) { + group := waf.FindRuleGroup(this.GroupId) + if group == nil || !group.On { + return true + } + set := group.FindRuleSet(this.SetId) + if set == nil || !set.On { + return true + } + + b, err := set.MatchRequest(request) + if err != nil { + logs.Error(err) + return true + } + if !b { + return true + } + actionObject := FindActionInstance(set.Action, set.ActionOptions) + if actionObject == nil { + return true + } + return actionObject.Perform(waf, request, writer) +} diff --git a/teawaf/actions/action_instance.go b/teawaf/action_instance.go similarity index 60% rename from teawaf/actions/action_instance.go rename to teawaf/action_instance.go index 5adffae..a48fdc1 100644 --- a/teawaf/actions/action_instance.go +++ b/teawaf/action_instance.go @@ -1,4 +1,4 @@ -package actions +package teawaf type Action struct { diff --git a/teawaf/action_log.go b/teawaf/action_log.go new file mode 100644 index 0000000..1976613 --- /dev/null +++ b/teawaf/action_log.go @@ -0,0 +1,13 @@ +package teawaf + +import ( + "github.com/TeaWeb/code/teawaf/requests" + "net/http" +) + +type LogAction struct { +} + +func (this *LogAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) { + return true +} diff --git a/teawaf/action_type.go b/teawaf/action_type.go new file mode 100644 index 0000000..3c3a74e --- /dev/null +++ b/teawaf/action_type.go @@ -0,0 +1,21 @@ +package teawaf + +import ( + "github.com/TeaWeb/code/teawaf/requests" + "net/http" +) + +type ActionString = string + +const ( + ActionLog = "log" // allow and log + ActionBlock = "block" // block + ActionCaptcha = "captcha" // block and show captcha + ActionAllow = "allow" // allow + ActionGoGroup = "go_group" // go to next rule group + ActionGoSet = "go_set" // go to next rule set +) + +type ActionInterface interface { + Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) +} diff --git a/teawaf/action_utils.go b/teawaf/action_utils.go new file mode 100644 index 0000000..62f353c --- /dev/null +++ b/teawaf/action_utils.go @@ -0,0 +1,82 @@ +package teawaf + +import ( + "github.com/iwind/TeaGo/maps" + "reflect" +) + +var AllActions = []*ActionDefinition{ + { + Name: "阻止", + Code: ActionBlock, + Instance: new(BlockAction), + }, + { + Name: "允许通过", + Code: ActionAllow, + Instance: new(AllowAction), + }, + { + Name: "允许并记录日志", + Code: ActionLog, + Instance: new(LogAction), + }, + { + Name: "Captcha验证码", + Code: ActionCaptcha, + Instance: new(CaptchaAction), + }, + { + Name: "跳到下一个规则分组", + Code: ActionGoGroup, + Instance: new(GoGroupAction), + Type: reflect.TypeOf(new(GoGroupAction)).Elem(), + }, + { + Name: "跳到下一个规则集", + Code: ActionGoSet, + Instance: new(GoSetAction), + Type: reflect.TypeOf(new(GoSetAction)).Elem(), + }, +} + +func FindActionInstance(action ActionString, options maps.Map) ActionInterface { + for _, def := range AllActions { + if def.Code == action { + if def.Type != nil { + // create new instance + ptrValue := reflect.New(def.Type) + instance := ptrValue.Interface().(ActionInterface) + + if len(options)> 0 { + count := def.Type.NumField() + for i := 0; i < count; i++ { + field := def.Type.Field(i) + tag, ok := field.Tag.Lookup("yaml") + if ok { + v, ok := options[tag] + if ok && reflect.TypeOf(v) == field.Type { + ptrValue.Elem().FieldByName(field.Name).Set(reflect.ValueOf(v)) + } + } + } + } + + return instance + } + + // return shared instance + return def.Instance + } + } + return nil +} + +func FindActionName(action ActionString) string { + for _, def := range AllActions { + if def.Code == action { + return def.Name + } + } + return "" +} diff --git a/teawaf/action_utils_test.go b/teawaf/action_utils_test.go new file mode 100644 index 0000000..342348a --- /dev/null +++ b/teawaf/action_utils_test.go @@ -0,0 +1,29 @@ +package teawaf + +import ( + "github.com/iwind/TeaGo/assert" + "github.com/iwind/TeaGo/maps" + "runtime" + "testing" +) + +func TestFindActionInstance(t *testing.T) { + a := assert.NewAssertion(t) + + t.Logf("ActionBlock: %p", FindActionInstance(ActionBlock, nil)) + t.Logf("ActionBlock: %p", FindActionInstance(ActionBlock, nil)) + t.Logf("ActionGoGroup: %p", FindActionInstance(ActionGoGroup, nil)) + t.Logf("ActionGoGroup: %p", FindActionInstance(ActionGoGroup, nil)) + t.Logf("ActionGoSet: %p", FindActionInstance(ActionGoSet, nil)) + t.Logf("ActionGoSet: %p", FindActionInstance(ActionGoSet, nil)) + t.Logf("ActionGoSet: %#v", FindActionInstance(ActionGoSet, maps.Map{"groupId": "a", "setId": "b",})) + + a.IsTrue(FindActionInstance(ActionGoSet, nil) != FindActionInstance(ActionGoSet, nil)) +} + +func BenchmarkFindActionInstance(b *testing.B) { + runtime.GOMAXPROCS(1) + for i := 0; i < b.N; i++ { + FindActionInstance(ActionGoSet, nil) + } +} diff --git a/teawaf/actions/action_allow.go b/teawaf/actions/action_allow.go deleted file mode 100644 index 828a3f8..0000000 --- a/teawaf/actions/action_allow.go +++ /dev/null @@ -1,11 +0,0 @@ -package actions - -import "net/http" - -type AllowAction struct { -} - -func (this *AllowAction) Perform(request *http.Request, writer http.ResponseWriter) (allow bool) { - // do nothing - return true -} diff --git a/teawaf/actions/action_log.go b/teawaf/actions/action_log.go deleted file mode 100644 index 26e3519..0000000 --- a/teawaf/actions/action_log.go +++ /dev/null @@ -1,12 +0,0 @@ -package actions - -import ( - "net/http" -) - -type LogAction struct { -} - -func (this *LogAction) Perform(request *http.Request, writer http.ResponseWriter) (allow bool) { - return true -} diff --git a/teawaf/actions/action_type.go b/teawaf/actions/action_type.go deleted file mode 100644 index d4772fc..0000000 --- a/teawaf/actions/action_type.go +++ /dev/null @@ -1,16 +0,0 @@ -package actions - -import "net/http" - -type ActionString = string - -const ( - ActionLog = "log" // allow and log - ActionBlock = "block" // block - ActionCaptcha = "captcha" // block and show captcha - ActionAllow = "allow" // allow -) - -type ActionInterface interface { - Perform(request *http.Request, writer http.ResponseWriter) (allow bool) -} diff --git a/teawaf/actions/utils.go b/teawaf/actions/utils.go deleted file mode 100644 index 1fe0855..0000000 --- a/teawaf/actions/utils.go +++ /dev/null @@ -1,42 +0,0 @@ -package actions - -var AllActions = []*ActionDefinition{ - { - Name: "阻止", - Code: ActionBlock, - Instance: new(BlockAction), - }, - { - Name: "允许通过", - Code: ActionAllow, - Instance: new(AllowAction), - }, - { - Name: "允许并记录日志", - Code: ActionLog, - Instance: new(LogAction), - }, - { - Name: "Captcha验证码", - Code: ActionCaptcha, - Instance: new(CaptchaAction), - }, -} - -func FindActionInstance(action ActionString) ActionInterface { - for _, def := range AllActions { - if def.Code == action { - return def.Instance - } - } - return nil -} - -func FindActionName(action ActionString) string { - for _, def := range AllActions { - if def.Code == action { - return def.Name - } - } - return "" -} diff --git a/teawaf/requests/request.go b/teawaf/requests/request.go index 7e3e741..a1b982b 100644 --- a/teawaf/requests/request.go +++ b/teawaf/requests/request.go @@ -18,6 +18,10 @@ func NewRequest(raw *http.Request) *Request { } } +func (this *Request) Raw() *http.Request { + return this.Request +} + func (this *Request) ReadBody(max int64) (data []byte, err error) { data, err = ioutil.ReadAll(io.LimitReader(this.Request.Body, max)) return diff --git a/teawaf/rules/rule.go b/teawaf/rule.go similarity index 99% rename from teawaf/rules/rule.go rename to teawaf/rule.go index 73a2614..2824a05 100644 --- a/teawaf/rules/rule.go +++ b/teawaf/rule.go @@ -1,4 +1,4 @@ -package rules +package teawaf import ( "bytes" diff --git a/teawaf/rules/rule_group.go b/teawaf/rule_group.go similarity index 99% rename from teawaf/rules/rule_group.go rename to teawaf/rule_group.go index 05f2a50..33ec2cf 100644 --- a/teawaf/rules/rule_group.go +++ b/teawaf/rule_group.go @@ -1,4 +1,4 @@ -package rules +package teawaf import ( "github.com/TeaWeb/code/teawaf/requests" diff --git a/teawaf/rules/rule_operator.go b/teawaf/rule_operator.go similarity index 99% rename from teawaf/rules/rule_operator.go rename to teawaf/rule_operator.go index 85b1bfe..5d70b42 100644 --- a/teawaf/rules/rule_operator.go +++ b/teawaf/rule_operator.go @@ -1,4 +1,4 @@ -package rules +package teawaf type RuleOperator = string type RuleCaseInsensitive = string diff --git a/teawaf/rules/rule_operator_test.go b/teawaf/rule_operator_test.go similarity index 95% rename from teawaf/rules/rule_operator_test.go rename to teawaf/rule_operator_test.go index c46bdd6..bb3507b 100644 --- a/teawaf/rules/rule_operator_test.go +++ b/teawaf/rule_operator_test.go @@ -1,4 +1,4 @@ -package rules +package teawaf import ( "fmt" diff --git a/teawaf/rules/rule_set.go b/teawaf/rule_set.go similarity index 92% rename from teawaf/rules/rule_set.go rename to teawaf/rule_set.go index 7c30fcf..0264ddf 100644 --- a/teawaf/rules/rule_set.go +++ b/teawaf/rule_set.go @@ -1,7 +1,6 @@ -package rules +package teawaf import ( - "github.com/TeaWeb/code/teawaf/actions" "github.com/TeaWeb/code/teawaf/requests" "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/utils/string" @@ -23,8 +22,8 @@ type RuleSet struct { Rules []*Rule `yaml:"rules" json:"rules"` Connector RuleConnector `yaml:"connector" json:"connector"` // rules connector - Action actions.ActionString `yaml:"action" json:"action"` - ActionOptions maps.Map `yaml:"actionOptions" json:"actionOptions"` // TODO TO BE IMPLEMENTED + Action ActionString `yaml:"action" json:"action"` + ActionOptions maps.Map `yaml:"actionOptions" json:"actionOptions"` // TODO TO BE IMPLEMENTED hasRules bool } diff --git a/teawaf/rules/rule_set_test.go b/teawaf/rule_set_test.go similarity index 99% rename from teawaf/rules/rule_set_test.go rename to teawaf/rule_set_test.go index 395f5fa..8237542 100644 --- a/teawaf/rules/rule_set_test.go +++ b/teawaf/rule_set_test.go @@ -1,4 +1,4 @@ -package rules +package teawaf import ( "bytes" diff --git a/teawaf/rules/rule_test.go b/teawaf/rule_test.go similarity index 99% rename from teawaf/rules/rule_test.go rename to teawaf/rule_test.go index 36bc569..efc4687 100644 --- a/teawaf/rules/rule_test.go +++ b/teawaf/rule_test.go @@ -1,4 +1,4 @@ -package rules +package teawaf import ( "github.com/TeaWeb/code/teawaf/checkpoints" diff --git a/teawaf/template.go b/teawaf/template.go index 563b338..5e69c71 100644 --- a/teawaf/template.go +++ b/teawaf/template.go @@ -1,10 +1,5 @@ package teawaf -import ( - "github.com/TeaWeb/code/teawaf/actions" - "github.com/TeaWeb/code/teawaf/rules" -) - // 感谢以下规则来源: // - Janusec: https://www.janusec.com/ func Template() *WAF { @@ -14,7 +9,7 @@ func Template() *WAF { // black list { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = false group.IsInbound = true group.Name = "白名单" @@ -23,15 +18,15 @@ func Template() *WAF { { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "IP白名单" set.Code = "9001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionAllow - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionAllow + set.AddRule(&Rule{ Param: "${remoteAddr}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `127\.0\.0\.1|0\.0\.0\.0`, IsCaseInsensitive: false, }) @@ -43,7 +38,7 @@ func Template() *WAF { // black list { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = false group.IsInbound = true group.Name = "黑名单" @@ -52,15 +47,15 @@ func Template() *WAF { { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "IP黑名单" set.Code = "10001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${remoteAddr}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `1\.1\.1\.1|2\.2\.2\.2`, IsCaseInsensitive: false, }) @@ -72,7 +67,7 @@ func Template() *WAF { // xss { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = true group.IsInbound = true group.Name = "XSS" @@ -80,15 +75,15 @@ func Template() *WAF { group.Description = "防跨站脚本攻击(Cross Site Scripting)" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "Javascript事件" set.Code = "1001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${requestURI}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `(onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick|onkeydown|onkeyup|onkeypress)\s*=`, // TODO more keywords here IsCaseInsensitive: true, }) @@ -96,15 +91,15 @@ func Template() *WAF { } { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "Javascript函数" set.Code = "1002" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${requestURI}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `(alert|eval|prompt|confirm)\s*\(`, // TODO more keywords here IsCaseInsensitive: true, }) @@ -112,15 +107,15 @@ func Template() *WAF { } { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "HTML标签" set.Code = "1003" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${requestURI}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `<(script|iframe|link)`, // TODO more keywords here IsCaseInsensitive: true, }) @@ -132,7 +127,7 @@ func Template() *WAF { // upload { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = true group.IsInbound = true group.Name = "文件上传" @@ -140,15 +135,15 @@ func Template() *WAF { group.Description = "防止上传可执行脚本文件到服务器" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "上传文件扩展名" set.Code = "2001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${requestUpload.ext}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `\.(php|jsp|aspx|asp|exe|asa|rb|py)\b`, // TODO more keywords here IsCaseInsensitive: true, }) @@ -160,7 +155,7 @@ func Template() *WAF { // web shell { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = true group.IsInbound = true group.Name = "Web Shell" @@ -168,15 +163,15 @@ func Template() *WAF { group.Description = "防止远程执行服务器命令" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "Web Shell" set.Code = "3001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\s*\(`, // TODO more keywords here IsCaseInsensitive: true, }) @@ -188,28 +183,28 @@ func Template() *WAF { // command injection { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = true group.IsInbound = true group.Name = "命令注入" group.Code = "commandInjection" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "命令注入" set.Code = "4001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${requestURI}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `\b(pwd|ls|ll|whoami|id|net\s+user)\s*$`, // TODO more keywords here IsCaseInsensitive: false, }) - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestBody}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `\b(pwd|ls|ll|whoami|id|net\s+user)\s*$`, // TODO more keywords here IsCaseInsensitive: false, }) @@ -221,7 +216,7 @@ func Template() *WAF { // path traversal { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = true group.IsInbound = true group.Name = "路径穿越" @@ -229,15 +224,15 @@ func Template() *WAF { group.Description = "防止读取网站目录之外的其他系统文件" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "路径穿越" set.Code = "5001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${requestURI}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `((\.+)(/+)){2,}`, // TODO more keywords here IsCaseInsensitive: false, }) @@ -249,7 +244,7 @@ func Template() *WAF { // special dirs { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = true group.IsInbound = true group.Name = "特殊目录" @@ -257,15 +252,15 @@ func Template() *WAF { group.Description = "防止通过Web访问到一些特殊目录" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "特殊目录" set.Code = "6001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorOr + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${requestPath}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `/\.(git|svn|htaccess|idea)\b`, // TODO more keywords here IsCaseInsensitive: true, }) @@ -277,7 +272,7 @@ func Template() *WAF { // sql injection { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = true group.IsInbound = true group.Name = "SQL注入" @@ -285,16 +280,16 @@ func Template() *WAF { group.Description = "防止SQL注入漏洞" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "Union SQL Injection" set.Code = "7001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock + set.Connector = RuleConnectorOr + set.Action = ActionBlock - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `union[\s/\*]+select`, IsCaseInsensitive: true, }) @@ -303,16 +298,16 @@ func Template() *WAF { } { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "SQL注释" set.Code = "7002" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock + set.Connector = RuleConnectorOr + set.Action = ActionBlock - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `/\*(!|\x00)`, IsCaseInsensitive: true, }) @@ -321,34 +316,34 @@ func Template() *WAF { } { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "SQL条件" set.Code = "7003" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock + set.Connector = RuleConnectorOr + set.Action = ActionBlock - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `\s(and|or|rlike)\s+(if|updatexml)\s*\(`, IsCaseInsensitive: true, }) - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `\s+(and|or|rlike)\s+(select|case)\s+`, IsCaseInsensitive: true, }) - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `\s+(and|or|procedure)\s+[\w\p{L}]+\s*=\s*[\w\p{L}]+(\s|$|--|#)`, IsCaseInsensitive: true, }) - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `\(\s*case\s+when\s+[\w\p{L}]+\s*=\s*[\w\p{L}]+\s+then\s+`, IsCaseInsensitive: true, }) @@ -357,16 +352,16 @@ func Template() *WAF { } { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "SQL函数" set.Code = "7004" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock + set.Connector = RuleConnectorOr + set.Action = ActionBlock - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `(updatexml|extractvalue|ascii|ord|char|chr|count|concat|rand|floor|substr|length|len|user|database|benchmark|analyse)\s*\(`, IsCaseInsensitive: true, }) @@ -375,16 +370,16 @@ func Template() *WAF { } { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "SQL附加语句" set.Code = "7005" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock + set.Connector = RuleConnectorOr + set.Action = ActionBlock - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${requestAll}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `;\s*(declare|use|drop|create|exec|delete|update|insert)\s`, IsCaseInsensitive: true, }) @@ -397,7 +392,7 @@ func Template() *WAF { // bot { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = false group.IsInbound = true group.Name = "网络爬虫" @@ -405,16 +400,16 @@ func Template() *WAF { group.Description = "禁止一些网络爬虫" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "常见网络爬虫" set.Code = "20001" - set.Connector = rules.RuleConnectorOr - set.Action = actions.ActionBlock + set.Connector = RuleConnectorOr + set.Action = ActionBlock - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${userAgent}", - Operator: rules.RuleOperatorMatch, + Operator: RuleOperatorMatch, Value: `Googlebot|AdsBot|bingbot|BingPreview|facebookexternalhit|Slurp|Sogou|proximic|Baiduspider|yandex|twitterbot|spider|python`, IsCaseInsensitive: true, }) @@ -427,7 +422,7 @@ func Template() *WAF { // cc { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = false group.IsInbound = true group.Name = "CC攻击" @@ -435,43 +430,43 @@ func Template() *WAF { group.Code = "cc" { - set := rules.NewRuleSet() + set := NewRuleSet() set.On = true set.Name = "CC请求数" set.Description = "限制单IP在一定时间内的请求数" set.Code = "8001" - set.Connector = rules.RuleConnectorAnd - set.Action = actions.ActionBlock - set.AddRule(&rules.Rule{ + set.Connector = RuleConnectorAnd + set.Action = ActionBlock + set.AddRule(&Rule{ Param: "${cc.requests}", - Operator: rules.RuleOperatorGt, + Operator: RuleOperatorGt, Value: "1000", CheckpointOptions: map[string]string{ "period": "60", }, IsCaseInsensitive: false, }) - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${remoteAddr}", - Operator: rules.RuleOperatorNotIPRange, + Operator: RuleOperatorNotIPRange, Value: `127.0.0.1/8`, IsCaseInsensitive: false, }) - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${remoteAddr}", - Operator: rules.RuleOperatorNotIPRange, + Operator: RuleOperatorNotIPRange, Value: `192.168.0.1/16`, IsCaseInsensitive: false, }) - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${remoteAddr}", - Operator: rules.RuleOperatorNotIPRange, + Operator: RuleOperatorNotIPRange, Value: `10.0.0.1/8`, IsCaseInsensitive: false, }) - set.AddRule(&rules.Rule{ + set.AddRule(&Rule{ Param: "${remoteAddr}", - Operator: rules.RuleOperatorNotIPRange, + Operator: RuleOperatorNotIPRange, Value: `172.16.0.1/12`, IsCaseInsensitive: false, }) @@ -484,7 +479,7 @@ func Template() *WAF { // custom { - group := rules.NewRuleGroup() + group := NewRuleGroup() group.On = true group.IsInbound = true group.Name = "自定义规则分组" diff --git a/teawaf/template_test.go b/teawaf/template_test.go index 555472c..52569c6 100644 --- a/teawaf/template_test.go +++ b/teawaf/template_test.go @@ -2,7 +2,6 @@ package teawaf import ( "bytes" - "github.com/TeaWeb/code/teawaf/actions" "github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" @@ -23,8 +22,8 @@ func Test_Template(t *testing.T) { t.Fatal(err) } - template.OnAction(func(action actions.ActionString) (goNext bool) { - return action != actions.ActionBlock + template.OnAction(func(action ActionString) (goNext bool) { + return action != ActionBlock }) testTemplate1001(a, t, template) @@ -53,7 +52,7 @@ func Test_Template2(t *testing.T) { } now := time.Now() - goNext, set, err := waf.MatchRequest(req, nil) + goNext, _, set, err := waf.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -81,7 +80,7 @@ func BenchmarkTemplate(b *testing.B) { b.Fatal(err) } - _, _, _ = waf.MatchRequest(req, nil) + _, _, _, _ = waf.MatchRequest(req, nil) } } @@ -90,7 +89,7 @@ func testTemplate1001(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -105,7 +104,7 @@ func testTemplate1002(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -120,7 +119,7 @@ func testTemplate1003(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -186,7 +185,7 @@ func testTemplate2001(a *assert.Assertion, t *testing.T, template *WAF) { req.Header.Add("Content-Type", writer.FormDataContentType()) - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -201,7 +200,7 @@ func testTemplate3001(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -216,7 +215,7 @@ func testTemplate4001(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -232,7 +231,7 @@ func testTemplate5001(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -247,7 +246,7 @@ func testTemplate5001(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -264,7 +263,7 @@ func testTemplate6001(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -279,7 +278,7 @@ func testTemplate6001(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -302,7 +301,7 @@ func testTemplate7001(a *assert.Assertion, t *testing.T, template *WAF) { if err != nil { t.Fatal(err) } - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } @@ -339,7 +338,7 @@ func testTemplate20001(a *assert.Assertion, t *testing.T, template *WAF) { t.Fatal(err) } req.Header.Set("User-Agent", bot) - _, result, err := template.MatchRequest(req, nil) + _, _, result, err := template.MatchRequest(req, nil) if err != nil { t.Fatal(err) } diff --git a/teawaf/waf.go b/teawaf/waf.go index 991eab5..7111f33 100644 --- a/teawaf/waf.go +++ b/teawaf/waf.go @@ -5,10 +5,8 @@ import ( "github.com/TeaWeb/code/teaconfigs/shared" "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" - "github.com/TeaWeb/code/teawaf/actions" "github.com/TeaWeb/code/teawaf/checkpoints" "github.com/TeaWeb/code/teawaf/requests" - "github.com/TeaWeb/code/teawaf/rules" "github.com/go-yaml/yaml" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" @@ -22,16 +20,16 @@ type WAF struct { Id string `yaml:"id" json:"id"` On bool `yaml:"on" json:"on"` Name string `yaml:"name" json:"name"` - Inbound []*rules.RuleGroup `yaml:"inbound" json:"inbound"` - Outbound []*rules.RuleGroup `yaml:"outbound" json:"outbound"` + Inbound []*RuleGroup `yaml:"inbound" json:"inbound"` + Outbound []*RuleGroup `yaml:"outbound" json:"outbound"` CreatedVersion string `yaml:"createdVersion" json:"createdVersion"` Cond []*shared.RequestCond `yaml:"cond" json:"cond"` - ActionBlock *actions.BlockAction `yaml:"actionBlock" json:"actionBlock"` // action block config + ActionBlock *BlockAction `yaml:"actionBlock" json:"actionBlock"` // action block config hasInboundRules bool hasOutboundRules bool - onActionCallback func(action actions.ActionString) (goNext bool) + onActionCallback func(action ActionString) (goNext bool) checkpointsMap map[string]checkpoints.CheckpointInterface // prefix => checkpoint } @@ -124,7 +122,7 @@ func (this *WAF) Init() error { return nil } -func (this *WAF) AddRuleGroup(ruleGroup *rules.RuleGroup) { +func (this *WAF) AddRuleGroup(ruleGroup *RuleGroup) { if ruleGroup.IsInbound { this.Inbound = append(this.Inbound, ruleGroup) } else { @@ -138,7 +136,7 @@ func (this *WAF) RemoveRuleGroup(ruleGroupId string) { } { - result := []*rules.RuleGroup{} + result := []*RuleGroup{} for _, group := range this.Inbound { if group.Id == ruleGroupId { continue @@ -149,7 +147,7 @@ func (this *WAF) RemoveRuleGroup(ruleGroupId string) { } { - result := []*rules.RuleGroup{} + result := []*RuleGroup{} for _, group := range this.Outbound { if group.Id == ruleGroupId { continue @@ -160,7 +158,7 @@ func (this *WAF) RemoveRuleGroup(ruleGroupId string) { } } -func (this *WAF) FindRuleGroup(ruleGroupId string) *rules.RuleGroup { +func (this *WAF) FindRuleGroup(ruleGroupId string) *RuleGroup { if len(ruleGroupId) == 0 { return nil } @@ -177,7 +175,7 @@ func (this *WAF) FindRuleGroup(ruleGroupId string) *rules.RuleGroup { return nil } -func (this *WAF) FindRuleGroupWithCode(ruleGroupCode string) *rules.RuleGroup { +func (this *WAF) FindRuleGroupWithCode(ruleGroupCode string) *RuleGroup { if len(ruleGroupCode) == 0 { return nil } @@ -206,7 +204,7 @@ func (this *WAF) MoveInboundRuleGroup(fromIndex int, toIndex int) { } group := this.Inbound[fromIndex] - result := []*rules.RuleGroup{} + result := []*RuleGroup{} for i := 0; i < len(this.Inbound); i++ { if i == fromIndex { continue @@ -235,7 +233,7 @@ func (this *WAF) MoveOutboundRuleGroup(fromIndex int, toIndex int) { } group := this.Outbound[fromIndex] - result := []*rules.RuleGroup{} + result := []*RuleGroup{} for i := 0; i < len(this.Outbound); i++ { if i == fromIndex { continue @@ -252,7 +250,7 @@ func (this *WAF) MoveOutboundRuleGroup(fromIndex int, toIndex int) { this.Outbound = result } -func (this *WAF) MatchRequest(rawReq *http.Request, writer http.ResponseWriter) (goNext bool, group *rules.RuleGroup, set *rules.RuleSet, err error) { +func (this *WAF) MatchRequest(rawReq *http.Request, writer http.ResponseWriter) (goNext bool, group *RuleGroup, set *RuleSet, err error) { if !this.hasInboundRules { return true, nil, nil, nil } @@ -267,14 +265,14 @@ func (this *WAF) MatchRequest(rawReq *http.Request, writer http.ResponseWriter) } if b { if this.onActionCallback == nil { - if set.Action == actions.ActionBlock && this.ActionBlock != nil { - return this.ActionBlock.Perform(rawReq, writer), group, set, nil + if set.Action == ActionBlock && this.ActionBlock != nil { + return this.ActionBlock.Perform(this, req, writer), group, set, nil } else { - actionObject := actions.FindActionInstance(set.Action) + actionObject := FindActionInstance(set.Action, set.ActionOptions) if actionObject == nil { return true, group, set, errors.New("no action called '" + set.Action + "'") } - goNext := actionObject.Perform(rawReq, writer) + goNext := actionObject.Perform(this, req, writer) return goNext, group, set, nil } } else { @@ -286,7 +284,7 @@ func (this *WAF) MatchRequest(rawReq *http.Request, writer http.ResponseWriter) return true, nil, nil, nil } -func (this *WAF) MatchResponse(rawReq *http.Request, rawResp *http.Response, writer http.ResponseWriter) (goNext bool, group *rules.RuleGroup, set *rules.RuleSet, err error) { +func (this *WAF) MatchResponse(rawReq *http.Request, rawResp *http.Response, writer http.ResponseWriter) (goNext bool, group *RuleGroup, set *RuleSet, err error) { if !this.hasOutboundRules { return true, nil, nil, nil } @@ -302,14 +300,14 @@ func (this *WAF) MatchResponse(rawReq *http.Request, rawResp *http.Response, wri } if b { if this.onActionCallback == nil { - if set.Action == actions.ActionBlock && this.ActionBlock != nil { - return this.ActionBlock.Perform(rawReq, writer), group, set, nil + if set.Action == ActionBlock && this.ActionBlock != nil { + return this.ActionBlock.Perform(this, req, writer), group, set, nil } else { - actionObject := actions.FindActionInstance(set.Action) + actionObject := FindActionInstance(set.Action, set.ActionOptions) if actionObject == nil { return true, group, set, errors.New("no action called '" + set.Action + "'") } - goNext := actionObject.Perform(rawReq, writer) + goNext := actionObject.Perform(this, req, writer) return goNext, group, set, nil } } else { @@ -380,7 +378,7 @@ func (this *WAF) CountOutboundRuleSets() int { return count } -func (this *WAF) OnAction(onActionCallback func(action actions.ActionString) (goNext bool)) { +func (this *WAF) OnAction(onActionCallback func(action ActionString) (goNext bool)) { this.onActionCallback = onActionCallback } @@ -417,7 +415,7 @@ func (this *WAF) MergeTemplate() (changedItems []string) { this.CreatedVersion = teaconst.TeaVersion template := Template() - groups := []*rules.RuleGroup{} + groups := []*RuleGroup{} groups = append(groups, template.Inbound...) groups = append(groups, template.Outbound...) for _, group := range groups { diff --git a/teawaf/waf_test.go b/teawaf/waf_test.go index 4c78c46..987ad4f 100644 --- a/teawaf/waf_test.go +++ b/teawaf/waf_test.go @@ -1,8 +1,6 @@ package teawaf import ( - "github.com/TeaWeb/code/teawaf/actions" - "github.com/TeaWeb/code/teawaf/rules" "github.com/iwind/TeaGo/assert" "net/http" "testing" @@ -11,24 +9,24 @@ import ( func TestWAF_MatchRequest(t *testing.T) { a := assert.NewAssertion(t) - set := rules.NewRuleSet() + set := NewRuleSet() set.Name = "Name_Age" - set.Connector = rules.RuleConnectorAnd - set.Rules = []*rules.Rule{ + set.Connector = RuleConnectorAnd + set.Rules = []*Rule{ { Param: "${arg.name}", - Operator: rules.RuleOperatorEqString, + Operator: RuleOperatorEqString, Value: "lu", }, { Param: "${arg.age}", - Operator: rules.RuleOperatorEq, + Operator: RuleOperatorEq, Value: "20", }, } - set.Action = actions.ActionBlock + set.Action = ActionBlock - group := rules.NewRuleGroup() + group := NewRuleGroup() group.AddRuleSet(set) group.IsInbound = true @@ -39,15 +37,15 @@ func TestWAF_MatchRequest(t *testing.T) { t.Fatal(err) } - waf.OnAction(func(action actions.ActionString) (goNext bool) { - return action != actions.ActionBlock + waf.OnAction(func(action ActionString) (goNext bool) { + return action != ActionBlock }) req, err := http.NewRequest(http.MethodGet, "http://teaos.cn/hello?name=lu&age=20", nil) if err != nil { t.Fatal(err) } - goNext, set, err := waf.MatchRequest(req, nil) + goNext, _, set, err := waf.MatchRequest(req, nil) if err != nil { t.Fatal(err) } diff --git a/teaweb/actions/default/proxy/waf/add.go b/teaweb/actions/default/proxy/waf/add.go index 96ebd94..9f2428f 100644 --- a/teaweb/actions/default/proxy/waf/add.go +++ b/teaweb/actions/default/proxy/waf/add.go @@ -4,8 +4,6 @@ import ( "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teaconfigs/shared" "github.com/TeaWeb/code/teawaf" - actions2 "github.com/TeaWeb/code/teawaf/actions" - "github.com/TeaWeb/code/teawaf/rules" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/actions" @@ -23,7 +21,7 @@ type AddAction actions.Action // 添加策略 func (this *AddAction) RunGet(params struct{}) { this.Data["groups"] = lists.Map(teawaf.Template().Inbound, func(k int, v interface{}) interface{} { - g := v.(*rules.RuleGroup) + g := v.(*teawaf.RuleGroup) return maps.Map{ "name": g.Name, "code": g.Code, @@ -63,7 +61,7 @@ func (this *AddAction) RunPost(params struct { waf := teawaf.NewWAF() waf.Name = params.Name waf.On = params.On - waf.ActionBlock = &actions2.BlockAction{ + waf.ActionBlock = &teawaf.BlockAction{ StatusCode: statusCode, Body: params.BlockBody, URL: params.BlockURL, @@ -79,7 +77,7 @@ func (this *AddAction) RunPost(params struct { template := teawaf.Template() for _, g := range template.Inbound { - newGroup := rules.NewRuleGroup() + newGroup := teawaf.NewRuleGroup() newGroup.Id = stringutil.Rand(16) newGroup.On = lists.ContainsString(params.GroupCodes, g.Code) newGroup.Code = g.Code diff --git a/teaweb/actions/default/proxy/waf/detail.go b/teaweb/actions/default/proxy/waf/detail.go index 5018518..b3c3977 100644 --- a/teaweb/actions/default/proxy/waf/detail.go +++ b/teaweb/actions/default/proxy/waf/detail.go @@ -3,7 +3,6 @@ package waf import ( "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teawaf" - "github.com/TeaWeb/code/teawaf/rules" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/maps" @@ -31,7 +30,7 @@ func (this *DetailAction) RunGet(params struct { } this.Data["groups"] = lists.Map(teawaf.Template().Inbound, func(k int, v interface{}) interface{} { - g := v.(*rules.RuleGroup) + g := v.(*teawaf.RuleGroup) group := waf.FindRuleGroupWithCode(g.Code) return maps.Map{ diff --git a/teaweb/actions/default/proxy/waf/export.go b/teaweb/actions/default/proxy/waf/export.go index 026115c..21a7327 100644 --- a/teaweb/actions/default/proxy/waf/export.go +++ b/teaweb/actions/default/proxy/waf/export.go @@ -2,7 +2,7 @@ package waf import ( "github.com/TeaWeb/code/teaconfigs" - "github.com/TeaWeb/code/teawaf/rules" + "github.com/TeaWeb/code/teawaf" "github.com/go-yaml/yaml" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/maps" @@ -26,8 +26,8 @@ func (this *ExportAction) RunGet(params struct { // 导出 if params.Export { waf1 := waf.Copy() - waf1.Inbound = []*rules.RuleGroup{} - waf1.Outbound = []*rules.RuleGroup{} + waf1.Inbound = []*teawaf.RuleGroup{} + waf1.Outbound = []*teawaf.RuleGroup{} if len(params.GroupIds)> 0 { groupIds := strings.Split(params.GroupIds, ",") for _, groupId := range groupIds { diff --git a/teaweb/actions/default/proxy/waf/group.go b/teaweb/actions/default/proxy/waf/group.go index 1f92bce..98a65a1 100644 --- a/teaweb/actions/default/proxy/waf/group.go +++ b/teaweb/actions/default/proxy/waf/group.go @@ -2,13 +2,11 @@ package waf import ( "github.com/TeaWeb/code/teaconfigs" - "github.com/TeaWeb/code/teawaf/rules" + "github.com/TeaWeb/code/teawaf" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/maps" "strings" - - wafactions "github.com/TeaWeb/code/teawaf/actions" ) type GroupAction actions.Action @@ -43,12 +41,41 @@ func (this *GroupAction) RunGet(params struct { // rule sets this.Data["sets"] = lists.Map(group.RuleSets, func(k int, v interface{}) interface{} { - set := v.(*rules.RuleSet) + set := v.(*teawaf.RuleSet) + + // 动作说明 + actionLinks := []maps.Map{} + if set.Action == teawaf.ActionGoGroup { + nextGroup := waf.FindRuleGroup(set.ActionOptions.GetString("groupId")) + if nextGroup != nil { + actionLinks = append(actionLinks, maps.Map{ + "name": nextGroup.Name, + "url": "/proxy/waf/group?wafId=" + waf.Id + "&groupId=" + nextGroup.Id, + }) + } + } else if set.Action == teawaf.ActionGoSet { + nextGroup := waf.FindRuleGroup(set.ActionOptions.GetString("groupId")) + if nextGroup != nil { + actionLinks = append(actionLinks, maps.Map{ + "name": nextGroup.Name, + "url": "/proxy/waf/group?wafId=" + waf.Id + "&groupId=" + nextGroup.Id, + }) + + nextSet := nextGroup.FindRuleSet(set.ActionOptions.GetString("setId")) + if nextSet != nil { + actionLinks = append(actionLinks, maps.Map{ + "name": nextSet.Name, + "url": "/proxy/waf/group/rule/update?wafId=" + waf.Id + "&groupId=" + nextGroup.Id + "&setId=" + nextSet.Id, + }) + } + } + } + return maps.Map{ "id": set.Id, "name": set.Name, "rules": lists.Map(set.Rules, func(k int, v interface{}) interface{} { - rule := v.(*rules.Rule) + rule := v.(*teawaf.Rule) return maps.Map{ "param": rule.Param, @@ -57,10 +84,12 @@ func (this *GroupAction) RunGet(params struct { "isCaseInsensitive": rule.IsCaseInsensitive, } }), - "on": set.On, - "action": strings.ToUpper(set.Action), - "actionName": wafactions.FindActionName(set.Action), - "connector": strings.ToUpper(set.Connector), + "on": set.On, + "action": strings.ToUpper(set.Action), + "actionOptions": set.ActionOptions, + "actionName": teawaf.FindActionName(set.Action), + "actionLinks": actionLinks, + "connector": strings.ToUpper(set.Connector), } }) diff --git a/teaweb/actions/default/proxy/waf/groupAdd.go b/teaweb/actions/default/proxy/waf/groupAdd.go index b6eceb9..f839d9e 100644 --- a/teaweb/actions/default/proxy/waf/groupAdd.go +++ b/teaweb/actions/default/proxy/waf/groupAdd.go @@ -2,7 +2,7 @@ package waf import ( "github.com/TeaWeb/code/teaconfigs" - "github.com/TeaWeb/code/teawaf/rules" + "github.com/TeaWeb/code/teawaf" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/utils/string" @@ -51,7 +51,7 @@ func (this *GroupAddAction) RunPost(params struct { Field("name", params.Name). Require("请输入分组名称") - group := rules.NewRuleGroup() + group := teawaf.NewRuleGroup() group.Id = stringutil.Rand(16) group.On = params.On group.Name = params.Name diff --git a/teaweb/actions/default/proxy/waf/ruleAdd.go b/teaweb/actions/default/proxy/waf/ruleAdd.go index 2acc6a1..35421e1 100644 --- a/teaweb/actions/default/proxy/waf/ruleAdd.go +++ b/teaweb/actions/default/proxy/waf/ruleAdd.go @@ -3,15 +3,15 @@ package waf import ( "encoding/json" "github.com/TeaWeb/code/teaconfigs" - wafactions "github.com/TeaWeb/code/teawaf/actions" + "github.com/TeaWeb/code/teawaf" "github.com/TeaWeb/code/teawaf/checkpoints" - "github.com/TeaWeb/code/teawaf/rules" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/waf/wafutils" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" + "strings" ) type RuleAddAction actions.Action @@ -29,6 +29,7 @@ func (this *RuleAddAction) RunGet(params struct { "name": waf.Name, "countInbound": waf.CountInboundRuleSets(), "countOutbound": waf.CountOutboundRuleSets(), + "inbound": waf.Inbound, } group := waf.FindRuleGroup(params.GroupId) @@ -43,12 +44,12 @@ func (this *RuleAddAction) RunGet(params struct { this.Data["connectors"] = []maps.Map{ { "name": "和(AND)", - "value": rules.RuleConnectorAnd, + "value": teawaf.RuleConnectorAnd, "description": "所有规则都满足才视为匹配", }, { "name": "或(OR)", - "value": rules.RuleConnectorOr, + "value": teawaf.RuleConnectorOr, "description": "任一规则满足了就视为匹配", }, } @@ -107,8 +108,8 @@ func (this *RuleAddAction) RunGet(params struct { this.Data["checkpoints"] = checkpointList - this.Data["operators"] = lists.Map(rules.AllRuleOperators, func(k int, v interface{}) interface{} { - def := v.(*rules.RuleOperatorDefinition) + this.Data["operators"] = lists.Map(teawaf.AllRuleOperators, func(k int, v interface{}) interface{} { + def := v.(*teawaf.RuleOperatorDefinition) return maps.Map{ "name": def.Name, "code": def.Code, @@ -117,8 +118,8 @@ func (this *RuleAddAction) RunGet(params struct { } }) - this.Data["actions"] = lists.Map(wafactions.AllActions, func(k int, v interface{}) interface{} { - def := v.(*wafactions.ActionDefinition) + this.Data["actions"] = lists.Map(teawaf.AllActions, func(k int, v interface{}) interface{} { + def := v.(*teawaf.ActionDefinition) return maps.Map{ "name": def.Name, "description": def.Description, @@ -154,11 +155,11 @@ func (this *RuleAddAction) RunPost(params struct { Must *actions.Must }) { - set := rules.NewRuleSet() + set := teawaf.NewRuleSet() set.Name = params.Name for index, prefix := range params.RulePrefixes { if index < len(params.RuleParams) && index < len(params.RuleOperators) && index < len(params.RuleValues) && index < len(params.RuleCases) && index < len(params.RuleOptions) { - rule := rules.NewRule() + rule := teawaf.NewRule() rule.Operator = params.RuleOperators[index] param := params.RuleParams[index] @@ -198,7 +199,19 @@ func (this *RuleAddAction) RunPost(params struct { } set.Connector = params.Connector + + // action set.Action = params.Action + set.ActionOptions = maps.Map{} + for k, v := range this.ParamsMap { + if len(v) == 0 { + continue + } + index := strings.Index(k, "action_") + if index> -1 { + set.ActionOptions[k[len("action_"):]] = v[0] + } + } // 测试 if params.Test { @@ -228,18 +241,18 @@ func (this *RuleAddAction) RunPost(params struct { if rule.Test(value) { matchLogs = append(matchLogs, "rule: "+rule.Param+" "+rule.Operator+" "+rule.Value+"\ncompare: "+value+"\nresult:true") - if set.Connector == rules.RuleConnectorOr { + if set.Connector == teawaf.RuleConnectorOr { matchedIndex = index break Loop } - if set.Connector == rules.RuleConnectorAnd { + if set.Connector == teawaf.RuleConnectorAnd { matchedIndex = index } } else { matchLogs = append(matchLogs, "rule: "+rule.Param+" "+rule.Operator+" "+rule.Value+"\ncompare: "+value+"\nresult:false") - if set.Connector == rules.RuleConnectorAnd { + if set.Connector == teawaf.RuleConnectorAnd { matchedIndex = -1 break Loop } diff --git a/teaweb/actions/default/proxy/waf/ruleUpdate.go b/teaweb/actions/default/proxy/waf/ruleUpdate.go index 19a5852..4705d26 100644 --- a/teaweb/actions/default/proxy/waf/ruleUpdate.go +++ b/teaweb/actions/default/proxy/waf/ruleUpdate.go @@ -3,9 +3,8 @@ package waf import ( "encoding/json" "github.com/TeaWeb/code/teaconfigs" - wafactions "github.com/TeaWeb/code/teawaf/actions" + "github.com/TeaWeb/code/teawaf" "github.com/TeaWeb/code/teawaf/checkpoints" - "github.com/TeaWeb/code/teawaf/rules" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/waf/wafutils" "github.com/iwind/TeaGo/actions" @@ -28,11 +27,13 @@ func (this *RuleUpdateAction) RunGet(params struct { if waf == nil { this.Fail("找不到WAF") } + this.Data["config"] = maps.Map{ "id": waf.Id, "name": waf.Name, "countInbound": waf.CountInboundRuleSets(), "countOutbound": waf.CountOutboundRuleSets(), + "inbound": waf.Inbound, } group := waf.FindRuleGroup(params.GroupId) @@ -55,7 +56,7 @@ func (this *RuleUpdateAction) RunGet(params struct { reg := regexp.MustCompile("^\\${([\\w.-]+)}$") this.Data["set"] = set this.Data["oldRules"] = lists.Map(set.Rules, func(k int, v interface{}) interface{} { - rule := v.(*rules.Rule) + rule := v.(*teawaf.Rule) prefix := "" param := "" @@ -83,12 +84,12 @@ func (this *RuleUpdateAction) RunGet(params struct { this.Data["connectors"] = []maps.Map{ { "name": "和 (AND)", - "value": rules.RuleConnectorAnd, + "value": teawaf.RuleConnectorAnd, "description": "所有规则都满足才视为匹配", }, { "name": "或 (OR)", - "value": rules.RuleConnectorOr, + "value": teawaf.RuleConnectorOr, "description": "任一规则满足了就视为匹配", }, } @@ -147,8 +148,8 @@ func (this *RuleUpdateAction) RunGet(params struct { this.Data["checkpoints"] = checkpointList - this.Data["operators"] = lists.Map(rules.AllRuleOperators, func(k int, v interface{}) interface{} { - def := v.(*rules.RuleOperatorDefinition) + this.Data["operators"] = lists.Map(teawaf.AllRuleOperators, func(k int, v interface{}) interface{} { + def := v.(*teawaf.RuleOperatorDefinition) return maps.Map{ "name": def.Name, "code": def.Code, @@ -157,8 +158,8 @@ func (this *RuleUpdateAction) RunGet(params struct { } }) - this.Data["actions"] = lists.Map(wafactions.AllActions, func(k int, v interface{}) interface{} { - def := v.(*wafactions.ActionDefinition) + this.Data["actions"] = lists.Map(teawaf.AllActions, func(k int, v interface{}) interface{} { + def := v.(*teawaf.ActionDefinition) return maps.Map{ "name": def.Name, "description": def.Description, @@ -213,10 +214,10 @@ func (this *RuleUpdateAction) RunPost(params struct { set.Name = params.Name set.ActionOptions = maps.Map{} - set.Rules = []*rules.Rule{} + set.Rules = []*teawaf.Rule{} for index, prefix := range params.RulePrefixes { if index < len(params.RuleParams) && index < len(params.RuleOperators) && index < len(params.RuleValues) && index < len(params.RuleCases) { - rule := rules.NewRule() + rule := teawaf.NewRule() rule.Operator = params.RuleOperators[index] param := params.RuleParams[index] @@ -255,7 +256,19 @@ func (this *RuleUpdateAction) RunPost(params struct { } } set.Connector = params.Connector + + // action set.Action = params.Action + set.ActionOptions = maps.Map{} + for k, v := range this.ParamsMap { + if len(v) == 0 { + continue + } + index := strings.Index(k, "action_") + if index> -1 { + set.ActionOptions[k[len("action_"):]] = v[0] + } + } // 测试 if params.Test { @@ -285,18 +298,18 @@ func (this *RuleUpdateAction) RunPost(params struct { if rule.Test(value) { matchLogs = append(matchLogs, "rule: "+rule.Param+" "+rule.Operator+" "+rule.Value+"\ncompare: "+value+"\nresult:true") - if set.Connector == rules.RuleConnectorOr { + if set.Connector == teawaf.RuleConnectorOr { matchedIndex = index break Loop } - if set.Connector == rules.RuleConnectorAnd { + if set.Connector == teawaf.RuleConnectorAnd { matchedIndex = index } } else { matchLogs = append(matchLogs, "rule: "+rule.Param+" "+rule.Operator+" "+rule.Value+"\ncompare: "+value+"\nresult:false") - if set.Connector == rules.RuleConnectorAnd { + if set.Connector == teawaf.RuleConnectorAnd { matchedIndex = -1 break Loop } diff --git a/teaweb/actions/default/proxy/waf/rules.go b/teaweb/actions/default/proxy/waf/rules.go index 888fcbf..71c198e 100644 --- a/teaweb/actions/default/proxy/waf/rules.go +++ b/teaweb/actions/default/proxy/waf/rules.go @@ -2,7 +2,7 @@ package waf import ( "github.com/TeaWeb/code/teaconfigs" - "github.com/TeaWeb/code/teawaf/rules" + "github.com/TeaWeb/code/teawaf" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/maps" @@ -31,7 +31,7 @@ func (this *RulesAction) RunGet(params struct { if params.Inbound { this.Data["groups"] = lists.Map(config.Inbound, func(k int, v interface{}) interface{} { - group := v.(*rules.RuleGroup) + group := v.(*teawaf.RuleGroup) return maps.Map{ "id": group.Id, "code": group.Code, @@ -44,7 +44,7 @@ func (this *RulesAction) RunGet(params struct { }) } else { this.Data["groups"] = lists.Map(config.Outbound, func(k int, v interface{}) interface{} { - group := v.(*rules.RuleGroup) + group := v.(*teawaf.RuleGroup) return maps.Map{ "id": group.Id, "code": group.Code, diff --git a/teaweb/actions/default/proxy/waf/test.go b/teaweb/actions/default/proxy/waf/test.go index 31ab02c..3632aca 100644 --- a/teaweb/actions/default/proxy/waf/test.go +++ b/teaweb/actions/default/proxy/waf/test.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teautils" + "github.com/TeaWeb/code/teawaf" "github.com/TeaWeb/code/teawaf/checkpoints" - "github.com/TeaWeb/code/teawaf/rules" "github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/maps" @@ -139,7 +139,7 @@ func (this *TestAction) RunPost(params struct { setName := "" action := "" - groups := []*rules.RuleGroup{} + groups := []*teawaf.RuleGroup{} if params.Inbound { groups = waf.Inbound } else { @@ -165,7 +165,7 @@ Loop: } found := false - if set.Connector == rules.RuleConnectorAnd { + if set.Connector == teawaf.RuleConnectorAnd { found = true } for _, rule := range set.Rules { @@ -174,11 +174,11 @@ Loop: return types.String(v) }) if rule.Test(value) { - if set.Connector == rules.RuleConnectorOr { + if set.Connector == teawaf.RuleConnectorOr { found = true } } else { - if set.Connector == rules.RuleConnectorAnd { + if set.Connector == teawaf.RuleConnectorAnd { found = false } } diff --git a/teaweb/actions/default/proxy/waf/update.go b/teaweb/actions/default/proxy/waf/update.go index 5dc09f7..d462036 100644 --- a/teaweb/actions/default/proxy/waf/update.go +++ b/teaweb/actions/default/proxy/waf/update.go @@ -4,8 +4,6 @@ import ( "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teaconfigs/shared" "github.com/TeaWeb/code/teawaf" - actions2 "github.com/TeaWeb/code/teawaf/actions" - "github.com/TeaWeb/code/teawaf/rules" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/waf/wafutils" "github.com/iwind/TeaGo/Tea" @@ -30,7 +28,7 @@ func (this *UpdateAction) RunGet(params struct { } if waf.ActionBlock == nil { - waf.ActionBlock = &actions2.BlockAction{ + waf.ActionBlock = &teawaf.BlockAction{ StatusCode: http.StatusForbidden, } } @@ -46,7 +44,7 @@ func (this *UpdateAction) RunGet(params struct { } this.Data["groups"] = lists.Map(teawaf.Template().Inbound, func(k int, v interface{}) interface{} { - g := v.(*rules.RuleGroup) + g := v.(*teawaf.RuleGroup) group := waf.FindRuleGroupWithCode(g.Code) return maps.Map{ @@ -93,7 +91,7 @@ func (this *UpdateAction) RunPost(params struct { waf.Name = params.Name waf.On = params.On - waf.ActionBlock = &actions2.BlockAction{ + waf.ActionBlock = &teawaf.BlockAction{ StatusCode: statusCode, Body: params.BlockBody, URL: params.BlockURL, From 3b04442cec6fc4a11b4cbb2ab1344f603f3ae44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月30日 20:40:24 +0800 Subject: [PATCH 086/121] =?UTF-8?q?[proxy]=E5=86=99=E5=85=A5=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E6=97=B6=E5=88=A4=E6=96=ADContent-Length=E5=92=8CBody?= =?UTF-8?q?=E6=98=AF=E5=90=A6=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacache/process.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/teacache/process.go b/teacache/process.go index b09a32a..63c47a1 100644 --- a/teacache/process.go +++ b/teacache/process.go @@ -8,6 +8,7 @@ import ( "github.com/TeaWeb/code/teaweb/configs" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/types" "io" "net/http" "strings" @@ -179,6 +180,12 @@ func ProcessAfterRequest(req *teaproxy.Request, writer *teaproxy.ResponseWriter) return true } + // check length + contentLength := types.Int(writer.Header().Get("Content-Length")) + if contentLength != len(writer.Body()) { + return true + } + // validate cache control if len(cacheConfig.SkipResponseCacheControlValues)> 0 { cacheControl := writer.Header().Get("Cache-Control") From 0130f810053a04c5530030c669adb8248d7cee4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月31日 18:59:23 +0800 Subject: [PATCH 087/121] =?UTF-8?q?[proxy]=E5=B0=86=E8=AE=BF=E9=97=AE?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=9A=84=E9=83=A8=E5=88=86=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E6=9C=80=E5=A4=A7=E9=95=BF=E5=BA=A6=E4=BB=8E1024=E6=94=B9?= =?UTF-8?q?=E6=88=902048?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teadb/dao_acess_log_sql.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/teadb/dao_acess_log_sql.go b/teadb/dao_acess_log_sql.go index 8e7971b..95501e8 100644 --- a/teadb/dao_acess_log_sql.go +++ b/teadb/dao_acess_log_sql.go @@ -387,8 +387,8 @@ func (this *SQLAccessLogDAO) initTable(table string) { "`remoteAddr` varchar(64) DEFAULT NULL,"+ "`remotePort` int(11) unsigned DEFAULT '0',"+ "`remoteUser` varchar(128) DEFAULT NULL,"+ - "`requestURI` varchar(1024) DEFAULT NULL,"+ - "`requestPath` varchar(1024) DEFAULT NULL,"+ + "`requestURI` varchar(2048) DEFAULT NULL,"+ + "`requestPath` varchar(2048) DEFAULT NULL,"+ "`requestLength` bigint(20) unsigned DEFAULT '0',"+ "`requestTime` decimal(20,6) unsigned DEFAULT '0.000000',"+ "`requestMethod` varchar(16) DEFAULT NULL,"+ @@ -405,9 +405,9 @@ func (this *SQLAccessLogDAO) initTable(table string) { "`msec` decimal(20,6) unsigned DEFAULT '0.000000',"+ "`timestamp` int(11) unsigned DEFAULT '0',"+ "`host` varchar(128) DEFAULT NULL,"+ - "`referer` varchar(1024) DEFAULT NULL,"+ + "`referer` varchar(2048) DEFAULT NULL,"+ "`userAgent` varchar(1024) DEFAULT NULL,"+ - "`request` varchar(1024) DEFAULT NULL,"+ + "`request` varchar(2048) DEFAULT NULL,"+ "`contentType` varchar(256) DEFAULT NULL,"+ "`cookie` json DEFAULT NULL,"+ "`arg` json DEFAULT NULL,"+ @@ -450,8 +450,8 @@ func (this *SQLAccessLogDAO) initTable(table string) { "remoteAddr" varchar(64), "remotePort" int4 default 0, "remoteUser" varchar(128), - "requestURI" varchar(1024), - "requestPath" varchar(1024), + "requestURI" varchar(2048), + "requestPath" varchar(2048), "requestLength" int8 default 0, "requestTime" float8 default 0, "requestMethod" varchar(16), @@ -468,9 +468,9 @@ func (this *SQLAccessLogDAO) initTable(table string) { "msec" float8 default 0, "timestamp" int4 default 0, "host" varchar(128), - "referer" varchar(1024), + "referer" varchar(2048), "userAgent" varchar(1024), - "request" varchar(1024), + "request" varchar(2048), "contentType" varchar(256), "cookie" json, "arg" json, From 3a8ee3dd28f1bf06aa958950d04820eff8651491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月31日 20:37:18 +0800 Subject: [PATCH 088/121] =?UTF-8?q?[waf]=E5=AF=BC=E5=85=A5=E8=A7=84?= =?UTF-8?q?=E5=88=99=E9=9B=86=E6=96=87=E4=BB=B6=E6=97=B6=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=98=AF=E5=90=A6=E8=A6=86=E7=9B=96=E5=90=8C=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E5=92=8C=E5=90=8C=E5=90=8D=E9=80=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaweb/actions/default/proxy/waf/import.go | 53 ++++++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/teaweb/actions/default/proxy/waf/import.go b/teaweb/actions/default/proxy/waf/import.go index ba27957..0080361 100644 --- a/teaweb/actions/default/proxy/waf/import.go +++ b/teaweb/actions/default/proxy/waf/import.go @@ -38,10 +38,13 @@ func (this *ImportAction) RunPost(params struct { File *actions.File // step2 - WafId string - GroupIds []string - Data string - Step string + WafId string + GroupIds []string + OverwriteType bool + OverwriteName bool + + Data string + Step string Must *actions.Must }) { @@ -104,7 +107,29 @@ func (this *ImportAction) RunPost(params struct { if group == nil { continue } - countGroups ++ + + // 删除老的分组 + if params.OverwriteType && len(group.Code)> 0 { + oldGroup := currentWAF.FindRuleGroupWithCode(group.Code) + if oldGroup != nil { + currentWAF.RemoveRuleGroup(oldGroup.Id) + } + } else if params.OverwriteName && len(group.Name)> 0 { + if group.IsInbound { + oldGroup := this.findInboundGroupWithName(currentWAF, group.Name) + if oldGroup != nil { + currentWAF.RemoveRuleGroup(oldGroup.Id) + } + } else { + oldGroup := this.findOutboundGroupWithName(currentWAF, group.Name) + if oldGroup != nil { + currentWAF.RemoveRuleGroup(oldGroup.Id) + } + } + } + + // 添加新的分组 + countGroups++ countSets += len(group.RuleSets) group.Id = stringutil.Rand(16) // 重新生成ID,避免和现有的ID冲突 currentWAF.AddRuleGroup(group) @@ -126,3 +151,21 @@ func (this *ImportAction) RunPost(params struct { this.Success() } } + +func (this *ImportAction) findInboundGroupWithName(waf *teawaf.WAF, name string) *teawaf.RuleGroup { + for _, g := range waf.Inbound { + if g.Name == name { + return g + } + } + return nil +} + +func (this *ImportAction) findOutboundGroupWithName(waf *teawaf.WAF, name string) *teawaf.RuleGroup { + for _, g := range waf.Outbound { + if g.Name == name { + return g + } + } + return nil +} From ddad757702f6839f402d5f794c9f1d5795775d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年3月31日 20:38:08 +0800 Subject: [PATCH 089/121] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconst/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/teaconst/const.go b/teaconst/const.go index 879a84c..a2258af 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.10" + TeaVersion = "0.1.11" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 From 381fec62b6f46355f532f42d7f7261c4fd788725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Wed, 1 Apr 2020 18:01:14 +0800 Subject: [PATCH 090/121] =?UTF-8?q?[proxy]=E7=BC=93=E5=AD=98=E7=AD=96?= =?UTF-8?q?=E7=95=A5=E5=AE=9E=E7=8E=B0=E6=89=B9=E9=87=8F=E5=88=B7=E6=96=B0?= =?UTF-8?q?Key=E5=89=8D=E7=BC=80=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacache/file.go | 189 ++++++++++++++++-- teacache/file_test.go | 46 +++++ teacache/leveldb.go | 32 +++ teacache/leveldb_test.go | 19 ++ teacache/manager.go | 3 + teacache/memory.go | 36 +++- teacache/memory_test.go | 24 +++ teacache/redis.go | 140 +++++++++---- teacache/redis_test.go | 30 ++- teaconfigs/shared/ip_range.go | 2 +- teamemory/cell.go | 11 +- teamemory/grid.go | 5 + teatesting/require.go | 10 +- teawaf/ip_table.go | 153 ++++++++++++++ teawaf/ip_table_test.go | 142 +++++++++++++ teawaf/waf.go | 2 + .../default/actionutils/ParentAction.go | 26 +++ teaweb/actions/default/cache/cleanPolicy.go | 12 +- teaweb/actions/default/cache/init.go | 1 + teaweb/actions/default/cache/policy.go | 8 +- teaweb/actions/default/cache/refreshPolicy.go | 77 +++++++ teaweb/actions/default/cache/statPolicy.go | 8 +- teaweb/actions/default/cache/testPolicy.go | 8 +- teaweb/actions/default/cache/updatePolicy.go | 7 +- 24 files changed, 918 insertions(+), 73 deletions(-) create mode 100644 teawaf/ip_table.go create mode 100644 teawaf/ip_table_test.go create mode 100644 teaweb/actions/default/actionutils/ParentAction.go create mode 100644 teaweb/actions/default/cache/refreshPolicy.go diff --git a/teacache/file.go b/teacache/file.go index 07000b7..de8f0d7 100644 --- a/teacache/file.go +++ b/teacache/file.go @@ -1,22 +1,28 @@ package teacache import ( + "bufio" "errors" "fmt" + "github.com/TeaWeb/code/teautils" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/timers" "github.com/iwind/TeaGo/types" "github.com/iwind/TeaGo/utils/string" + "io" "os" "path/filepath" "regexp" + "strconv" "strings" "sync" "time" ) +var bytePool = teautils.NewBytePool(40960, 2048) + // 文件缓存管理器 type FileManager struct { Manager @@ -55,13 +61,13 @@ func NewFileManager() *FileManager { logs.Error(err) continue } - data := reader.Read(12) - if len(data) != 12 { - reader.Close() + data := reader.Read(10) + if len(data) != 10 { + _ = reader.Close() continue } timestamp := types.Int64(string(data)) - reader.Close() + _ = reader.Close() if timestamp < time.Now().Unix()-100 { // 超时100秒以上的 err := file.Delete() if err != nil { @@ -86,10 +92,14 @@ func (this *FileManager) SetOptions(options map[string]interface{}) { dir, found := options["dir"] if found { this.dir = types.String(dir) + if !filepath.IsAbs(this.dir) { + this.dir = Tea.Root + Tea.DS + this.dir + } } } // 写入 +// 内容格式 timestamp | key length (20 bytes) | key | data (n bytes) | func (this *FileManager) Write(key string, data []byte) error { if len(this.dir) == 0 { return errors.New("cache dir should not be empty") @@ -112,7 +122,18 @@ func (this *FileManager) Write(key string, data []byte) error { } } - newFile := files.NewFile(newDir.Path() + Tea.DS + md5 + ".cache") + path := newDir.Path() + Tea.DS + md5 + ".cache" + fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_SYNC, 0666) + if err != nil { + return err + } + defer func() { + _ = fp.Close() + + if err != nil { + _ = os.Remove(path) + } + }() // 头部加入有效期 var life = int64(this.Life.Seconds()) @@ -121,8 +142,37 @@ func (this *FileManager) Write(key string, data []byte) error { } else if life>= 365*86400 { // 最大值限制 life = 365 * 86400 } - data = append([]byte(fmt.Sprintf("%012d", time.Now().Unix()+life)), data ...) - err := newFile.Write(data) + _, err = fp.WriteString(fmt.Sprintf("%d", time.Now().Unix()+life)) + if err != nil { + return err + } + + _, err = fp.WriteString("|") + if err != nil { + return err + } + + _, err = fp.WriteString(strconv.Itoa(len(key))) + if err != nil { + return err + } + + _, err = fp.WriteString("|") + if err != nil { + return err + } + + _, err = fp.WriteString(key) + if err != nil { + return err + } + + _, err = fp.WriteString("|") + if err != nil { + return err + } + + _, err = fp.Write(data) return err } @@ -130,23 +180,78 @@ func (this *FileManager) Write(key string, data []byte) error { // 读取 func (this *FileManager) Read(key string) (data []byte, err error) { md5 := stringutil.Md5(key) - file := files.NewFile(this.dir + Tea.DS + md5[:2] + Tea.DS + md5[2:4] + Tea.DS + md5 + ".cache") + fp, err := os.Open(this.dir + Tea.DS + md5[:2] + Tea.DS + md5[2:4] + Tea.DS + md5 + ".cache") + if err != nil { + return nil, ErrNotFound + } + defer func() { + _ = fp.Close() + }() this.writeLocker.RLock() defer this.writeLocker.RUnlock() - if !file.Exists() { + r := bufio.NewReader(fp) + key, ok := this.readHead(r) + if !ok { return nil, ErrNotFound } - data, err = file.ReadAll() - if err != nil || len(data) < 12 { // 12是时间戳长度 - return nil, err + + buf := bytePool.Get() + defer bytePool.Put(buf) + for { + n, err := r.Read(buf) + if n> 0 { + data = append(data, buf[:n]...) + } + if err != nil { + if err == io.EOF { + break + } + return nil, ErrNotFound + } } - timestamp := types.Int64(string(data[:12])) - if timestamp < time.Now().Unix() { - return nil, ErrNotFound + return data, nil +} + +func (this *FileManager) readHead(r *bufio.Reader) (key string, ok bool) { + timestampBytes, err := r.ReadBytes('|') + if err != nil { + return + } + if len(timestampBytes)> 12 { + return + } + if types.Int64(string(timestampBytes[:len(timestampBytes)-1])) < time.Now().Unix() { + return + } + + keyLengthBytes, err := r.ReadBytes('|') + if err != nil { + return + } + + keyLength := types.Int(string(keyLengthBytes[:len(keyLengthBytes)-1])) + if keyLength> 0 { + keyBytes := []byte{} + for i := 0; i < keyLength; i++ { + b, err := r.ReadByte() + if err != nil { + return + } + keyBytes = append(keyBytes, b) + } + + key = string(keyBytes) + } + _, err = r.ReadByte() + if err != nil { + return } - return data[12:], nil + + ok = true + + return } // 删除 @@ -176,6 +281,52 @@ func (this *FileManager) Delete(key string) error { return newFile.Delete() } +// 删除key前缀 +func (this *FileManager) DeletePrefixes(prefixes []string) (int, error) { + if len(prefixes) == 0 { + return 0, nil + } + // 检查目录是否存在 + info, err := os.Stat(this.dir) + if err != nil || !info.IsDir() { + return 0, nil + } + + count := 0 + err = filepath.Walk(this.dir, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + if !strings.HasSuffix(info.Name(), ".cache") { + return nil + } + + fp, err := os.Open(path) + if err != nil { + return err + } + defer func() { + _ = fp.Close() + }() + + key, ok := this.readHead(bufio.NewReader(fp)) + if !ok { + return nil + } + for _, prefix := range prefixes { + if strings.HasPrefix(key, prefix) || strings.HasPrefix("http://"+key, prefix) || strings.HasPrefix("https://"+key, prefix) { + _ = fp.Close() + count++ + return os.Remove(path) + } + } + + return nil + }) + + return count, err +} + // 统计 func (this *FileManager) Stat() (size int64, countKeys int, err error) { // 检查目录是否存在 @@ -184,7 +335,7 @@ func (this *FileManager) Stat() (size int64, countKeys int, err error) { return } - filepath.Walk(this.dir, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(this.dir, func(path string, info os.FileInfo, err error) error { if info.IsDir() { return nil } @@ -197,6 +348,10 @@ func (this *FileManager) Stat() (size int64, countKeys int, err error) { return nil }) + if err != nil { + logs.Error(err) + } + return } diff --git a/teacache/file_test.go b/teacache/file_test.go index b5c5a58..41f1fb9 100644 --- a/teacache/file_test.go +++ b/teacache/file_test.go @@ -1,9 +1,11 @@ package teacache import ( + "fmt" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/logs" + "runtime" "sync" "testing" ) @@ -12,6 +14,8 @@ func TestFileManager(t *testing.T) { a := assert.NewAssertion(t) m := NewFileManager() + m.dir = Tea.Root + "/cache" + err := m.Write("123456", []byte("abc")) t.Log(err) a.IsNotNil(err) @@ -73,3 +77,45 @@ func TestFileManager_Clean(t *testing.T) { m.dir = Tea.Root + "/./cache" t.Log(m.Clean()) } + +func TestFileManager_DeletePrefixes(t *testing.T) { + m := NewFileManager() + m.dir = Tea.Root + "/cache/" + + canWrite := false + if canWrite { + for i := 0; i < 3; i++ { + err := m.Write("abc"+fmt.Sprintf("%03d", i), []byte("I AM DATA")) + if err != nil { + t.Fatal(err) + } + } + _ = m.Write("bcd000", []byte("I AM BCD")) + + data, err := m.Read("abc000") + if err != nil { + t.Fatal(err) + } + t.Log("read:["+string(data)+"]", len(data), "bytes") + } + + count, err := m.DeletePrefixes([]string{"abc"}) + if err != nil { + t.Fatal(err) + } + t.Log("OK", count, "keys") +} + +func BenchmarkFileManager_Read(b *testing.B) { + runtime.GOMAXPROCS(1) + + m := NewFileManager() + m.dir = Tea.Root + "/cache/" + + for i := 0; i < b.N; i++ { + data, _ := m.Read("abc000") + if len(data) == 0 { + b.Fatal("invalid data:", string(data)) + } + } +} diff --git a/teacache/leveldb.go b/teacache/leveldb.go index b315cc8..a54fd01 100644 --- a/teacache/leveldb.go +++ b/teacache/leveldb.go @@ -12,6 +12,7 @@ import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" "path/filepath" + "strings" "time" ) @@ -109,6 +110,37 @@ func (this *LevelDBManager) Delete(key string) error { return this.db.Delete(append([]byte("KEY"), []byte(key)...), nil) } +// 删除key前缀 +func (this *LevelDBManager) DeletePrefixes(prefixes []string) (int, error) { + if len(prefixes) == 0 { + return 0, nil + } + if this.db == nil { + return 0, nil + } + + it := this.db.NewIterator(util.BytesPrefix([]byte("KEY")), nil) + defer it.Release() + + count := 0 + for it.Next() { + keyBytes := it.Key() + key := string(keyBytes[3:]) // skip "KEY" + for _, prefix := range prefixes { + if strings.HasPrefix(key, prefix) || strings.HasPrefix("http://"+key, prefix) || strings.HasPrefix("https://"+key, prefix) { + err := this.db.Delete(keyBytes, nil) + if err != nil { + return count, err + } + count++ + break + } + } + } + + return count, nil +} + func (this *LevelDBManager) CleanExpired() error { if this.db == nil { return nil diff --git a/teacache/leveldb_test.go b/teacache/leveldb_test.go index 648f970..cd6822d 100644 --- a/teacache/leveldb_test.go +++ b/teacache/leveldb_test.go @@ -64,3 +64,22 @@ func TestLevelDBManager_Clean(t *testing.T) { } _ = m.Close() } + +func TestLevelDBManager_DeletePrefixes(t *testing.T) { + m := NewLevelDBManager() + m.SetOptions(map[string]interface{}{ + "dir": Tea.Root + "/cache", + }) + canWrite := false + if canWrite { + err := m.Write("abc000", []byte("Hello")) + if err != nil { + t.Fatal(err) + } + } + count, err := m.DeletePrefixes([]string{"abc"}) + if err != nil { + t.Fatal(err) + } + t.Log(count, "keys deleted") +} diff --git a/teacache/manager.go b/teacache/manager.go index dd3a18c..e42838f 100644 --- a/teacache/manager.go +++ b/teacache/manager.go @@ -24,6 +24,9 @@ type ManagerInterface interface { // 删除 Delete(key string) error + // 删除key前缀 + DeletePrefixes(prefixes []string) (int, error) + // 设置选项 SetOptions(options map[string]interface{}) diff --git a/teacache/memory.go b/teacache/memory.go index f199e63..65223ef 100644 --- a/teacache/memory.go +++ b/teacache/memory.go @@ -3,6 +3,7 @@ package teacache import ( "errors" "github.com/TeaWeb/code/teamemory" + "strings" "time" ) @@ -34,7 +35,7 @@ func (this *MemoryManager) SetOptions(options map[string]interface{}) { opts = append(opts, teamemory.NewLimitSizeOpt(capacityBytes)) countCells = int(capacityBytes / 1024 / 1024 / 128) } - this.grid = teamemory.NewGrid(countCells, opts ...) + this.grid = teamemory.NewGrid(countCells, opts...) } func (this *MemoryManager) Write(key string, data []byte) error { @@ -63,6 +64,39 @@ func (this *MemoryManager) Delete(key string) error { return nil } +// 删除key前缀 +func (this *MemoryManager) DeletePrefixes(prefixes []string) (int, error) { + if len(prefixes) == 0 { + return 0, nil + } + + grid := this.grid + if grid == nil { + return 0, nil + } + + count := 0 + keys := [][]byte{} + for _, cell := range grid.Cells() { + cell.Range(func(item *teamemory.Item) { + key := string(item.Key) + for _, prefix := range prefixes { + if strings.HasPrefix(key, prefix) || strings.HasPrefix("http://"+key, prefix) || strings.HasPrefix("https://"+key, prefix) { + keys = append(keys, item.Key) + count++ + break + } + } + }) + } + + for _, key := range keys { + grid.Delete(key) + } + + return count, nil +} + // 统计 func (this *MemoryManager) Stat() (size int64, countKeys int, err error) { stat := this.grid.Stat() diff --git a/teacache/memory_test.go b/teacache/memory_test.go index 65a1200..4d0caef 100644 --- a/teacache/memory_test.go +++ b/teacache/memory_test.go @@ -1,6 +1,7 @@ package teacache import ( + "fmt" "github.com/iwind/TeaGo/assert" "testing" "time" @@ -33,3 +34,26 @@ func TestCacheMemoryConfig(t *testing.T) { t.Log(string(data)) } + +func TestMemoryManager_DeletePrefixes(t *testing.T) { + m := NewMemoryManager() + m.Capacity = 1024 * 128 + m.Life = 30 * time.Second + m.SetOptions(nil) + + for i := 0; i < 100; i++ { + err := m.Write("abc"+fmt.Sprintf("%03d", i), []byte("1")) + if err != nil { + t.Fatal(err) + } + } + _ = m.Write("bcd", []byte("1")) + + t.Log(m.Stat()) + count, err := m.DeletePrefixes([]string{"http://abc"}) + if err != nil { + t.Fatal(err) + } + t.Log("deleted", count, "keys") + t.Log(m.Stat()) +} diff --git a/teacache/redis.go b/teacache/redis.go index bc97105..3816ce2 100644 --- a/teacache/redis.go +++ b/teacache/redis.go @@ -5,6 +5,7 @@ import ( "github.com/go-redis/redis" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" + "strings" "time" ) @@ -94,55 +95,118 @@ func (this *RedisManager) Delete(key string) error { return cmd.Err() } -// 统计 -func (this *RedisManager) Stat() (size int64, countKeys int, err error) { - scan := this.client.Scan(0, "TEA_CACHE_"+this.Id()+"*", 100000) - if scan == nil { - return +// 删除key前缀 +func (this *RedisManager) DeletePrefixes(prefixes []string) (int, error) { + if len(prefixes) == 0 { + return 0, nil } - if scan.Err() != nil { - err = scan.Err() - return - } - it := scan.Iterator() - if it.Err() != nil { - err = it.Err() - return + + cursor := uint64(0) + var err error + loopCount := 0 + count := 0 + keyPrefix := "TEA_CACHE_" + this.Id() + keyPrefixLength := len(keyPrefix) + for { + loopCount++ + + var keys []string + keys, cursor, err = this.client.Scan(cursor, keyPrefix+"*", 10000).Result() + if err != nil { + return count, err + } + if len(keys)> 0 { + for _, key := range keys { + realKey := key[keyPrefixLength:] + for _, prefix := range prefixes { + if strings.HasPrefix(realKey, prefix) || strings.HasPrefix("http://"+realKey, prefix) || strings.HasPrefix("https://"+realKey, prefix) { + err1 := this.client.Del(key).Err() + if err1 != nil { + err = err1 + break + } + count++ + break + } + } + } + } + + // 防止单个操作时间过长 + if loopCount> 10000 { + break + } + + if cursor == 0 { + break + } } - for it.Next() { - key := it.Val() - b, err := this.client.Get(key).Bytes() + + return count, nil +} + +// 统计 +func (this *RedisManager) Stat() (size int64, countKeys int, err error) { + cursor := uint64(0) + loopCount := 0 + for { + loopCount++ + + var keys []string + keys, cursor, err = this.client.Scan(cursor, "TEA_CACHE_"+this.Id()+"*", 10000).Result() if err != nil { - continue + return + } + if len(keys)> 0 { + countKeys += len(keys) + for _, key := range keys { + val, _ := this.client.Get(key).Bytes() + size += int64(len(val)) + } + } + + // 防止单个操作时间过长 + if loopCount> 10000 { + break + } + + if cursor == 0 { + break } - countKeys++ - size += int64(len(b) + len(key)) } return } // 清理 func (this *RedisManager) Clean() error { - scan := this.client.Scan(0, "TEA_CACHE_"+this.Id()+"*", 100000) - if scan == nil { - return nil - } - if scan.Err() != nil { - return scan.Err() - } - it := scan.Iterator() - if it.Err() != nil { - return it.Err() - } - keys := []string{} - for it.Next() { - key := it.Val() - keys = append(keys, key) - } + cursor := uint64(0) + var err error + loopCount := 0 + for { + loopCount++ + + var keys []string + keys, cursor, err = this.client.Scan(cursor, "TEA_CACHE_"+this.Id()+"*", 10000).Result() + if err != nil { + return err + } + if len(keys)> 0 { + for _, key := range keys { + err1 := this.client.Del(key).Err() + if err1 != nil { + err = err1 + break + } + } + } + + // 防止单个操作时间过长 + if loopCount> 10000 { + break + } - if len(keys)> 0 { - for _, key := range keys { - this.client.Del(key) + if cursor == 0 { + break } } diff --git a/teacache/redis_test.go b/teacache/redis_test.go index a4ca6a8..f5dea0c 100644 --- a/teacache/redis_test.go +++ b/teacache/redis_test.go @@ -8,11 +8,12 @@ import ( func TestRedisManager(t *testing.T) { if !teatesting.RequireRedis() { + t.Log("skip for redis missing") return } manager := NewRedisManager() - manager.Life = 30 * time.Second + manager.Life = 3600 * time.Second manager.SetOptions(map[string]interface{}{ "host": "127.0.0.1", }) @@ -28,6 +29,7 @@ func TestRedisManager(t *testing.T) { func TestRedisManager_Stat(t *testing.T) { if !teatesting.RequireRedis() { + t.Log("skip for redis missing") return } @@ -40,13 +42,14 @@ func TestRedisManager_Stat(t *testing.T) { "port": "6379", }) t.Log(manager.Write("key1", []byte("value1"))) - t.Log(manager.Write("key2", []byte("value1"))) - t.Log(manager.Write("key3", []byte("value1"))) + t.Log(manager.Write("key2", []byte("value2"))) + t.Log(manager.Write("key3", []byte("value3"))) t.Log(manager.Stat()) } func TestRedisManager_Clean(t *testing.T) { if !teatesting.RequireRedis() { + t.Log("skip for redis missing") return } @@ -60,3 +63,24 @@ func TestRedisManager_Clean(t *testing.T) { }) t.Log(manager.Clean()) } + +func TestRedisManager_DeletePrefixes(t *testing.T) { + if !teatesting.RequireRedis() { + t.Log("skip for redis missing") + return + } + + manager := NewRedisManager() + manager.SetId("abc") + manager.Life = 1800 * time.Second + manager.SetOptions(map[string]interface{}{ + "network": "tcp", + "host": "127.0.0.1", + "port": "6379", + }) + t.Log(manager.Write("key1", []byte("value1"))) + t.Log(manager.Write("key2", []byte("value2"))) + t.Log(manager.Write("key3", []byte("value3"))) + t.Log(manager.DeletePrefixes([]string{"http://key"})) + t.Log(manager.Stat()) +} diff --git a/teaconfigs/shared/ip_range.go b/teaconfigs/shared/ip_range.go index 53c2623..5484594 100644 --- a/teaconfigs/shared/ip_range.go +++ b/teaconfigs/shared/ip_range.go @@ -51,7 +51,7 @@ func ParseIPRange(s string) (*IPRangeConfig, error) { ipRange := &IPRangeConfig{} - if s == "all" || s == "ALL" || s == "0.0.0.0" { + if s == "*" || s == "all" || s == "ALL" || s == "0.0.0.0" { ipRange.Type = IPRangeTypeAll return ipRange, nil } diff --git a/teamemory/cell.go b/teamemory/cell.go index d1f4cdb..536e9e2 100644 --- a/teamemory/cell.go +++ b/teamemory/cell.go @@ -123,7 +123,7 @@ func (this *Cell) Trim() { inactiveSize := int(math.Ceil(float64(l) / 10)) // trim 10% items this.list.Range(func(item *Item) (goNext bool) { - inactiveSize -- + inactiveSize-- delete(this.mapping, item.HashKey()) this.list.Remove(item) this.totalBytes -= item.Size() @@ -142,6 +142,15 @@ func (this *Cell) Delete(hashKey uint64) { this.locker.Unlock() } +// range all items in the cell +func (this *Cell) Range(f func(item *Item)) { + this.locker.Lock() + for _, item := range this.mapping { + f(item) + } + this.locker.Unlock() +} + func (this *Cell) Recycle() { this.locker.Lock() timestamp := time.Now().Unix() diff --git a/teamemory/grid.go b/teamemory/grid.go index 2639860..cd8eef7 100644 --- a/teamemory/grid.go +++ b/teamemory/grid.go @@ -61,6 +61,11 @@ func NewGrid(countCells int, opt ...interface{}) *Grid { return grid } +// get all cells in the grid +func (this *Grid) Cells() []*Cell { + return this.cells +} + func (this *Grid) WriteItem(item *Item) { if this.countCells <= 0 { return diff --git a/teatesting/require.go b/teatesting/require.go index 03c4e25..216b0b1 100644 --- a/teatesting/require.go +++ b/teatesting/require.go @@ -2,6 +2,7 @@ package teatesting import ( "github.com/TeaWeb/code/teautils" + "github.com/go-redis/redis" "github.com/iwind/TeaGo/logs" "net/http" "time" @@ -95,8 +96,13 @@ func RequireDNS() bool { // 需要Redis支持 func RequireRedis() bool { - // TODO - return false + client := redis.NewClient(&redis.Options{ + Network: "tcp", + Addr: "127.0.0.1:6379", + DialTimeout: 5 * time.Second, + }) + cmd := client.Ping() + return cmd.Err() == nil } // 需要ES支持 diff --git a/teawaf/ip_table.go b/teawaf/ip_table.go new file mode 100644 index 0000000..4a64994 --- /dev/null +++ b/teawaf/ip_table.go @@ -0,0 +1,153 @@ +package teawaf + +import ( + "github.com/TeaWeb/code/teaconfigs/shared" + "github.com/TeaWeb/code/teautils" + "github.com/iwind/TeaGo/lists" + "github.com/iwind/TeaGo/types" + stringutil "github.com/iwind/TeaGo/utils/string" + "regexp" + "strings" + "time" +) + +type IPAction = string + +const ( + IPActionAccept IPAction = "accept" + IPActionReject IPAction = "reject" +) + +// ip table +type IPTable struct { + Id string `yaml:"id" json:"id"` + On bool `yaml:"on" json:"on"` + IP string `yaml:"ip" json:"ip"` // single ip, cidr, ip range, TODO support * + Port string `yaml:"port" json:"port"` // single port, range, * + Action IPAction `yaml:"action" json:"action"` // accept, reject + TimeFrom int64 `yaml:"timeFrom" json:"timeFrom"` // from timestamp + TimeTo int64 `yaml:"timeTo" json:"timeTo"` // zero means forever + Remark string `yaml:"remark" json:"remark"` + + // port + minPort int + maxPort int + + minPortWildcard bool + maxPortWildcard bool + + ports []int + + // ip + ipRange *shared.IPRangeConfig +} + +func NewIPTable() *IPTable { + return &IPTable{ + On: true, + Id: stringutil.Rand(16), + } +} + +func (this *IPTable) Init() error { + // parse port + if teautils.RegexpDigitNumber.MatchString(this.Port) { + this.minPort = types.Int(this.Port) + this.maxPort = types.Int(this.Port) + } else if regexp.MustCompile(`[:-]`).MatchString(this.Port) { + pieces := regexp.MustCompile(`[:-]`).Split(this.Port, 2) + if pieces[0] == "*" { + this.minPortWildcard = true + } else { + this.minPort = types.Int(pieces[0]) + } + if pieces[1] == "*" { + this.maxPortWildcard = true + } else { + this.maxPort = types.Int(pieces[1]) + } + } else if strings.Contains(this.Port, ",") { + pieces := strings.Split(this.Port, ",") + for _, piece := range pieces { + piece = strings.TrimSpace(piece) + if len(piece)> 0 { + this.ports = append(this.ports, types.Int(piece)) + } + } + } else if this.Port == "*" { + this.minPortWildcard = true + this.maxPortWildcard = true + } + + // parse ip + if len(this.IP)> 0 { + ipRange, err := shared.ParseIPRange(this.IP) + if err != nil { + return err + } + this.ipRange = ipRange + } + + return nil +} + +// check ip +func (this *IPTable) Match(ip string, port int) (isMatched bool) { + if !this.On { + return + } + + now := time.Now().Unix() + if this.TimeFrom> 0 && now < this.TimeFrom { + return + } + if this.TimeTo> 0 && now> this.TimeTo { + return + } + + if !this.matchPort(port) { + return + } + + if !this.matchIP(ip) { + return + } + + return true +} + +func (this *IPTable) matchPort(port int) bool { + if port == 0 { + return false + } + if this.minPortWildcard { + if this.maxPortWildcard { + return true + } + if this.maxPort>= port { + return true + } + } + if this.maxPortWildcard { + if this.minPortWildcard { + return true + } + if this.minPort <= port { + return true + } + } + if (this.minPort> 0 || this.maxPort> 0) && this.minPort <= port && this.maxPort>= port { + return true + } + if len(this.ports)> 0 { + return lists.ContainsInt(this.ports, port) + } + return false +} + +func (this *IPTable) matchIP(ip string) bool { + if this.ipRange == nil { + return false + } + return this.ipRange.Contains(ip) +} diff --git a/teawaf/ip_table_test.go b/teawaf/ip_table_test.go new file mode 100644 index 0000000..ea57a67 --- /dev/null +++ b/teawaf/ip_table_test.go @@ -0,0 +1,142 @@ +package teawaf + +import ( + "github.com/iwind/TeaGo/assert" + "testing" + "time" +) + +func TestIPTable_MatchIP(t *testing.T) { + a := assert.NewAssertion(t) + + { + table := NewIPTable() + err := table.Init() + if err != nil { + t.Fatal(err) + } + a.IsFalse(table.Match("192.168.1.100", 8080)) + } + + { + table := NewIPTable() + table.IP = "*" + table.Port = "8080" + err := table.Init() + if err != nil { + t.Fatal(err) + } + t.Log("port:", table.minPort, table.maxPort) + a.IsTrue(table.Match("192.168.1.100", 8080)) + a.IsFalse(table.Match("192.168.1.100", 8081)) + } + + { + table := NewIPTable() + table.IP = "*" + table.Port = "8080-8082" + err := table.Init() + if err != nil { + t.Fatal(err) + } + t.Log("port:", table.minPort, table.maxPort) + a.IsTrue(table.Match("192.168.1.100", 8080)) + a.IsTrue(table.Match("192.168.1.100", 8081)) + a.IsFalse(table.Match("192.168.1.100", 8083)) + } + + { + table := NewIPTable() + table.IP = "*" + table.Port = "*-8082" + err := table.Init() + if err != nil { + t.Fatal(err) + } + t.Log("port:", table.minPort, table.maxPort) + a.IsTrue(table.Match("192.168.1.100", 8079)) + a.IsTrue(table.Match("192.168.1.100", 8080)) + a.IsTrue(table.Match("192.168.1.100", 8081)) + a.IsFalse(table.Match("192.168.1.100", 8083)) + } + + { + table := NewIPTable() + table.IP = "*" + table.Port = "8080-*" + err := table.Init() + if err != nil { + t.Fatal(err) + } + t.Log("port:", table.minPort, table.maxPort) + a.IsFalse(table.Match("192.168.1.100", 8079)) + a.IsTrue(table.Match("192.168.1.100", 8080)) + a.IsTrue(table.Match("192.168.1.100", 8081)) + a.IsTrue(table.Match("192.168.1.100", 8083)) + } + + { + table := NewIPTable() + table.IP = "*" + table.Port = "*" + err := table.Init() + if err != nil { + t.Fatal(err) + } + t.Log("port:", table.minPort, table.maxPort) + a.IsTrue(table.Match("192.168.1.100", 8079)) + a.IsTrue(table.Match("192.168.1.100", 8080)) + a.IsTrue(table.Match("192.168.1.100", 8081)) + a.IsTrue(table.Match("192.168.1.100", 8083)) + } + + { + table := NewIPTable() + table.IP = "192.168.1.100" + table.Port = "*" + err := table.Init() + if err != nil { + t.Fatal(err) + } + t.Log("port:", table.minPort, table.maxPort) + a.IsTrue(table.Match("192.168.1.100", 8080)) + } + + { + table := NewIPTable() + table.IP = "192.168.1.99-192.168.1.101" + table.Port = "*" + err := table.Init() + if err != nil { + t.Fatal(err) + } + t.Log("port:", table.minPort, table.maxPort) + a.IsTrue(table.Match("192.168.1.100", 8080)) + } + + { + table := NewIPTable() + table.IP = "192.168.1.99/24" + table.Port = "*" + err := table.Init() + if err != nil { + t.Fatal(err) + } + t.Log("ip:", table.ipRange) + a.IsTrue(table.Match("192.168.1.100", 8080)) + a.IsFalse(table.Match("192.168.2.100", 8080)) + } + + { + table := NewIPTable() + table.IP = "192.168.1.99/24" + table.TimeTo = time.Now().Unix() - 10 + table.Port = "*" + err := table.Init() + if err != nil { + t.Fatal(err) + } + a.IsFalse(table.Match("192.168.1.100", 8080)) + a.IsFalse(table.Match("192.168.2.100", 8080)) + } +} diff --git a/teawaf/waf.go b/teawaf/waf.go index 7111f33..009ed1a 100644 --- a/teawaf/waf.go +++ b/teawaf/waf.go @@ -27,6 +27,8 @@ type WAF struct { ActionBlock *BlockAction `yaml:"actionBlock" json:"actionBlock"` // action block config + IPTables []*IPTable `yaml:"ipTables" json:"ipTables"` // IP table list + hasInboundRules bool hasOutboundRules bool onActionCallback func(action ActionString) (goNext bool) diff --git a/teaweb/actions/default/actionutils/ParentAction.go b/teaweb/actions/default/actionutils/ParentAction.go new file mode 100644 index 0000000..7f441c0 --- /dev/null +++ b/teaweb/actions/default/actionutils/ParentAction.go @@ -0,0 +1,26 @@ +package actionutils + +import ( + "github.com/iwind/TeaGo/actions" + "net/http" +) + +type ParentAction struct { + actions.ActionObject +} + +func (this *ParentAction) MainMenu(menuItem string) { + this.Data["mainMenu"] = menuItem +} + +func (this *ParentAction) SecondMenu(menuItem string) { + this.Data["secondMenu"] = menuItem +} + +func (this *ParentAction) ErrorPage(message string) { + if this.Request.Method == http.MethodGet { + this.WriteString(message) + } else { + this.Fail(message) + } +} diff --git a/teaweb/actions/default/cache/cleanPolicy.go b/teaweb/actions/default/cache/cleanPolicy.go index e094f15..427a07c 100644 --- a/teaweb/actions/default/cache/cleanPolicy.go +++ b/teaweb/actions/default/cache/cleanPolicy.go @@ -3,15 +3,19 @@ package cache import ( "github.com/TeaWeb/code/teacache" "github.com/TeaWeb/code/teaconfigs/shared" - "github.com/iwind/TeaGo/actions" + "github.com/TeaWeb/code/teaweb/actions/default/actionutils" ) -type CleanPolicyAction actions.Action +type CleanPolicyAction struct { + actionutils.ParentAction +} // 清理 func (this *CleanPolicyAction) Run(params struct { Filename string }) { + this.SecondMenu("clean") + policy := shared.NewCachePolicyFromFile(params.Filename) if policy == nil { this.Data["result"] = "找不到Policy" @@ -34,7 +38,9 @@ func (this *CleanPolicyAction) RunPost(params struct { manager := teacache.FindCachePolicyManager(params.Filename) if manager == nil { manager = teacache.NewManagerFromConfig(policy) - defer manager.Close() + defer func() { + _ = manager.Close() + }() } if manager == nil { diff --git a/teaweb/actions/default/cache/init.go b/teaweb/actions/default/cache/init.go index 6b2b4a6..cf81011 100644 --- a/teaweb/actions/default/cache/init.go +++ b/teaweb/actions/default/cache/init.go @@ -19,6 +19,7 @@ func init() { GetPost("/statPolicy", new(StatPolicyAction)). Get("/policy", new(PolicyAction)). GetPost("/cleanPolicy", new(CleanPolicyAction)). + GetPost("/refreshPolicy", new(RefreshPolicyAction)). EndAll() }) } diff --git a/teaweb/actions/default/cache/policy.go b/teaweb/actions/default/cache/policy.go index b35aa6c..9a79085 100644 --- a/teaweb/actions/default/cache/policy.go +++ b/teaweb/actions/default/cache/policy.go @@ -4,16 +4,20 @@ import ( "github.com/TeaWeb/code/teacache" "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teaconfigs/shared" - "github.com/iwind/TeaGo/actions" + "github.com/TeaWeb/code/teaweb/actions/default/actionutils" "github.com/iwind/TeaGo/maps" ) -type PolicyAction actions.Action +type PolicyAction struct { + actionutils.ParentAction +} // 缓存策略详情 func (this *PolicyAction) Run(params struct { Filename string }) { + this.SecondMenu("policy") + policy := shared.NewCachePolicyFromFile(params.Filename) if policy == nil { this.Fail("找不到Policy") diff --git a/teaweb/actions/default/cache/refreshPolicy.go b/teaweb/actions/default/cache/refreshPolicy.go new file mode 100644 index 0000000..b02ea00 --- /dev/null +++ b/teaweb/actions/default/cache/refreshPolicy.go @@ -0,0 +1,77 @@ +package cache + +import ( + "github.com/TeaWeb/code/teacache" + "github.com/TeaWeb/code/teaconfigs/shared" + "github.com/TeaWeb/code/teaweb/actions/default/actionutils" + "github.com/iwind/TeaGo/actions" + "strings" +) + +type RefreshPolicyAction struct { + actionutils.ParentAction +} + +func (this *RefreshPolicyAction) RunGet(params struct { + Filename string +}) { + this.SecondMenu("refresh") + policy := shared.NewCachePolicyFromFile(params.Filename) + if policy == nil { + this.ErrorPage("找不到Policy") + return + } + + this.Data["policy"] = policy + + this.Show() +} + +func (this *RefreshPolicyAction) RunPost(params struct { + Filename string + Prefixes string + Must *actions.Must +}) { + policy := shared.NewCachePolicyFromFile(params.Filename) + if policy == nil { + this.Data["result"] = "找不到Policy" + this.Fail() + } + + prefixes := []string{} + if len(params.Prefixes)> 0 { + lines := strings.Split(params.Prefixes, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if len(line) == 0 { + continue + } + prefixes = append(prefixes, line) + } + } + + if len(prefixes) == 0 { + this.Success() + } + + manager := teacache.FindCachePolicyManager(params.Filename) + if manager == nil { + manager = teacache.NewManagerFromConfig(policy) + defer func() { + _ = manager.Close() + }() + } + + if manager == nil { + this.Fail("找不到管理器") + } + + count, err := manager.DeletePrefixes(prefixes) + if err != nil { + this.Fail("刷新失败:" + err.Error()) + } + + this.Data["count"] = count + + this.Success() +} diff --git a/teaweb/actions/default/cache/statPolicy.go b/teaweb/actions/default/cache/statPolicy.go index 1f534d5..02a123b 100644 --- a/teaweb/actions/default/cache/statPolicy.go +++ b/teaweb/actions/default/cache/statPolicy.go @@ -4,16 +4,20 @@ import ( "fmt" "github.com/TeaWeb/code/teacache" "github.com/TeaWeb/code/teaconfigs/shared" - "github.com/iwind/TeaGo/actions" + "github.com/TeaWeb/code/teaweb/actions/default/actionutils" "github.com/iwind/TeaGo/maps" ) -type StatPolicyAction actions.Action +type StatPolicyAction struct { + actionutils.ParentAction +} // 统计 func (this *StatPolicyAction) Run(params struct { Filename string }) { + this.SecondMenu("stat") + policy := shared.NewCachePolicyFromFile(params.Filename) if policy == nil { this.Fail("找不到Policy") diff --git a/teaweb/actions/default/cache/testPolicy.go b/teaweb/actions/default/cache/testPolicy.go index e74479e..d1237c3 100644 --- a/teaweb/actions/default/cache/testPolicy.go +++ b/teaweb/actions/default/cache/testPolicy.go @@ -3,15 +3,19 @@ package cache import ( "github.com/TeaWeb/code/teacache" "github.com/TeaWeb/code/teaconfigs/shared" - "github.com/iwind/TeaGo/actions" + "github.com/TeaWeb/code/teaweb/actions/default/actionutils" ) -type TestPolicyAction actions.Action +type TestPolicyAction struct { + actionutils.ParentAction +} // 测试缓存策略 func (this *TestPolicyAction) Run(params struct { Filename string }) { + this.SecondMenu("test") + policy := shared.NewCachePolicyFromFile(params.Filename) if policy == nil { this.Fail("找不到Policy") diff --git a/teaweb/actions/default/cache/updatePolicy.go b/teaweb/actions/default/cache/updatePolicy.go index 4fe8a0e..e167fc9 100644 --- a/teaweb/actions/default/cache/updatePolicy.go +++ b/teaweb/actions/default/cache/updatePolicy.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/TeaWeb/code/teacache" "github.com/TeaWeb/code/teaconfigs/shared" + "github.com/TeaWeb/code/teaweb/actions/default/actionutils" "github.com/TeaWeb/code/teaweb/actions/default/cache/cacheutils" "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" "github.com/iwind/TeaGo/actions" @@ -12,12 +13,16 @@ import ( "github.com/iwind/TeaGo/types" ) -type UpdatePolicyAction actions.Action +type UpdatePolicyAction struct { + actionutils.ParentAction +} // 修改缓存策略 func (this *UpdatePolicyAction) Run(params struct { Filename string }) { + this.SecondMenu("policy") + policy := shared.NewCachePolicyFromFile(params.Filename) if policy == nil { this.Fail("找不到要修改的缓存策略") From e27b23bc1c3b1bf05353c8f00b1b7d7294e9ced2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Fri, 3 Apr 2020 16:11:14 +0800 Subject: [PATCH 091/121] =?UTF-8?q?[proxy]TCP=20on=20TLS=E6=94=AF=E6=8C=81?= =?UTF-8?q?SNI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/listener.go | 33 ++++++++++++++++++++++++++++++--- teaproxy/manager.go | 15 ++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/teaproxy/listener.go b/teaproxy/listener.go index 48de39f..1a6256a 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -369,7 +369,22 @@ func (this *Listener) startTCPServer() error { break } - go this.connectTCPBackend(clientConn) + var serverName = "" + tlsConn, ok := clientConn.(*tls.Conn) + if ok { + go func() { + err = tlsConn.Handshake() + if err != nil { + logs.Error(err) + return + } + + serverName = tlsConn.ConnectionState().ServerName + this.connectTCPBackend(clientConn, serverName) + }() + } else { + go this.connectTCPBackend(clientConn, serverName) + } } } @@ -671,14 +686,26 @@ func (this *Listener) buildTLSConfig() *tls.Config { } // 连接TCP后端 -func (this *Listener) connectTCPBackend(clientConn net.Conn) { +func (this *Listener) connectTCPBackend(clientConn net.Conn, serverName string) { defer teautils.Recover() client := NewTCPClient(func() *teaconfigs.ServerConfig { + this.serversLocker.RLock() + if len(this.currentServers) == 0 { + this.serversLocker.RUnlock() return nil } - return this.currentServers[0] + + if len(serverName) == 0 { + defer this.serversLocker.RUnlock() + return this.currentServers[0] + } + + this.serversLocker.RUnlock() + + server, _ := this.findNamedServer(serverName) + return server }, clientConn) this.connectingTCPLocker.Lock() this.connectingTCPMap[clientConn] = client diff --git a/teaproxy/manager.go b/teaproxy/manager.go index b8ffe69..3c30b0a 100644 --- a/teaproxy/manager.go +++ b/teaproxy/manager.go @@ -147,7 +147,10 @@ func (this *Manager) ApplyServer(server *teaconfigs.ServerConfig) { // HTTPS if server.SSL != nil && server.SSL.On { - server.SSL.Validate() + err := server.SSL.Validate() + if err != nil { + logs.Error(err) + } for _, address := range server.SSL.ParseListenAddresses() { // 是否有端口 if shared.RegexpDigitNumber.MatchString(address) { @@ -181,7 +184,10 @@ func (this *Manager) ApplyServer(server *teaconfigs.ServerConfig) { // TCP+TLS if server.SSL != nil && server.SSL.On { - server.SSL.Validate() + err := server.SSL.Validate() + if err != nil { + logs.Error(err) + } for _, address := range server.SSL.ParseListenAddresses() { // 是否有端口 if shared.RegexpDigitNumber.MatchString(address) { @@ -261,7 +267,10 @@ func (this *Manager) RemoveServer(serverId string) { for _, listener := range this.listeners { if listener.HasServer(serverId) { - listener.Reload() + err := listener.Reload() + if err != nil { + logs.Error(err) + } } } } From 54793a4b84900306866a7c0b96ee9b36bfeb5ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Wed, 6 May 2020 10:05:26 +0800 Subject: [PATCH 092/121] =?UTF-8?q?=E4=BD=BF=E7=94=A8TeaWeb=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E7=9A=84MongoDB=EF=BC=8C=E8=B0=83=E6=95=B4=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E4=BD=BF=E7=94=A8=E7=9A=84=E5=86=85=E5=AD=98=E4=B8=8A?= =?UTF-8?q?=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teadb/driver_mongo.go | 4 +++- teaweb/actions/default/settings/mongo/install.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/teadb/driver_mongo.go b/teadb/driver_mongo.go index a6ece02..367564e 100644 --- a/teadb/driver_mongo.go +++ b/teadb/driver_mongo.go @@ -890,7 +890,9 @@ func (this *MongoDriver) startInstalledMongo() { stat, err := mem.VirtualMemory() if err == nil && stat.Total> 0 { count := stat.Total / 1024 / 1024 / 1024 - if count>= 3 { + if count>= 6 { + args = append(args, "--wiredTigerCacheSizeGB=2") + } else if count>= 3 { args = append(args, "--wiredTigerCacheSizeGB=1") } } diff --git a/teaweb/actions/default/settings/mongo/install.go b/teaweb/actions/default/settings/mongo/install.go index e932b11..a05c03d 100644 --- a/teaweb/actions/default/settings/mongo/install.go +++ b/teaweb/actions/default/settings/mongo/install.go @@ -278,8 +278,10 @@ func (this *InstallAction) start(mongodbDir string) { // 控制内存不能超过1G stat, err := mem.VirtualMemory() if err == nil && stat.Total> 0 { - count := stat.Total / 1024 / 1024 / 1024 - if count>= 3 { + size := stat.Total / 1024 / 1024 / 1024 + if size>= 6 { + args = append(args, "--wiredTigerCacheSizeGB=2") + } else if size> 3 { args = append(args, "--wiredTigerCacheSizeGB=1") } } From 9769120d68e71ba38643bafff674e1a6a04f3665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Wed, 6 May 2020 10:06:42 +0800 Subject: [PATCH 093/121] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E7=BB=86=E8=8A=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/notices/notice.go | 4 ++-- teaweb/actions/default/login/index.go | 4 ++-- teaweb/helpers/user_must_auth.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/teaconfigs/notices/notice.go b/teaconfigs/notices/notice.go index b075dc9..7edf85f 100644 --- a/teaconfigs/notices/notice.go +++ b/teaconfigs/notices/notice.go @@ -53,7 +53,7 @@ func NewNotice() *Notice { // 设置时间 func (this *Notice) SetTime(t time.Time) { - this.Timestamp = time.Now().Unix() + this.Timestamp = t.Unix() } // 计算Hash @@ -139,4 +139,4 @@ func (this *Notice) DBColumns() maps.Map { "agentItemId": this.Agent.ItemId, "agentThreshold": this.Agent.Threshold, } -} +} \ No newline at end of file diff --git a/teaweb/actions/default/login/index.go b/teaweb/actions/default/login/index.go index db4d100..9b55dd6 100644 --- a/teaweb/actions/default/login/index.go +++ b/teaweb/actions/default/login/index.go @@ -25,7 +25,7 @@ func (this *IndexAction) RunGet() { // 检查IP限制 if !configs.SharedAdminConfig().AllowIP(this.RequestRemoteIP()) { this.ResponseWriter.WriteHeader(http.StatusForbidden) - this.WriteString("TeaWeb Access Forbidden") + this.WriteString(teaconst.TeaProductName + " Access Forbidden") return } @@ -54,7 +54,7 @@ func (this *IndexAction) RunPost(params struct { // 检查IP限制 if !configs.SharedAdminConfig().AllowIP(this.RequestRemoteIP()) { this.ResponseWriter.WriteHeader(http.StatusForbidden) - this.WriteString("TeaWeb Access Forbidden") + this.WriteString(teaconst.TeaProductName + " Access Forbidden") return } diff --git a/teaweb/helpers/user_must_auth.go b/teaweb/helpers/user_must_auth.go index 8a5ddda..651a51a 100644 --- a/teaweb/helpers/user_must_auth.go +++ b/teaweb/helpers/user_must_auth.go @@ -20,7 +20,7 @@ func (this *UserMustAuth) BeforeAction(actionPtr actions.ActionWrapper, paramNam // 检查IP if !configs.SharedAdminConfig().AllowIP(action.RequestRemoteIP()) { action.ResponseWriter.WriteHeader(http.StatusForbidden) - action.WriteString("TeaWeb Access Forbidden") + action.WriteString(teaconst.TeaProductName + " Access Forbidden") return false } From 7f9111e24bd5e7d216fa8eaacb980f7234aeae09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Wed, 6 May 2020 10:22:45 +0800 Subject: [PATCH 094/121] =?UTF-8?q?[proxy]=E5=9F=9F=E5=90=8D=E4=B8=A5?= =?UTF-8?q?=E6=A0=BC=E5=8C=B9=E9=85=8D=E6=A8=A1=E5=BC=8F=E4=B8=8B=EF=BC=8C?= =?UTF-8?q?=E7=94=A8IP=E8=AE=BF=E9=97=AEHTTPS=E6=9C=8D=E5=8A=A1=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E7=A9=BA=E5=93=8D=E5=BA=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/listener.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/teaproxy/listener.go b/teaproxy/listener.go index 1a6256a..a04f89f 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -601,6 +601,10 @@ func (this *Listener) matchSSL(domain string) (*teaconfigs.SSLConfig, *tls.Certi // 如果域名为空,则取第一个 // 通常域名为空是因为是直接通过IP访问的 if len(domain) == 0 { + if teaconfigs.SharedProxySetting().MatchDomainStrictly { + return nil, nil, errors.New("[proxy]no tls server name matched") + } + if len(this.currentServers)> 0 && this.currentServers[0].SSL != nil { return this.currentServers[0].SSL, this.currentServers[0].SSL.FirstCert(), nil } From f84ce5d471a7987ee77d0ac3862353459f3eb073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Wed, 6 May 2020 11:06:50 +0800 Subject: [PATCH 095/121] =?UTF-8?q?[proxy]=E6=96=87=E4=BB=B6=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E5=A2=9E=E5=8A=A0=E2=80=9C=E6=98=AF=E5=90=A6=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E5=88=9B=E5=BB=BA=E7=9B=AE=E5=BD=95=E2=80=9D=E9=80=89?= =?UTF-8?q?=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacache/file.go | 20 +++++++++++++++++--- teaweb/actions/default/cache/createPolicy.go | 6 ++++-- teaweb/actions/default/cache/updatePolicy.go | 6 ++++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/teacache/file.go b/teacache/file.go index de8f0d7..f48d4e4 100644 --- a/teacache/file.go +++ b/teacache/file.go @@ -32,6 +32,7 @@ type FileManager struct { looper *timers.Looper dir string + autoCreate bool // 是否自动创建 writeLocker sync.RWMutex } @@ -89,13 +90,18 @@ func (this *FileManager) SetOptions(options map[string]interface{}) { this.Life = 1800 * time.Second } - dir, found := options["dir"] - if found { + dir, ok := options["dir"] + if ok { this.dir = types.String(dir) if !filepath.IsAbs(this.dir) { this.dir = Tea.Root + Tea.DS + this.dir } } + + autoCreate, ok := options["autoCreate"] + if ok { + this.autoCreate = types.Bool(autoCreate) + } } // 写入 @@ -110,7 +116,15 @@ func (this *FileManager) Write(key string, data []byte) error { dirFile := files.NewFile(this.dir) if !dirFile.IsDir() { - return errors.New("cache dir should be a valid dir") + // 自动创建 + if this.autoCreate { + err := dirFile.MkdirAll() + if err != nil { + return errors.New("can not create cache dir: " + err.Error()) + } + } else { + return errors.New("cache dir should be a valid dir") + } } md5 := stringutil.Md5(key) diff --git a/teaweb/actions/default/cache/createPolicy.go b/teaweb/actions/default/cache/createPolicy.go index acb2f7f..65c9ed8 100644 --- a/teaweb/actions/default/cache/createPolicy.go +++ b/teaweb/actions/default/cache/createPolicy.go @@ -42,7 +42,8 @@ func (this *CreatePolicyAction) RunPost(params struct { SkipSetCookie bool EnableRequestCachePragma bool - FileDir string + FileDir string + FileAutoCreate bool RedisNetwork string RedisHost string @@ -88,7 +89,8 @@ func (this *CreatePolicyAction) RunPost(params struct { Field("fileDir", params.FileDir). Require("请输入缓存存放目录") policy.Options = map[string]interface{}{ - "dir": params.FileDir, + "dir": params.FileDir, + "autoCreate": params.FileAutoCreate, } case "redis": params.Must. diff --git a/teaweb/actions/default/cache/updatePolicy.go b/teaweb/actions/default/cache/updatePolicy.go index e167fc9..3e3fc63 100644 --- a/teaweb/actions/default/cache/updatePolicy.go +++ b/teaweb/actions/default/cache/updatePolicy.go @@ -80,7 +80,8 @@ func (this *UpdatePolicyAction) RunPost(params struct { SkipSetCookie bool EnableRequestCachePragma bool - FileDir string + FileDir string + FileAutoCreate bool RedisNetwork string RedisHost string @@ -129,7 +130,8 @@ func (this *UpdatePolicyAction) RunPost(params struct { Field("fileDir", params.FileDir). Require("请输入缓存存放目录") policy.Options = map[string]interface{}{ - "dir": params.FileDir, + "dir": params.FileDir, + "autoCreate": params.FileAutoCreate, } case "redis": params.Must. From 5b201194587051e775d4c3b3d75d76ca018771c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Thu, 7 May 2020 15:30:21 +0800 Subject: [PATCH 096/121] =?UTF-8?q?=E5=B0=86=E9=83=A8=E5=88=86"TeaWeb"?= =?UTF-8?q?=E6=96=87=E5=AD=97=E6=94=B9=E6=88=90=E5=B8=B8=E9=87=8F=EF=BC=8C?= =?UTF-8?q?=E4=BB=A5=E4=BE=BF=E4=BA=8E=E7=94=A8=E6=88=B7=E5=8F=AF=E4=BB=A5?= =?UTF-8?q?=E8=87=AA=E8=A1=8C=E7=BC=96=E8=AF=91=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/agents/source_nginx_status.go | 2 +- teaconfigs/agents/source_teaweb.go | 2 +- teaconfigs/agents/source_url_connectivity.go | 4 ++-- teaconfigs/agents/source_webhook.go | 4 ++-- teaconfigs/backend.go | 2 +- teaconfigs/fastcgi.go | 2 +- teaconfigs/notices/media_webhook.go | 2 +- teaconfigs/notices/setting.go | 7 ++++--- teaconst/const.go | 1 + 9 files changed, 14 insertions(+), 12 deletions(-) diff --git a/teaconfigs/agents/source_nginx_status.go b/teaconfigs/agents/source_nginx_status.go index 07b903f..5084485 100644 --- a/teaconfigs/agents/source_nginx_status.go +++ b/teaconfigs/agents/source_nginx_status.go @@ -68,7 +68,7 @@ func (this *NginxStatusSource) Execute(params map[string]string) (value interfac "cost": time.Since(before).Seconds(), }, err } - req.Header.Set("User-Agent", "TeaWeb/"+teaconst.TeaVersion) + req.Header.Set("User-Agent", teaconst.TeaProductCode+"/"+teaconst.TeaVersion) client := teautils.SharedHttpClient(timeout) resp, err := client.Do(req) diff --git a/teaconfigs/agents/source_teaweb.go b/teaconfigs/agents/source_teaweb.go index 053173b..f0cdc6e 100644 --- a/teaconfigs/agents/source_teaweb.go +++ b/teaconfigs/agents/source_teaweb.go @@ -59,7 +59,7 @@ func (this *TeaWebSource) Execute(params map[string]string) (value interface{}, } return value, err } - req.Header.Set("User-Agent", "TeaWeb/"+teaconst.TeaVersion) + req.Header.Set("User-Agent", teaconst.TeaProductCode+"/"+teaconst.TeaVersion) timeout := this.Timeout if timeout <= 0 { diff --git a/teaconfigs/agents/source_url_connectivity.go b/teaconfigs/agents/source_url_connectivity.go index f75de78..c2badf0 100644 --- a/teaconfigs/agents/source_url_connectivity.go +++ b/teaconfigs/agents/source_url_connectivity.go @@ -76,7 +76,7 @@ func (this *URLConnectivitySource) Execute(params map[string]string) (value inte for _, param := range this.Params { _, ok := query[param.Name] if ok { - query[param.Name ] = append(query[param.Name], param.Value) + query[param.Name] = append(query[param.Name], param.Value) } else { query[param.Name] = []string{param.Value} } @@ -116,7 +116,7 @@ func (this *URLConnectivitySource) Execute(params map[string]string) (value inte _, ok := this.lookupHeader("User-Agent") if !ok { - req.Header.Set("User-Agent", "TeaWeb/"+teaconst.TeaVersion) + req.Header.Set("User-Agent", teaconst.TeaProductCode+"/"+teaconst.TeaVersion) } if this.Method == "POST" { diff --git a/teaconfigs/agents/source_webhook.go b/teaconfigs/agents/source_webhook.go index fff4bd7..4dc6b29 100644 --- a/teaconfigs/agents/source_webhook.go +++ b/teaconfigs/agents/source_webhook.go @@ -86,7 +86,7 @@ func (this *WebHookSource) Execute(params map[string]string) (value interface{}, for _, param := range this.Params { _, ok := query[param.Name] if ok { - query[param.Name ] = append(query[param.Name], param.Value) + query[param.Name] = append(query[param.Name], param.Value) } else { query[param.Name] = []string{param.Value} } @@ -119,7 +119,7 @@ func (this *WebHookSource) Execute(params map[string]string) (value interface{}, _, ok := this.lookupHeader("User-Agent") if !ok { - req.Header.Set("User-Agent", "TeaWeb/"+teaconst.TeaVersion) + req.Header.Set("User-Agent", teaconst.TeaProductCode+"/"+teaconst.TeaVersion) } if this.Method == "POST" { diff --git a/teaconfigs/backend.go b/teaconfigs/backend.go index 12525d8..ffe5bf5 100644 --- a/teaconfigs/backend.go +++ b/teaconfigs/backend.go @@ -297,7 +297,7 @@ func (this *BackendConfig) CheckHealth() bool { logs.Error(err) return false } - req.Header.Set("User-Agent", "TeaWeb/"+teaconst.TeaVersion) + req.Header.Set("User-Agent", teaconst.TeaProductCode+"/"+teaconst.TeaVersion) client := teautils.SharedHttpClient(timeout) resp, err := client.Do(req) if err != nil { diff --git a/teaconfigs/fastcgi.go b/teaconfigs/fastcgi.go index 3de4b95..8ab3c30 100644 --- a/teaconfigs/fastcgi.go +++ b/teaconfigs/fastcgi.go @@ -54,7 +54,7 @@ func (this *FastcgiConfig) Validate() error { this.paramsMap["SCRIPT_FILENAME"] = "" } if !this.paramsMap.Has("SERVER_SOFTWARE") { - this.paramsMap["SERVER_SOFTWARE"] = "teaweb/" + teaconst.TeaVersion + this.paramsMap["SERVER_SOFTWARE"] = teaconst.TeaProductCode + "/" + teaconst.TeaVersion } if !this.paramsMap.Has("REDIRECT_STATUS") { this.paramsMap["REDIRECT_STATUS"] = "200" diff --git a/teaconfigs/notices/media_webhook.go b/teaconfigs/notices/media_webhook.go index 680166f..8b68965 100644 --- a/teaconfigs/notices/media_webhook.go +++ b/teaconfigs/notices/media_webhook.go @@ -88,7 +88,7 @@ func (this *NoticeWebhookMedia) Send(user string, subject string, body string) ( if err != nil { return nil, err } - req.Header.Set("User-Agent", "TeaWeb/"+teaconst.TeaVersion) + req.Header.Set("User-Agent", teaconst.TeaProductCode+"/"+teaconst.TeaVersion) if len(this.Headers)> 0 { for _, h := range this.Headers { diff --git a/teaconfigs/notices/setting.go b/teaconfigs/notices/setting.go index f4b1859..2510518 100644 --- a/teaconfigs/notices/setting.go +++ b/teaconfigs/notices/setting.go @@ -2,6 +2,7 @@ package notices import ( "github.com/TeaWeb/code/teaconfigs/shared" + "github.com/TeaWeb/code/teaconst" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/files" "github.com/iwind/TeaGo/logs" @@ -163,9 +164,9 @@ func (this *NoticeSetting) NotifyReceivers(level NoticeLevel, receivers []*Notic } subjectContent := subject if len(subjectContent) == 0 { - subjectContent = "[TeaWeb][" + FindNoticeLevelName(level) + "]有新的通知" - } else if !strings.HasPrefix(subject, "[TeaWeb]") { - subjectContent = "[TeaWeb][" + FindNoticeLevelName(level) + "]" + subject + subjectContent = "[" + teaconst.TeaProductName + "][" + FindNoticeLevelName(level) + "]有新的通知" + } else if !strings.HasPrefix(subject, "["+teaconst.TeaProductName+"]") { + subjectContent = "[" + teaconst.TeaProductName + "][" + FindNoticeLevelName(level) + "]" + subject } _, err := raw.Send(user, subjectContent, body) if err != nil { diff --git a/teaconst/const.go b/teaconst/const.go index a2258af..b2239b1 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -5,4 +5,5 @@ const ( TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 + TeaProductCode = "TeaWeb" // 产品代号,会用在一些对外访问的User-Agent里面,一定要是英文 ) From b8ed6b7843a922c3e57b810267e40b0a204defda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月21日 10:55:56 +0800 Subject: [PATCH 097/121] =?UTF-8?q?=E4=B8=80=E4=BA=9B=E4=BA=A7=E5=93=81?= =?UTF-8?q?=E5=90=8D=E4=BD=BF=E7=94=A8=E5=B8=B8=E9=87=8F=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/notices/media_email.go | 3 ++- teawaf/action_block.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/teaconfigs/notices/media_email.go b/teaconfigs/notices/media_email.go index 6f9f34a..a831f3a 100644 --- a/teaconfigs/notices/media_email.go +++ b/teaconfigs/notices/media_email.go @@ -3,6 +3,7 @@ package notices import ( "crypto/tls" "errors" + "github.com/TeaWeb/code/teaconst" "net" "net/mail" "net/smtp" @@ -38,7 +39,7 @@ func (this *NoticeEmailMedia) Send(user string, subject string, body string) (re } contentType := "Content-Type: text/html; charset=UTF-8" - msg := []byte("To: " + user + "\r\nFrom: \"TeaWeb\" <" + this.From + ">\r\nSubject: " + subject + "\r\n" + contentType + "\r\n\r\n" + body) + msg := []byte("To: " + user + "\r\nFrom: \"" + teaconst.TeaProductName + "\" <" + this.From + ">\r\nSubject: " + subject + "\r\n" + contentType + "\r\n\r\n" + body) return nil, this.SendMail(this.From, []string{user}, msg) } diff --git a/teawaf/action_block.go b/teawaf/action_block.go index d5c9f8f..bab3d87 100644 --- a/teawaf/action_block.go +++ b/teawaf/action_block.go @@ -1,6 +1,7 @@ package teawaf import ( + "github.com/TeaWeb/code/teaconst" "github.com/TeaWeb/code/teautils" "github.com/TeaWeb/code/teawaf/requests" "github.com/iwind/TeaGo/Tea" @@ -73,7 +74,7 @@ func (this *BlockAction) Perform(waf *WAF, request *requests.Request, writer htt if len(this.Body)> 0 { _, _ = writer.Write([]byte(this.Body)) } else { - _, _ = writer.Write([]byte("The request is blocked by TeaWAF")) + _, _ = writer.Write([]byte("The request is blocked by " + teaconst.TeaProductName)) } } return false From ea0a75e292e844449383d64c046518d4727619c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月21日 10:56:06 +0800 Subject: [PATCH 098/121] =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=94=AF=E6=8C=81gzi?= =?UTF-8?q?p?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacache/process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/teacache/process.go b/teacache/process.go index 63c47a1..7a402a4 100644 --- a/teacache/process.go +++ b/teacache/process.go @@ -182,7 +182,7 @@ func ProcessAfterRequest(req *teaproxy.Request, writer *teaproxy.ResponseWriter) // check length contentLength := types.Int(writer.Header().Get("Content-Length")) - if contentLength != len(writer.Body()) { + if contentLength != len(writer.Body()) && writer.Header().Get("Content-Encoding") != "gzip" { return true } From b144672f90e282298cdccb6b03460248a7f66c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月25日 22:27:18 +0800 Subject: [PATCH 099/121] =?UTF-8?q?HTTPS=E6=94=AF=E6=8C=81TLS=E7=9A=84SNI?= =?UTF-8?q?=EF=BC=9B=E4=BF=AE=E6=94=B9=E6=8B=92=E7=BB=9D=E9=9D=9E=E6=B3=95?= =?UTF-8?q?=E5=9F=9F=E5=90=8D=E7=9A=84=E5=A4=84=E7=90=86=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/listener.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/teaproxy/listener.go b/teaproxy/listener.go index a04f89f..aefa97a 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -405,6 +405,14 @@ func (this *Listener) handleHTTP(writer http.ResponseWriter, rawRequest *http.Re // 域名 reqHost := rawRequest.Host + // TLS域名 + if rawRequest.TLS != nil { + serverName := rawRequest.TLS.ServerName + if len(serverName)> 0 { + reqHost = serverName + } + } + // 防止空Host if len(reqHost) == 0 { ctx := rawRequest.Context() @@ -422,6 +430,18 @@ func (this *Listener) handleHTTP(writer http.ResponseWriter, rawRequest *http.Re } server, serverName := this.findNamedServer(domain) if server == nil { + // 严格匹配域名模式下,我们拒绝用户访问 + if teaconfigs.SharedProxySetting().MatchDomainStrictly { + hijacker, ok := writer.(http.Hijacker) + if ok { + conn, _, _ := hijacker.Hijack() + if conn != nil { + _ = conn.Close() + return + } + } + } + http.Error(writer, "404 page not found: '"+rawRequest.URL.String()+"'", http.StatusNotFound) return } From e61a40b02b92c88123f1e6fb32d422225059d3d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月28日 15:12:23 +0800 Subject: [PATCH 100/121] =?UTF-8?q?[cache]=E6=89=B9=E9=87=8F=E5=88=B7?= =?UTF-8?q?=E6=96=B0=E7=BC=93=E5=AD=98=E6=97=B6=EF=BC=8C=E4=BC=9A=E5=B0=86?= =?UTF-8?q?=E6=AD=A4=E5=8A=A8=E4=BD=9C=E5=90=8C=E6=AD=A5=E5=88=B0=E9=9B=86?= =?UTF-8?q?=E7=BE=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacluster/action_fail.go | 2 +- teacluster/action_notify.go | 2 +- teacluster/action_ping.go | 2 +- teacluster/action_pull.go | 2 +- teacluster/action_push.go | 2 +- teacluster/action_register.go | 2 +- teacluster/action_run.go | 57 +++++++++++++++++++ teacluster/action_success.go | 2 +- teacluster/action_sum.go | 2 +- teacluster/action_sync.go | 2 +- teacluster/codes.go | 16 ++++++ teacluster/init.go | 1 + teaconfigs/proxy_setting.go | 1 + teaweb/actions/default/cache/refreshPolicy.go | 14 +++++ 14 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 teacluster/action_run.go create mode 100644 teacluster/codes.go diff --git a/teacluster/action_fail.go b/teacluster/action_fail.go index 5988d7d..2f766a8 100644 --- a/teacluster/action_fail.go +++ b/teacluster/action_fail.go @@ -11,5 +11,5 @@ func (this *FailAction) Name() string { } func (this *FailAction) TypeId() int8 { - return 2 + return ActionCodeFail } diff --git a/teacluster/action_notify.go b/teacluster/action_notify.go index 11e29f6..fbc00dc 100644 --- a/teacluster/action_notify.go +++ b/teacluster/action_notify.go @@ -10,7 +10,7 @@ func (this *NotifyAction) Name() string { } func (this *NotifyAction) TypeId() int8 { - return 4 + return ActionCodeNotify } func (this *NotifyAction) Execute() error { diff --git a/teacluster/action_ping.go b/teacluster/action_ping.go index e88045a..2166705 100644 --- a/teacluster/action_ping.go +++ b/teacluster/action_ping.go @@ -12,5 +12,5 @@ func (this *PingAction) Name() string { } func (this *PingAction) TypeId() int8 { - return 7 + return ActionCodePing } diff --git a/teacluster/action_pull.go b/teacluster/action_pull.go index a635aa7..287dd91 100644 --- a/teacluster/action_pull.go +++ b/teacluster/action_pull.go @@ -20,5 +20,5 @@ func (this *PullAction) Execute() error { } func (this *PullAction) TypeId() int8 { - return 6 + return ActionCodePull } diff --git a/teacluster/action_push.go b/teacluster/action_push.go index c4e7366..608f0d1 100644 --- a/teacluster/action_push.go +++ b/teacluster/action_push.go @@ -44,5 +44,5 @@ func (this *PushAction) OnFail(fail *FailAction) error { } func (this *PushAction) TypeId() int8 { - return 5 + return ActionCodePush } diff --git a/teacluster/action_register.go b/teacluster/action_register.go index d8f765c..c358e72 100644 --- a/teacluster/action_register.go +++ b/teacluster/action_register.go @@ -40,5 +40,5 @@ func (this *RegisterAction) OnFail(fail *FailAction) error { } func (this *RegisterAction) TypeId() int8 { - return 3 + return ActionCodeRegister } diff --git a/teacluster/action_run.go b/teacluster/action_run.go new file mode 100644 index 0000000..c414d8c --- /dev/null +++ b/teacluster/action_run.go @@ -0,0 +1,57 @@ +package teacluster + +import ( + "github.com/TeaWeb/code/teacache" + "github.com/TeaWeb/code/teaconfigs/shared" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" + "github.com/iwind/TeaGo/types" +) + +type RunAction struct { + Action + + Cmd string + Data maps.Map +} + +func (this *RunAction) Name() string { + return "run" +} + +func (this *RunAction) Execute() error { + switch this.Cmd { + case "cache.refresh": + this.runCacheRefresh() + } + return nil +} + +func (this *RunAction) TypeId() int8 { + return ActionCodeRun +} + +// clean cache with prefixes +func (this *RunAction) runCacheRefresh() { + filename := this.Data.GetString("filename") + policy := shared.NewCachePolicyFromFile(filename) + if policy == nil { + logs.Println("[cluster][cache.refresh]can not find policy with '" + filename + "'") + return + } + + manager := teacache.FindCachePolicyManager(filename) + if manager == nil { + manager = teacache.NewManagerFromConfig(policy) + } + prefixes := this.Data.GetSlice("prefixes") + prefixStrings := []string{} + for _, prefix := range prefixes { + prefixStrings = append(prefixStrings, types.String(prefix)) + } + _, err := manager.DeletePrefixes(prefixStrings) + if err != nil { + logs.Println("[cluster][cache.refresh]delete prefixes") + return + } +} diff --git a/teacluster/action_success.go b/teacluster/action_success.go index 000db6a..1e83eed 100644 --- a/teacluster/action_success.go +++ b/teacluster/action_success.go @@ -14,5 +14,5 @@ func (this *SuccessAction) Name() string { } func (this *SuccessAction) TypeId() int8 { - return 1 + return ActionCodeSuccess } diff --git a/teacluster/action_sum.go b/teacluster/action_sum.go index 5ee20d7..93d4ebb 100644 --- a/teacluster/action_sum.go +++ b/teacluster/action_sum.go @@ -63,5 +63,5 @@ func (this *SumAction) OnFail(fail *FailAction) error { } func (this *SumAction) TypeId() int8 { - return 9 + return ActionCodeSum } diff --git a/teacluster/action_sync.go b/teacluster/action_sync.go index 53d7a46..2a4ea0e 100644 --- a/teacluster/action_sync.go +++ b/teacluster/action_sync.go @@ -66,5 +66,5 @@ func (this *SyncAction) Execute() error { } func (this *SyncAction) TypeId() int8 { - return 8 + return ActionCodeSync } diff --git a/teacluster/codes.go b/teacluster/codes.go new file mode 100644 index 0000000..1c3edc0 --- /dev/null +++ b/teacluster/codes.go @@ -0,0 +1,16 @@ +package teacluster + +type ActionCode = int8 + +const ( + ActionCodeSuccess ActionCode = 1 + ActionCodeFail ActionCode = 2 + ActionCodeRegister ActionCode = 3 + ActionCodeNotify ActionCode = 4 + ActionCodePush ActionCode = 5 + ActionCodePull ActionCode = 6 + ActionCodePing ActionCode = 7 + ActionCodeSync ActionCode = 8 + ActionCodeSum ActionCode = 9 + ActionCodeRun ActionCode = 10 +) diff --git a/teacluster/init.go b/teacluster/init.go index 8f3cf58..1991598 100644 --- a/teacluster/init.go +++ b/teacluster/init.go @@ -26,6 +26,7 @@ func init() { new(SumAction), new(SyncAction), new(PingAction), + new(RunAction), ) TeaGo.BeforeStart(func(server *TeaGo.Server) { diff --git a/teaconfigs/proxy_setting.go b/teaconfigs/proxy_setting.go index f573fee..65d705e 100644 --- a/teaconfigs/proxy_setting.go +++ b/teaconfigs/proxy_setting.go @@ -15,6 +15,7 @@ var sharedProxySetting *ProxySetting = nil // 代理全局设置 type ProxySetting struct { MatchDomainStrictly bool `yaml:"matchDomainStrictly" json:"matchDomainStrictly"` // 是否严格匹配域名 + } // 取得共享的代理设置 diff --git a/teaweb/actions/default/cache/refreshPolicy.go b/teaweb/actions/default/cache/refreshPolicy.go index b02ea00..0199e4b 100644 --- a/teaweb/actions/default/cache/refreshPolicy.go +++ b/teaweb/actions/default/cache/refreshPolicy.go @@ -2,9 +2,11 @@ package cache import ( "github.com/TeaWeb/code/teacache" + "github.com/TeaWeb/code/teacluster" "github.com/TeaWeb/code/teaconfigs/shared" "github.com/TeaWeb/code/teaweb/actions/default/actionutils" "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/maps" "strings" ) @@ -73,5 +75,17 @@ func (this *RefreshPolicyAction) RunPost(params struct { this.Data["count"] = count + // 清除节点 + action := new(teacluster.RunAction) + action.Cmd = "cache.refresh" + action.Data = maps.Map{ + "filename": params.Filename, + "prefixes": prefixes, + } + err = teacluster.SharedManager.Write(action) + if err != nil { + this.Fail("刷新集群失败:" + err.Error()) + } + this.Success() } From 2839713892dd1521ff0bcba1cea8bcd524c82b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月28日 15:17:34 +0800 Subject: [PATCH 101/121] =?UTF-8?q?[cache]=E6=B8=85=E7=90=86=E6=89=80?= =?UTF-8?q?=E6=9C=89=E7=BC=93=E5=AD=98=E6=97=B6=EF=BC=8C=E4=BC=9A=E5=B0=86?= =?UTF-8?q?=E6=AD=A4=E5=8A=A8=E4=BD=9C=E5=90=8C=E6=AD=A5=E5=88=B0=E9=9B=86?= =?UTF-8?q?=E7=BE=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacluster/action_run.go | 30 ++++++++++++++++++- teaweb/actions/default/cache/cleanPolicy.go | 13 ++++++++ teaweb/actions/default/cache/refreshPolicy.go | 4 +-- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/teacluster/action_run.go b/teacluster/action_run.go index c414d8c..80c5f02 100644 --- a/teacluster/action_run.go +++ b/teacluster/action_run.go @@ -23,6 +23,8 @@ func (this *RunAction) Execute() error { switch this.Cmd { case "cache.refresh": this.runCacheRefresh() + case "cache.clean": + this.runCacheClean() } return nil } @@ -43,6 +45,9 @@ func (this *RunAction) runCacheRefresh() { manager := teacache.FindCachePolicyManager(filename) if manager == nil { manager = teacache.NewManagerFromConfig(policy) + defer func() { + _ = manager.Close() + }() } prefixes := this.Data.GetSlice("prefixes") prefixStrings := []string{} @@ -51,7 +56,30 @@ func (this *RunAction) runCacheRefresh() { } _, err := manager.DeletePrefixes(prefixStrings) if err != nil { - logs.Println("[cluster][cache.refresh]delete prefixes") + logs.Println("[cluster][cache.refresh]delete prefixes: " + err.Error()) + return + } +} + +// clean cache +func (this *RunAction) runCacheClean() { + filename := this.Data.GetString("filename") + policy := shared.NewCachePolicyFromFile(filename) + if policy == nil { + logs.Println("[cluster][cache.clean]can not find policy with '" + filename + "'") + return + } + + manager := teacache.FindCachePolicyManager(filename) + if manager == nil { + manager = teacache.NewManagerFromConfig(policy) + defer func() { + _ = manager.Close() + }() + } + err := manager.Clean() + if err != nil { + logs.Println("[cluster][cache.clean]clean: " + err.Error()) return } } diff --git a/teaweb/actions/default/cache/cleanPolicy.go b/teaweb/actions/default/cache/cleanPolicy.go index 427a07c..f3ea76f 100644 --- a/teaweb/actions/default/cache/cleanPolicy.go +++ b/teaweb/actions/default/cache/cleanPolicy.go @@ -2,8 +2,10 @@ package cache import ( "github.com/TeaWeb/code/teacache" + "github.com/TeaWeb/code/teacluster" "github.com/TeaWeb/code/teaconfigs/shared" "github.com/TeaWeb/code/teaweb/actions/default/actionutils" + "github.com/iwind/TeaGo/maps" ) type CleanPolicyAction struct { @@ -53,6 +55,17 @@ func (this *CleanPolicyAction) RunPost(params struct { this.Fail() } + // 同步到集群 + action := new(teacluster.RunAction) + action.Cmd = "cache.clean" + action.Data = maps.Map{ + "filename": params.Filename, + } + err = teacluster.SharedManager.Write(action) + if err != nil { + this.Fail("同步到集群失败:" + err.Error()) + } + this.Data["result"] = "清理完成" this.Success() } diff --git a/teaweb/actions/default/cache/refreshPolicy.go b/teaweb/actions/default/cache/refreshPolicy.go index 0199e4b..59d9f71 100644 --- a/teaweb/actions/default/cache/refreshPolicy.go +++ b/teaweb/actions/default/cache/refreshPolicy.go @@ -75,7 +75,7 @@ func (this *RefreshPolicyAction) RunPost(params struct { this.Data["count"] = count - // 清除节点 + // 同步到集群 action := new(teacluster.RunAction) action.Cmd = "cache.refresh" action.Data = maps.Map{ @@ -84,7 +84,7 @@ func (this *RefreshPolicyAction) RunPost(params struct { } err = teacluster.SharedManager.Write(action) if err != nil { - this.Fail("刷新集群失败:" + err.Error()) + this.Fail("同步到集群失败:" + err.Error()) } this.Success() From 9774d1478f1d5369d42a9e29a60914c319b4d308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月28日 15:57:15 +0800 Subject: [PATCH 102/121] =?UTF-8?q?[waf]=E5=A6=82=E6=9E=9C=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E7=A0=81=E6=98=AF444=EF=BC=8C=E5=88=99=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E5=85=B3=E9=97=AD=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/response.go | 11 +++++++++++ teawaf/action_block.go | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/teaproxy/response.go b/teaproxy/response.go index bcf3bcd..a0535ec 100644 --- a/teaproxy/response.go +++ b/teaproxy/response.go @@ -1,10 +1,12 @@ package teaproxy import ( + "bufio" "bytes" "compress/gzip" "github.com/TeaWeb/code/teaconfigs" "github.com/iwind/TeaGo/logs" + "net" "net/http" ) @@ -225,3 +227,12 @@ func (this *ResponseWriter) Close() { this.gzipWriter = nil } } + +// Hijack +func (this *ResponseWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) { + hijack, ok := this.writer.(http.Hijacker) + if ok { + return hijack.Hijack() + } + return +} diff --git a/teawaf/action_block.go b/teawaf/action_block.go index bab3d87..6bad340 100644 --- a/teawaf/action_block.go +++ b/teawaf/action_block.go @@ -27,6 +27,19 @@ type BlockAction struct { func (this *BlockAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) { if writer != nil { + // if status code eq 444, we close the connection + if this.StatusCode == 444 { + hijack, ok := writer.(http.Hijacker) + if ok { + conn, _, _ := hijack.Hijack() + if conn != nil { + _ = conn.Close() + return + } + } + } + + // output response if this.StatusCode> 0 { writer.WriteHeader(this.StatusCode) } else { From 669995ea4a9ad9fd660e25d25c84c320fa7cb9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月28日 19:38:43 +0800 Subject: [PATCH 103/121] =?UTF-8?q?[waf]POST=E9=AA=8C=E8=AF=81=E9=80=9A?= =?UTF-8?q?=E8=BF=87=E5=90=8E=E8=B7=B3=E8=BD=AC=E5=88=B0=E8=A1=A8=E5=8D=95?= =?UTF-8?q?=E6=8F=90=E4=BA=A4=E5=89=8D=E7=9A=84=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teatesting/server.go | 16 ++++++- teawaf/action_captcha.go | 59 +++----------------------- teawaf/captcha_validator.go | 84 +++++++++++++++++++++++++++++++++++++ teawaf/waf.go | 9 ++++ 4 files changed, 113 insertions(+), 55 deletions(-) create mode 100644 teawaf/captcha_validator.go diff --git a/teatesting/server.go b/teatesting/server.go index 0a8c545..2162277 100644 --- a/teatesting/server.go +++ b/teatesting/server.go @@ -120,7 +120,7 @@ func StartTestServer() { }). Get("/cookie", func(req *http.Request, resp http.ResponseWriter) { - resp.Header().Add("Set-Cookie", "Asset_UserId=1; expires=Sun, 05-May-2019 14:42:21 GMT; path=/", ) + resp.Header().Add("Set-Cookie", "Asset_UserId=1; expires=Sun, 05-May-2019 14:42:21 GMT; path=/") _, _ = resp.Write([]byte("set cookie")) }). GetPost("/json", func(req *http.Request, resp http.ResponseWriter) { @@ -146,6 +146,14 @@ func StartTestServer() { _, _ = resp.Write(data) } }). + Post("/post", func(req *http.Request, resp http.ResponseWriter) { + data, err := httputil.DumpRequest(req, true) + if err != nil { + _, _ = resp.Write([]byte(err.Error())) + return + } + _, _ = resp.Write(data) + }). Put("/put", func(req *http.Request, resp http.ResponseWriter) { data, err := httputil.DumpRequest(req, true) if err != nil { @@ -174,6 +182,12 @@ func StartTestServer() {

Verify Yourself

THIS IS HTML BODY + +
+ + +
+

AltStyle によって変換されたページ (->オリジナル) /

`)) diff --git a/teawaf/action_captcha.go b/teawaf/action_captcha.go index 1c15b47..27496aa 100644 --- a/teawaf/action_captcha.go +++ b/teawaf/action_captcha.go @@ -1,15 +1,11 @@ package teawaf import ( - "bytes" - "encoding/base64" - "fmt" "github.com/TeaWeb/code/teawaf/requests" - "github.com/dchest/captcha" - "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/types" stringutil "github.com/iwind/TeaGo/utils/string" "net/http" + "net/url" "time" ) @@ -33,56 +29,11 @@ func (this *CaptchaAction) Perform(waf *WAF, request *requests.Request, writer h } } - // verify - if request.Method == http.MethodPost { - captchaId := request.FormValue("TEAWEB_WAF_CAPTCHA_ID") - if len(captchaId)> 0 { - captchaCode := request.FormValue("TEAWEB_WAF_CAPTCHA_CODE") - if captcha.VerifyString(captchaId, captchaCode) { - // set cookie - timestamp := fmt.Sprintf("%d", time.Now().Unix()+CaptchaSeconds) - m := stringutil.Md5(captchaSalt + timestamp) - http.SetCookie(writer, &http.Cookie{ - Name: "TEAWEB_WAF_CAPTCHA", - Value: m + timestamp, - MaxAge: CaptchaSeconds, - Path: "/", // all of dirs - }) - - http.Redirect(writer, request.Raw(), request.URL.String(), http.StatusTemporaryRedirect) - - return false - } - } - } - - // show captcha - captchaId := captcha.NewLen(6) - buf := bytes.NewBuffer([]byte{}) - err = captcha.WriteImage(buf, captchaId, 200, 100) - if err != nil { - logs.Error(err) - return true + refURL := request.URL.String() + if len(request.Referer())> 0 { + refURL = request.Referer() } + http.Redirect(writer, request.Raw(), "/WAFCAPTCHA?url="+url.QueryEscape(refURL), http.StatusTemporaryRedirect) - _, _ = writer.Write([]byte(` - - - Verify Yourself - - -
- - -
-

Input verify code above:

- -
-
- -
-
- -`)) return false } diff --git a/teawaf/captcha_validator.go b/teawaf/captcha_validator.go new file mode 100644 index 0000000..80c953b --- /dev/null +++ b/teawaf/captcha_validator.go @@ -0,0 +1,84 @@ +package teawaf + +import ( + "bytes" + "encoding/base64" + "fmt" + "github.com/TeaWeb/code/teawaf/requests" + "github.com/dchest/captcha" + "github.com/iwind/TeaGo/logs" + stringutil "github.com/iwind/TeaGo/utils/string" + "net/http" + "time" +) + +var captchaValidator = &CaptchaValidator{} + +type CaptchaValidator struct { +} + +func (this *CaptchaValidator) Run(request *requests.Request, writer http.ResponseWriter) { + if request.Method == http.MethodPost && len(request.FormValue("TEAWEB_WAF_CAPTCHA_ID"))> 0 { + this.validate(request, writer) + } else { + this.show(request, writer) + } +} + +func (this *CaptchaValidator) show(request *requests.Request, writer http.ResponseWriter) { + // show captcha + captchaId := captcha.NewLen(6) + buf := bytes.NewBuffer([]byte{}) + err := captcha.WriteImage(buf, captchaId, 200, 100) + if err != nil { + logs.Error(err) + return + } + + _, _ = writer.Write([]byte(` + + + Verify Yourself + + +
+ + ` + ` +
+

Input verify code above:

+ +
+
+ +
+
+ +`)) +} + +func (this *CaptchaValidator) validate(request *requests.Request, writer http.ResponseWriter) (allow bool) { + captchaId := request.FormValue("TEAWEB_WAF_CAPTCHA_ID") + if len(captchaId)> 0 { + captchaCode := request.FormValue("TEAWEB_WAF_CAPTCHA_CODE") + if captcha.VerifyString(captchaId, captchaCode) { + // set cookie + timestamp := fmt.Sprintf("%d", time.Now().Unix()+CaptchaSeconds) + m := stringutil.Md5(captchaSalt + timestamp) + http.SetCookie(writer, &http.Cookie{ + Name: "TEAWEB_WAF_CAPTCHA", + Value: m + timestamp, + MaxAge: CaptchaSeconds, + Path: "/", // all of dirs + }) + + rawURL := request.URL.Query().Get("url") + http.Redirect(writer, request.Raw(), rawURL, http.StatusSeeOther) + + return false + } else { + http.Redirect(writer, request.Raw(), request.URL.String(), http.StatusSeeOther) + } + } + + return true +} diff --git a/teawaf/waf.go b/teawaf/waf.go index 009ed1a..204b502 100644 --- a/teawaf/waf.go +++ b/teawaf/waf.go @@ -256,7 +256,16 @@ func (this *WAF) MatchRequest(rawReq *http.Request, writer http.ResponseWriter) if !this.hasInboundRules { return true, nil, nil, nil } + req := requests.NewRequest(rawReq) + + // validate captcha + if rawReq.URL.Path == "/WAFCAPTCHA" { + captchaValidator.Run(req, writer) + return + } + + // match rules for _, group := range this.Inbound { if !group.On { continue From 4ca7096e8b2a211b801924e3959fecd73c764b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月28日 21:44:12 +0800 Subject: [PATCH 104/121] =?UTF-8?q?[proxy]=E6=B7=BB=E5=8A=A0=E7=AE=80?= =?UTF-8?q?=E5=8D=95=E7=9A=84=E5=88=86=E7=BB=84=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/server_group.go | 41 +++++++++ teaconfigs/server_group_list.go | 87 +++++++++++++++++++ teaconfigs/server_group_list_test.go | 10 +++ teaweb/actions/default/cache/helper.go | 2 +- teaweb/actions/default/proxy/certs/helper.go | 2 +- teaweb/actions/default/proxy/helper.go | 6 +- teaweb/actions/default/proxy/import.go | 5 ++ teaweb/actions/default/proxy/locations/waf.go | 2 +- teaweb/actions/default/proxy/log/day.go | 2 +- teaweb/actions/default/proxy/log/history.go | 2 +- teaweb/actions/default/proxy/log/index.go | 2 +- .../default/proxy/log/policies/helper.go | 2 +- .../actions/default/proxy/notices/helper.go | 2 +- .../actions/default/proxy/proxyutils/menu.go | 7 +- .../actions/default/proxy/servergroups/add.go | 37 ++++++++ .../default/proxy/servergroups/addServer.go | 65 ++++++++++++++ .../default/proxy/servergroups/delete.go | 22 +++++ .../proxy/servergroups/deleteServer.go | 27 ++++++ .../default/proxy/servergroups/helper.go | 17 ++++ .../default/proxy/servergroups/index.go | 75 ++++++++++++++++ .../default/proxy/servergroups/init.go | 23 +++++ .../default/proxy/servergroups/update.go | 47 ++++++++++ teaweb/actions/default/proxy/servers/waf.go | 2 +- .../actions/default/proxy/settings/helper.go | 2 + teaweb/actions/default/proxy/settings/init.go | 2 - teaweb/actions/default/proxy/tunnel/index.go | 2 +- teaweb/actions/default/proxy/tunnel/update.go | 2 +- teaweb/actions/default/proxy/waf/helper.go | 2 +- 28 files changed, 478 insertions(+), 19 deletions(-) create mode 100644 teaconfigs/server_group.go create mode 100644 teaconfigs/server_group_list.go create mode 100644 teaconfigs/server_group_list_test.go create mode 100644 teaweb/actions/default/proxy/import.go create mode 100644 teaweb/actions/default/proxy/servergroups/add.go create mode 100644 teaweb/actions/default/proxy/servergroups/addServer.go create mode 100644 teaweb/actions/default/proxy/servergroups/delete.go create mode 100644 teaweb/actions/default/proxy/servergroups/deleteServer.go create mode 100644 teaweb/actions/default/proxy/servergroups/helper.go create mode 100644 teaweb/actions/default/proxy/servergroups/index.go create mode 100644 teaweb/actions/default/proxy/servergroups/init.go create mode 100644 teaweb/actions/default/proxy/servergroups/update.go diff --git a/teaconfigs/server_group.go b/teaconfigs/server_group.go new file mode 100644 index 0000000..62d485f --- /dev/null +++ b/teaconfigs/server_group.go @@ -0,0 +1,41 @@ +package teaconfigs + +import ( + "github.com/iwind/TeaGo/lists" + stringutil "github.com/iwind/TeaGo/utils/string" +) + +type ServerGroup struct { + Id string `yaml:"id" json:"id"` + IsOn bool `yaml:"isOn" json:"isOn"` + Name string `yaml:"name" json:"name"` + ServerIds []string `yaml:"serverIds" json:"serverIds"` +} + +func NewServerGroup() *ServerGroup { + return &ServerGroup{ + Id: stringutil.Rand(16), + IsOn: true, + ServerIds: []string{}, + } +} + +func (this *ServerGroup) Add(serverId ...string) { + for _, id := range serverId { + if lists.ContainsString(this.ServerIds, id) { + continue + } + this.ServerIds = append(this.ServerIds, id) + } +} + +func (this *ServerGroup) Remove(serverId string) { + result := []string{} + for _, id := range this.ServerIds { + if id == serverId { + continue + } + result = append(result, id) + } + this.ServerIds = result +} diff --git a/teaconfigs/server_group_list.go b/teaconfigs/server_group_list.go new file mode 100644 index 0000000..ea72ec6 --- /dev/null +++ b/teaconfigs/server_group_list.go @@ -0,0 +1,87 @@ +package teaconfigs + +import ( + "github.com/TeaWeb/code/teaconfigs/shared" + "github.com/go-yaml/yaml" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/logs" + "io/ioutil" + "os" +) + +type ServerGroupList struct { + Groups []*ServerGroup `yaml:"groups" json:"groups"` +} + +func SharedServerGroupList() *ServerGroupList { + list := &ServerGroupList{ + Groups: []*ServerGroup{}, + } + filename := "servergrouplist.conf" + data, err := ioutil.ReadFile(Tea.ConfigFile(filename)) + if err != nil { + if !os.IsNotExist(err) { + logs.Error(err) + } + return list + } + + err = yaml.Unmarshal(data, list) + if err != nil { + logs.Error(err) + return list + } + + return list +} + +func (this *ServerGroupList) Add(group *ServerGroup) { + this.Groups = append(this.Groups, group) +} + +func (this *ServerGroupList) Remove(id string) { + result := []*ServerGroup{} + for _, group := range this.Groups { + if group.Id == id { + continue + } + result = append(result, group) + } + this.Groups = result +} + +// 查找分组 +func (this *ServerGroupList) Find(id string) *ServerGroup { + for _, group := range this.Groups { + if group.Id == id { + return group + } + } + return nil +} + +// 检查是否包含某个代理服务ID +func (this *ServerGroupList) ContainsServer(serverId string) bool { + for _, group := range this.Groups { + for _, id := range group.ServerIds { + if id == serverId { + return true + } + } + } + return false +} + +// 保存 +func (this *ServerGroupList) Save() error { + data, err := yaml.Marshal(this) + if err != nil { + return err + } + + shared.Locker.Lock() + defer shared.Locker.WriteUnlock() + + filename := "servergrouplist.conf" + return ioutil.WriteFile(Tea.ConfigFile(filename), data, 0666) +} diff --git a/teaconfigs/server_group_list_test.go b/teaconfigs/server_group_list_test.go new file mode 100644 index 0000000..0968809 --- /dev/null +++ b/teaconfigs/server_group_list_test.go @@ -0,0 +1,10 @@ +package teaconfigs + +import ( + "github.com/iwind/TeaGo/logs" + "testing" +) + +func TestSharedServerGroupList(t *testing.T) { + logs.PrintAsJSON(SharedServerGroupList(), t) +} diff --git a/teaweb/actions/default/cache/helper.go b/teaweb/actions/default/cache/helper.go index 0ccea4a..dc9a95e 100644 --- a/teaweb/actions/default/cache/helper.go +++ b/teaweb/actions/default/cache/helper.go @@ -12,6 +12,6 @@ type Helper struct { // 缓存相关Helper func (this *Helper) BeforeAction(action *actions.ActionObject) { if action.Request.Method == http.MethodGet { - proxyutils.AddServerMenu(action) + proxyutils.AddServerMenu(action, false) } } diff --git a/teaweb/actions/default/proxy/certs/helper.go b/teaweb/actions/default/proxy/certs/helper.go index 725687d..7db0652 100644 --- a/teaweb/actions/default/proxy/certs/helper.go +++ b/teaweb/actions/default/proxy/certs/helper.go @@ -13,5 +13,5 @@ func (this *Helper) BeforeAction(action *actions.ActionObject) { if action.Request.Method != http.MethodGet { return } - proxyutils.AddServerMenu(action) + proxyutils.AddServerMenu(action, false) } diff --git a/teaweb/actions/default/proxy/helper.go b/teaweb/actions/default/proxy/helper.go index ee92c68..7b13ad3 100644 --- a/teaweb/actions/default/proxy/helper.go +++ b/teaweb/actions/default/proxy/helper.go @@ -11,6 +11,10 @@ type Helper struct { func (this *Helper) BeforeAction(action *actions.ActionObject) { if action.Request.Method == http.MethodGet && !action.HasPrefix("/proxy/status") { - proxyutils.AddServerMenu(action) + if action.HasPrefix("/proxy/add") { + proxyutils.AddServerMenu(action, false) + } else { + proxyutils.AddServerMenu(action, true) + } } } diff --git a/teaweb/actions/default/proxy/import.go b/teaweb/actions/default/proxy/import.go new file mode 100644 index 0000000..51ce85f --- /dev/null +++ b/teaweb/actions/default/proxy/import.go @@ -0,0 +1,5 @@ +package proxy + +import ( + _ "github.com/TeaWeb/code/teaweb/actions/default/proxy/servergroups" +) diff --git a/teaweb/actions/default/proxy/locations/waf.go b/teaweb/actions/default/proxy/locations/waf.go index 1f8c2de..d4de916 100644 --- a/teaweb/actions/default/proxy/locations/waf.go +++ b/teaweb/actions/default/proxy/locations/waf.go @@ -22,7 +22,7 @@ func (this *WafAction) RunGet(params struct { _, location := locationutils.SetCommonInfo(this, params.ServerId, params.LocationId, "waf") - proxyutils.AddServerMenu(this) + proxyutils.AddServerMenu(this, true) wafList := []maps.Map{} for _, waf := range teaconfigs.SharedWAFList().FindAllConfigs() { diff --git a/teaweb/actions/default/proxy/log/day.go b/teaweb/actions/default/proxy/log/day.go index 14c2228..f314fcd 100644 --- a/teaweb/actions/default/proxy/log/day.go +++ b/teaweb/actions/default/proxy/log/day.go @@ -46,7 +46,7 @@ func (this *DayAction) Run(params struct { } this.Data["searchIP"] = params.SearchIP - proxyutils.AddServerMenu(this) + proxyutils.AddServerMenu(this, true) // 检查数据库连接 this.Data["mongoError"] = "" diff --git a/teaweb/actions/default/proxy/log/history.go b/teaweb/actions/default/proxy/log/history.go index b28f741..8fd6c76 100644 --- a/teaweb/actions/default/proxy/log/history.go +++ b/teaweb/actions/default/proxy/log/history.go @@ -26,7 +26,7 @@ func (this *HistoryAction) Run(params struct { "id": server.Id, } - proxyutils.AddServerMenu(this) + proxyutils.AddServerMenu(this, true) // 检查MongoDB连接 this.Data["mongoError"] = "" diff --git a/teaweb/actions/default/proxy/log/index.go b/teaweb/actions/default/proxy/log/index.go index 61568dd..f50635e 100644 --- a/teaweb/actions/default/proxy/log/index.go +++ b/teaweb/actions/default/proxy/log/index.go @@ -32,7 +32,7 @@ func (this *IndexAction) Run(params struct { this.Data["errs"] = teaproxy.SharedManager.FindServerErrors(params.ServerId) - proxyutils.AddServerMenu(this) + proxyutils.AddServerMenu(this, true) this.Show() } diff --git a/teaweb/actions/default/proxy/log/policies/helper.go b/teaweb/actions/default/proxy/log/policies/helper.go index 4d2462e..bfa9df7 100644 --- a/teaweb/actions/default/proxy/log/policies/helper.go +++ b/teaweb/actions/default/proxy/log/policies/helper.go @@ -12,7 +12,7 @@ type Helper struct { // 相关Helper func (this *Helper) BeforeAction(action *actions.ActionObject) { if action.Request.Method == http.MethodGet { - proxyutils.AddServerMenu(action) + proxyutils.AddServerMenu(action, false) action.Data["selectedMenu"] = "list" } diff --git a/teaweb/actions/default/proxy/notices/helper.go b/teaweb/actions/default/proxy/notices/helper.go index fb65ab0..b193a3b 100644 --- a/teaweb/actions/default/proxy/notices/helper.go +++ b/teaweb/actions/default/proxy/notices/helper.go @@ -15,5 +15,5 @@ func (this *Helper) BeforeAction(action *actions.ActionObject) { return } - proxyutils.AddServerMenu(action) + proxyutils.AddServerMenu(action, false) } diff --git a/teaweb/actions/default/proxy/proxyutils/menu.go b/teaweb/actions/default/proxy/proxyutils/menu.go index 827f0a0..54576ea 100644 --- a/teaweb/actions/default/proxy/proxyutils/menu.go +++ b/teaweb/actions/default/proxy/proxyutils/menu.go @@ -10,7 +10,7 @@ import ( ) // 添加服务器菜单 -func AddServerMenu(actionWrapper actions.ActionWrapper) { +func AddServerMenu(actionWrapper actions.ActionWrapper, isIndex bool) { action := actionWrapper.Object() // 选中代理 @@ -23,8 +23,6 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { var hasServer = false var isTCP = false - isIndex := !action.HasPrefix("/proxy/add", "/cache", "/proxy/waf", "/proxy/log/policies", "/proxy/certs", "/proxy/settings") - serverId := action.ParamString("serverId") serverList, err := teaconfigs.SharedServerList() if err != nil { @@ -113,7 +111,8 @@ func AddServerMenu(actionWrapper actions.ActionWrapper) { menu.AlwaysActive = true menuGroup.AlwaysMenu = menu menu.Add("代理服务", "", "/proxy", isIndex) - menu.Add("+添加新代理", "", "/proxy/add", action.Spec.ClassName == "proxy.AddAction", ) + menu.Add("+添加新代理", "", "/proxy/add", action.Spec.ClassName == "proxy.AddAction") + menu.Add("分组管理", "", "/proxy/servergroups", action.HasPrefix("/proxy/servergroups")) menu.Add("缓存策略", "", "/cache", action.HasPrefix("/cache")) menu.Add("WAF策略", "", "/proxy/waf", action.HasPrefix("/proxy/waf")) menu.Add("日志策略", "", "/proxy/log/policies", action.HasPrefix("/proxy/log/policies")) diff --git a/teaweb/actions/default/proxy/servergroups/add.go b/teaweb/actions/default/proxy/servergroups/add.go new file mode 100644 index 0000000..9757daa --- /dev/null +++ b/teaweb/actions/default/proxy/servergroups/add.go @@ -0,0 +1,37 @@ +package servergroups + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/iwind/TeaGo/actions" +) + +type AddAction actions.Action + +// 添加分组 +func (this *AddAction) RunGet(params struct{}) { + this.Data["selectedMenu"] = "add" + + this.Show() +} + +func (this *AddAction) RunPost(params struct { + Name string + + Must *actions.Must +}) { + params.Must. + Field("name", params.Name). + Require("请输入分组名称") + + group := teaconfigs.NewServerGroup() + group.Name = params.Name + + groupList := teaconfigs.SharedServerGroupList() + groupList.Add(group) + err := groupList.Save() + if err != nil { + this.Fail("保存失败:" + err.Error()) + } + + this.Success() +} diff --git a/teaweb/actions/default/proxy/servergroups/addServer.go b/teaweb/actions/default/proxy/servergroups/addServer.go new file mode 100644 index 0000000..963f919 --- /dev/null +++ b/teaweb/actions/default/proxy/servergroups/addServer.go @@ -0,0 +1,65 @@ +package servergroups + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/maps" + "net/http" +) + +type AddServerAction actions.Action + +func (this *AddServerAction) RunGet(params struct { + GroupId string +}) { + groupList := teaconfigs.SharedServerGroupList() + group := groupList.Find(params.GroupId) + if group == nil { + this.Error("not found", http.StatusNotFound) + return + } + + this.Data["group"] = group + + // 所有代理服务列表 + serverList, err := teaconfigs.SharedServerList() + if err != nil { + this.Error(err.Error(), http.StatusInternalServerError) + return + } + serverMaps := []maps.Map{} + for _, server := range serverList.FindAllServers() { + if groupList.ContainsServer(server.Id) { + continue + } + serverMaps = append(serverMaps, maps.Map{ + "id": server.Id, + "description": server.Description, + }) + } + this.Data["servers"] = serverMaps + + this.Show() +} + +func (this *AddServerAction) RunPost(params struct { + GroupId string + ServerIds []string + + Must *actions.Must +}) { + groupList := teaconfigs.SharedServerGroupList() + group := groupList.Find(params.GroupId) + if group == nil { + this.Error("not found", http.StatusNotFound) + return + } + + group.Add(params.ServerIds...) + err := groupList.Save() + if err != nil { + this.Fail("保存失败:" + err.Error()) + } + + this.Success() +} diff --git a/teaweb/actions/default/proxy/servergroups/delete.go b/teaweb/actions/default/proxy/servergroups/delete.go new file mode 100644 index 0000000..96e7fc3 --- /dev/null +++ b/teaweb/actions/default/proxy/servergroups/delete.go @@ -0,0 +1,22 @@ +package servergroups + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/iwind/TeaGo/actions" +) + +type DeleteAction actions.Action + +// 删除分组 +func (this *DeleteAction) RunPost(params struct { + GroupId string +}) { + groupList := teaconfigs.SharedServerGroupList() + groupList.Remove(params.GroupId) + err := groupList.Save() + if err != nil { + this.Fail("保存失败:" + err.Error()) + } + + this.Success() +} diff --git a/teaweb/actions/default/proxy/servergroups/deleteServer.go b/teaweb/actions/default/proxy/servergroups/deleteServer.go new file mode 100644 index 0000000..df1ffed --- /dev/null +++ b/teaweb/actions/default/proxy/servergroups/deleteServer.go @@ -0,0 +1,27 @@ +package servergroups + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/iwind/TeaGo/actions" +) + +type DeleteServerAction actions.Action + +func (this *DeleteServerAction) RunPost(params struct { + GroupId string + ServerId string +}) { + groupList := teaconfigs.SharedServerGroupList() + group := groupList.Find(params.GroupId) + if group == nil { + this.Fail("找不到要操作的组") + } + + group.Remove(params.ServerId) + err := groupList.Save() + if err != nil { + this.Fail("保存失败:" + err.Error()) + } + + this.Success() +} diff --git a/teaweb/actions/default/proxy/servergroups/helper.go b/teaweb/actions/default/proxy/servergroups/helper.go new file mode 100644 index 0000000..6a20b7c --- /dev/null +++ b/teaweb/actions/default/proxy/servergroups/helper.go @@ -0,0 +1,17 @@ +package servergroups + +import ( + "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" + "github.com/iwind/TeaGo/actions" + "net/http" +) + +type Helper struct { +} + +// 相关Helper +func (this *Helper) BeforeAction(action *actions.ActionObject) { + if action.Request.Method == http.MethodGet { + proxyutils.AddServerMenu(action, false) + } +} diff --git a/teaweb/actions/default/proxy/servergroups/index.go b/teaweb/actions/default/proxy/servergroups/index.go new file mode 100644 index 0000000..a33064e --- /dev/null +++ b/teaweb/actions/default/proxy/servergroups/index.go @@ -0,0 +1,75 @@ +package servergroups + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/maps" +) + +type IndexAction actions.Action + +func (this *IndexAction) RunGet(params struct{}) { + this.Data["selectedMenu"] = "index" + + groupList := teaconfigs.SharedServerGroupList() + groups := groupList.Groups + groupMaps := []maps.Map{} + isChanged := false + for _, group := range groups { + serverMaps := []maps.Map{} + for _, serverId := range group.ServerIds { + server := teaconfigs.NewServerConfigFromId(serverId) + if server == nil { + isChanged = true + continue + } + serverMaps = append(serverMaps, maps.Map{ + "id": server.Id, + "description": server.Description, + }) + } + + groupMaps = append(groupMaps, maps.Map{ + "id": group.Id, + "name": group.Name, + "isOn": group.IsOn, + "servers": serverMaps, + }) + } + + // 是否有未分组的 + serverList, err := teaconfigs.SharedServerList() + if err != nil { + logs.Error(err) + } else { + ungroupServerMaps := []maps.Map{} + for _, server := range serverList.FindAllServers() { + if !groupList.ContainsServer(server.Id) { + ungroupServerMaps = append(ungroupServerMaps, maps.Map{ + "id": server.Id, + "description": server.Description, + }) + } + } + if len(ungroupServerMaps)> 0 { + groupMaps = append(groupMaps, maps.Map{ + "id": "", + "name": "[未分组]", + "isOn": true, + "servers": ungroupServerMaps, + }) + } + } + + this.Data["groups"] = groupMaps + + if isChanged { + err := teaconfigs.SharedServerGroupList().Save() + if err != nil { + logs.Error(err) + } + } + + this.Show() +} diff --git a/teaweb/actions/default/proxy/servergroups/init.go b/teaweb/actions/default/proxy/servergroups/init.go new file mode 100644 index 0000000..83570ac --- /dev/null +++ b/teaweb/actions/default/proxy/servergroups/init.go @@ -0,0 +1,23 @@ +package servergroups + +import ( + "github.com/TeaWeb/code/teaweb/helpers" + "github.com/iwind/TeaGo" +) + +func init() { + TeaGo.BeforeStart(func(server *TeaGo.Server) { + // 注册路由 + server. + Prefix("/proxy/servergroups"). + Helper(new(helpers.UserMustAuth)). + Helper(new(Helper)). + Get("", new(IndexAction)). + GetPost("/add", new(AddAction)). + GetPost("/update", new(UpdateAction)). + Post("/delete", new(DeleteAction)). + GetPost("/addServer", new(AddServerAction)). + Post("/deleteServer", new(DeleteServerAction)). + EndAll() + }) +} diff --git a/teaweb/actions/default/proxy/servergroups/update.go b/teaweb/actions/default/proxy/servergroups/update.go new file mode 100644 index 0000000..5019dfa --- /dev/null +++ b/teaweb/actions/default/proxy/servergroups/update.go @@ -0,0 +1,47 @@ +package servergroups + +import ( + "github.com/TeaWeb/code/teaconfigs" + "github.com/iwind/TeaGo/actions" + "net/http" +) + +type UpdateAction actions.Action + +func (this *UpdateAction) RunGet(params struct { + GroupId string +}) { + group := teaconfigs.SharedServerGroupList().Find(params.GroupId) + if group == nil { + this.Error("not found", http.StatusNotFound) + return + } + + this.Data["group"] = group + + this.Show() +} + +func (this *UpdateAction) RunPost(params struct { + GroupId string + Name string + + Must *actions.Must +}) { + params.Must. + Field("name", params.Name). + Require("请输入分组名称") + + groupList := teaconfigs.SharedServerGroupList() + group := groupList.Find(params.GroupId) + if group == nil { + this.Fail("找不到要修改的分组") + } + group.Name = params.Name + err := groupList.Save() + if err != nil { + this.Fail("保存失败:" + err.Error()) + } + + this.Success() +} diff --git a/teaweb/actions/default/proxy/servers/waf.go b/teaweb/actions/default/proxy/servers/waf.go index f965d1d..dd60142 100644 --- a/teaweb/actions/default/proxy/servers/waf.go +++ b/teaweb/actions/default/proxy/servers/waf.go @@ -18,7 +18,7 @@ func (this *WafAction) RunGet(params struct { this.Fail("找不到Server") } - proxyutils.AddServerMenu(this) + proxyutils.AddServerMenu(this, true) wafList := []maps.Map{} for _, waf := range teaconfigs.SharedWAFList().FindAllConfigs() { diff --git a/teaweb/actions/default/proxy/settings/helper.go b/teaweb/actions/default/proxy/settings/helper.go index 378b519..523a23a 100644 --- a/teaweb/actions/default/proxy/settings/helper.go +++ b/teaweb/actions/default/proxy/settings/helper.go @@ -1,6 +1,7 @@ package settings import ( + "github.com/TeaWeb/code/teaweb/actions/default/proxy/proxyutils" "github.com/iwind/TeaGo/actions" "net/http" ) @@ -12,4 +13,5 @@ func (this *Helper) BeforeAction(action *actions.ActionObject) { if action.Request.Method != http.MethodGet { return } + proxyutils.AddServerMenu(action, false) } diff --git a/teaweb/actions/default/proxy/settings/init.go b/teaweb/actions/default/proxy/settings/init.go index 28c56d4..ec111e3 100644 --- a/teaweb/actions/default/proxy/settings/init.go +++ b/teaweb/actions/default/proxy/settings/init.go @@ -1,7 +1,6 @@ package settings import ( - "github.com/TeaWeb/code/teaweb/actions/default/proxy" "github.com/TeaWeb/code/teaweb/configs" "github.com/TeaWeb/code/teaweb/helpers" "github.com/iwind/TeaGo" @@ -15,7 +14,6 @@ func init() { Grant: configs.AdminGrantProxy, }). Helper(new(Helper)). - Helper(new(proxy.Helper)). Get("", new(IndexAction)). GetPost("/update", new(UpdateAction)). EndAll() diff --git a/teaweb/actions/default/proxy/tunnel/index.go b/teaweb/actions/default/proxy/tunnel/index.go index b782538..e786f06 100644 --- a/teaweb/actions/default/proxy/tunnel/index.go +++ b/teaweb/actions/default/proxy/tunnel/index.go @@ -18,7 +18,7 @@ func (this *IndexAction) RunGet(params struct { this.Fail("找不到Server") } - proxyutils.AddServerMenu(this) + proxyutils.AddServerMenu(this, true) this.Data["selectedTab"] = "tunnel" this.Data["server"] = proxyutils.WrapServerData(server) diff --git a/teaweb/actions/default/proxy/tunnel/update.go b/teaweb/actions/default/proxy/tunnel/update.go index 9f84ef4..4856ab2 100644 --- a/teaweb/actions/default/proxy/tunnel/update.go +++ b/teaweb/actions/default/proxy/tunnel/update.go @@ -18,7 +18,7 @@ func (this *UpdateAction) RunGet(params struct { this.Fail("找不到Server") } - proxyutils.AddServerMenu(this) + proxyutils.AddServerMenu(this, true) this.Data["selectedTab"] = "tunnel" this.Data["server"] = proxyutils.WrapServerData(server) diff --git a/teaweb/actions/default/proxy/waf/helper.go b/teaweb/actions/default/proxy/waf/helper.go index 4ce7623..d62b8ee 100644 --- a/teaweb/actions/default/proxy/waf/helper.go +++ b/teaweb/actions/default/proxy/waf/helper.go @@ -12,7 +12,7 @@ type Helper struct { // 相关Helper func (this *Helper) BeforeAction(action *actions.ActionObject) { if action.Request.Method == http.MethodGet { - proxyutils.AddServerMenu(action) + proxyutils.AddServerMenu(action, false) action.Data["selectedMenu"] = "list" if action.HasPrefix("/proxy/waf/add") { From 4bb8ab66a55d89ee2a93ffa6e744b58967b39954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月28日 22:02:27 +0800 Subject: [PATCH 105/121] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=B7=A6=E4=BE=A7?= =?UTF-8?q?=E8=8F=9C=E5=8D=95=E5=88=86=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/server_group.go | 4 + teaweb/actions/default/proxy/index.go | 20 ++- .../actions/default/proxy/proxyutils/menu.go | 152 +++++++++++------- 3 files changed, 113 insertions(+), 63 deletions(-) diff --git a/teaconfigs/server_group.go b/teaconfigs/server_group.go index 62d485f..4e54bdb 100644 --- a/teaconfigs/server_group.go +++ b/teaconfigs/server_group.go @@ -39,3 +39,7 @@ func (this *ServerGroup) Remove(serverId string) { } this.ServerIds = result } + +func (this *ServerGroup) Contains(serverId string) bool { + return lists.ContainsString(this.ServerIds, serverId) +} diff --git a/teaweb/actions/default/proxy/index.go b/teaweb/actions/default/proxy/index.go index 5c4b925..1b398c7 100644 --- a/teaweb/actions/default/proxy/index.go +++ b/teaweb/actions/default/proxy/index.go @@ -11,7 +11,25 @@ type IndexAction actions.Action // 代理首页 func (this *IndexAction) Run(params struct { }) { - // 跳转到第一个 + // 跳转到分组的第一个 + groupList := teaconfigs.SharedServerGroupList() + serverId := "" + for _, group := range groupList.Groups { + if !group.IsOn { + continue + } + if len(group.ServerIds) == 0 { + continue + } + serverId = group.ServerIds[0] + break + } + if len(serverId)> 0 { + this.RedirectURL("/proxy/board?serverId=" + serverId) + return + } + + // 没有分组,就跳到所有的服务的第一个 serverList, err := teaconfigs.SharedServerList() if err != nil { logs.Error(err) diff --git a/teaweb/actions/default/proxy/proxyutils/menu.go b/teaweb/actions/default/proxy/proxyutils/menu.go index 54576ea..f884b05 100644 --- a/teaweb/actions/default/proxy/proxyutils/menu.go +++ b/teaweb/actions/default/proxy/proxyutils/menu.go @@ -20,87 +20,54 @@ func AddServerMenu(actionWrapper actions.ActionWrapper, isIndex bool) { menuGroup := utils.NewMenuGroup() // 服务 - var hasServer = false var isTCP = false serverId := action.ParamString("serverId") serverList, err := teaconfigs.SharedServerList() if err != nil { logs.Error(err) + return } + var hasServer = len(serverList.Files)> 0 if isIndex { - menu := menuGroup.FindMenu("", "") - for _, server := range serverList.FindAllServers() { - urlPrefix := "/proxy/board" - if server.IsTCP() { // TCP - urlPrefix = "/proxy/detail" - } else { // HTTP - if action.HasPrefix("/proxy/stat") { - urlPrefix = "/proxy/stat" - } else if action.HasPrefix("/proxy/log") { - urlPrefix = "/proxy/log" - } else if action.HasPrefix("/proxy") && !action.HasPrefix("/proxy/board", "/proxy/add") { - urlPrefix = "/proxy/detail" - } + allServers := serverList.FindAllServers() + for _, server := range allServers { + if server.Id == serverId { + isTCP = server.IsTCP() } + } - item := menu.Add(server.Description, "", urlPrefix+"?serverId="+server.Id, serverId == server.Id) - item.IsSortable = true + // 分组列表 + groupList := teaconfigs.SharedServerGroupList() - if server.IsTCP() { - item.SupName = "tcp" - } else if server.ForwardHTTP != nil { - item.SupName = "forward" + // 已分组 + for _, group := range groupList.Groups { + if len(group.ServerIds) == 0 { + continue } - // port - ports := []string{} - if server.Http { - for _, listen := range server.Listen { - index := strings.LastIndex(listen, ":") - if index> -1 { - ports = append(ports, ":"+listen[index+1:]) - } else { - ports = append(ports, ":"+listen) - } - } - } - if server.SSL != nil && server.SSL.On { - for _, listen := range server.SSL.Listen { - index := strings.LastIndex(listen, ":") - if index> -1 { - ports = append(ports, ":"+listen[index+1:]) - } else { - ports = append(ports, ":"+listen) - } - } - } - if len(ports)> 0 { - if len(ports)> 2 { - item.SubName = ports[0] + ", " + ports[1] + "等 " - } else { - item.SubName = strings.Join(ports, ", ") + " " - } - } + menu := menuGroup.FindMenu("group_"+group.Id, group.Name) - // on | off - if (server.IsHTTP() && !server.Http && (server.SSL == nil || !server.SSL.On)) || (server.IsTCP() && (server.TCP == nil || !server.TCP.TCPOn)) { - item.SubName = "未启用" - item.SubColor = "red" + for _, server := range allServers { + if !group.Contains(server.Id) { + continue + } + AddServerToMenu(server, action, menu, serverId) } + } - if server.Id == serverId { - isTCP = server.IsTCP() + // 是否有未分组 + unGroupServers := []*teaconfigs.ServerConfig{} + for _, server := range allServers { + if !groupList.ContainsServer(server.Id) { + unGroupServers = append(unGroupServers, server) } - - hasServer = true } - if hasServer { - if action.Request.URL.Path == "/proxy/board" { - menu.Name = "代理服务" - } else { - menu.Name = "代理服务" + if len(unGroupServers)> 0 { + menu := menuGroup.FindMenu("", "[未分组]") + for _, server := range unGroupServers { + AddServerToMenu(server, action, menu, serverId) } } } @@ -203,6 +170,67 @@ func AddServerMenu(actionWrapper actions.ActionWrapper, isIndex bool) { } } +// 将Server添加到菜单上 +func AddServerToMenu(server *teaconfigs.ServerConfig, action *actions.ActionObject, menu *utils.Menu, serverId string) { + urlPrefix := "/proxy/board" + if server.IsTCP() { // TCP + urlPrefix = "/proxy/detail" + } else { // HTTP + if action.HasPrefix("/proxy/stat") { + urlPrefix = "/proxy/stat" + } else if action.HasPrefix("/proxy/log") { + urlPrefix = "/proxy/log" + } else if action.HasPrefix("/proxy") && !action.HasPrefix("/proxy/board", "/proxy/add") { + urlPrefix = "/proxy/detail" + } + } + + item := menu.Add(server.Description, "", urlPrefix+"?serverId="+server.Id, serverId == server.Id) + item.IsSortable = true + + if server.IsTCP() { + item.SupName = "tcp" + } else if server.ForwardHTTP != nil { + item.SupName = "forward" + } + + // port + ports := []string{} + if server.Http { + for _, listen := range server.Listen { + index := strings.LastIndex(listen, ":") + if index> -1 { + ports = append(ports, ":"+listen[index+1:]) + } else { + ports = append(ports, ":"+listen) + } + } + } + if server.SSL != nil && server.SSL.On { + for _, listen := range server.SSL.Listen { + index := strings.LastIndex(listen, ":") + if index> -1 { + ports = append(ports, ":"+listen[index+1:]) + } else { + ports = append(ports, ":"+listen) + } + } + } + if len(ports)> 0 { + if len(ports)> 2 { + item.SubName = ports[0] + ", " + ports[1] + "等 " + } else { + item.SubName = strings.Join(ports, ", ") + " " + } + } + + // on | off + if (server.IsHTTP() && !server.Http && (server.SSL == nil || !server.SSL.On)) || (server.IsTCP() && (server.TCP == nil || !server.TCP.TCPOn)) { + item.SubName = "未启用" + item.SubColor = "red" + } +} + // 包装Server相关数据 func WrapServerData(server *teaconfigs.ServerConfig) maps.Map { return maps.Map{ From 2874c7b7b15497f2c26de1506c7750cf2ec80143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月29日 14:54:00 +0800 Subject: [PATCH 106/121] =?UTF-8?q?[proxy]=E8=AF=B7=E6=B1=82=E3=80=81?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=A2=9E=E5=8A=A0=E8=8B=A5=E5=B9=B2=E4=B8=AA?= =?UTF-8?q?=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tealogs/accesslogs/README.md | 3 +++ tealogs/accesslogs/access_log.go | 28 ++++++++++++++++++++++- tealogs/accesslogs/access_log_easyjson.go | 28 +++++++++++++++++++++++ tealogs/accesslogs/access_log_test.go | 15 ++++++++++++ teaproxy/request.go | 22 ++++++++++++++++++ teawaf/checkpoints/cc.go | 2 +- 6 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 tealogs/accesslogs/README.md diff --git a/tealogs/accesslogs/README.md b/tealogs/accesslogs/README.md new file mode 100644 index 0000000..6676d71 --- /dev/null +++ b/tealogs/accesslogs/README.md @@ -0,0 +1,3 @@ +~~~bash +easyjson -all access_log.go +~~~ \ No newline at end of file diff --git a/tealogs/accesslogs/access_log.go b/tealogs/accesslogs/access_log.go index 9389911..6aa0732 100644 --- a/tealogs/accesslogs/access_log.go +++ b/tealogs/accesslogs/access_log.go @@ -34,7 +34,8 @@ type AccessLog struct { RewriteId string `var:"rewriteId" bson:"rewriteId" json:"rewriteId"` // 重写规则ID TeaVersion string `var:"teaVersion" bson:"teaVersion" json:"teaVersion"` // TeaWeb版本 - RemoteAddr string `var:"remoteAddr" bson:"remoteAddr" json:"remoteAddr"` // 终端地址,通常是:ip:port + RemoteAddr string `var:"remoteAddr" bson:"remoteAddr" json:"remoteAddr"` // 终端地址 + RawRemoteAddr string `var:"rawRemoteAddr" bson:"rawRemoteAddr" json:"rawRemoteAddr"` // 直接连接服务器的终端地址 RemotePort int `var:"remotePort" bson:"remotePort" json:"remotePort"` // 终端端口 RemoteUser string `var:"remoteUser" bson:"remoteUser" json:"remoteUser"` // 终端用户,基于BasicAuth认证 RequestURI string `var:"requestURI" bson:"requestURI" json:"requestURI"` // 请求URI @@ -71,9 +72,12 @@ type AccessLog struct { ServerName string `var:"serverName" bson:"serverName" json:"serverName"` // 接收请求的服务器名 ServerPort int `var:"serverPort" bson:"serverPort" json:"serverPort"` // 服务器端口 ServerProtocol string `var:"serverProtocol" bson:"serverProtocol" json:"serverProtocol"` // 服务器协议,类似于HTTP/1.0" + Hostname string `var:"hostname" bson:"hostname" json:"hostname"` // 代理相关 BackendAddress string `var:"backendAddress" bson:"backendAddress" json:"backendAddress"` // 代理的后端的地址 + BackendCode string `var:"backendCode" bson:"backendCode" json:"backendCode"` // 后端代号 + BackendScheme string `var:"backendScheme" bson:"backendScheme" json:"backendScheme"` // 后端协议 FastcgiAddress string `var:"fastcgiAddress" bson:"fastcgiAddress" json:"fastcgiAddress"` // Fastcgi后端地址 // 调试用 @@ -269,6 +273,28 @@ func (this *AccessLog) Format(format string) string { } } + // backend + if strings.HasPrefix(varName, "backend.") { + subVarName := varName[8:] + switch subVarName { + case "id": + return this.BackendId + case "address": + return this.BackendAddress + case "host": + index := strings.Index(this.BackendAddress, ":") + if index> -1 { + return this.BackendAddress[:index] + } else { + return "" + } + case "scheme": + return this.BackendScheme + case "code": + return this.BackendCode + } + } + return "${" + varName + "}" }) diff --git a/tealogs/accesslogs/access_log_easyjson.go b/tealogs/accesslogs/access_log_easyjson.go index df6e2ca..83c8bba 100644 --- a/tealogs/accesslogs/access_log_easyjson.go +++ b/tealogs/accesslogs/access_log_easyjson.go @@ -736,6 +736,8 @@ func easyjsonF44c504dDecodeGithubComTeaWebCodeTealogsAccesslogs8(in *jlexer.Lexe out.TeaVersion = string(in.String()) case "remoteAddr": out.RemoteAddr = string(in.String()) + case "rawRemoteAddr": + out.RawRemoteAddr = string(in.String()) case "remotePort": out.RemotePort = int(in.Int()) case "remoteUser": @@ -939,8 +941,14 @@ func easyjsonF44c504dDecodeGithubComTeaWebCodeTealogsAccesslogs8(in *jlexer.Lexe out.ServerPort = int(in.Int()) case "serverProtocol": out.ServerProtocol = string(in.String()) + case "hostname": + out.Hostname = string(in.String()) case "backendAddress": out.BackendAddress = string(in.String()) + case "backendCode": + out.BackendCode = string(in.String()) + case "backendScheme": + out.BackendScheme = string(in.String()) case "fastcgiAddress": out.FastcgiAddress = string(in.String()) case "requestData": @@ -1098,6 +1106,11 @@ func easyjsonF44c504dEncodeGithubComTeaWebCodeTealogsAccesslogs8(out *jwriter.Wr out.RawString(prefix) out.String(string(in.RemoteAddr)) } + { + const prefix string = ",\"rawRemoteAddr\":" + out.RawString(prefix) + out.String(string(in.RawRemoteAddr)) + } { const prefix string = ",\"remotePort\":" out.RawString(prefix) @@ -1365,11 +1378,26 @@ func easyjsonF44c504dEncodeGithubComTeaWebCodeTealogsAccesslogs8(out *jwriter.Wr out.RawString(prefix) out.String(string(in.ServerProtocol)) } + { + const prefix string = ",\"hostname\":" + out.RawString(prefix) + out.String(string(in.Hostname)) + } { const prefix string = ",\"backendAddress\":" out.RawString(prefix) out.String(string(in.BackendAddress)) } + { + const prefix string = ",\"backendCode\":" + out.RawString(prefix) + out.String(string(in.BackendCode)) + } + { + const prefix string = ",\"backendScheme\":" + out.RawString(prefix) + out.String(string(in.BackendScheme)) + } { const prefix string = ",\"fastcgiAddress\":" out.RawString(prefix) diff --git a/tealogs/accesslogs/access_log_test.go b/tealogs/accesslogs/access_log_test.go index 05327e5..a129dc9 100644 --- a/tealogs/accesslogs/access_log_test.go +++ b/tealogs/accesslogs/access_log_test.go @@ -331,6 +331,21 @@ func TestAccessLog_JSON(t *testing.T) { t.Log(string(data)) } +func TestAccessLog_Backend(t *testing.T) { + accessLog := NewAccessLog() + accessLog.BackendId = "1234" + accessLog.BackendAddress = "127.0.0.1:8001" + accessLog.BackendCode = "web001" + accessLog.BackendScheme = "http" + accessLog.Hostname = "Web001" + t.Log(accessLog.Format("${backend.address} ${backend.host} ${backend.id} ${backend.code} ${backend.scheme} ${hostname}")) + data, err := easyjson.Marshal(accessLog) + if err != nil { + t.Fatal(err) + } + t.Log(string(data)) +} + func TestAccessLog_JSON_compare(t *testing.T) { accessLog := &AccessLog{ RequestPath: "/hello", diff --git a/teaproxy/request.go b/teaproxy/request.go index 29d91f9..b45de60 100644 --- a/teaproxy/request.go +++ b/teaproxy/request.go @@ -54,6 +54,9 @@ var bytePool1k = teautils.NewBytePool(20480, 1024) var bytePool32k = teautils.NewBytePool(20480, 32*1024) var bytePool128k = teautils.NewBytePool(20480, 128*1024) +// 环境变量 +var HOSTNAME, _ = os.Hostname() + // 请求定义 // HTTP HEADER RFC: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html type Request struct { @@ -1270,6 +1273,8 @@ func (this *Request) Format(source string) string { return this.serverName case "serverPort": return strconv.Itoa(this.requestServerPort()) + case "hostname": + return HOSTNAME case "documentRoot": if this.locationContext != nil && len(this.locationContext.Root)> 0 { return this.locationContext.Root @@ -1305,6 +1310,13 @@ func (this *Request) Format(source string) string { switch suffix { case "address": return this.backend.Address + case "host": + index := strings.Index(this.backend.Address, ":") + if index> -1 { + return this.backend.Address[:index] + } else { + return "" + } case "id": return this.backend.Id case "scheme": @@ -1432,9 +1444,16 @@ func (this *Request) log() { cookies[cookie.Name] = cookie.Value } + addr := this.raw.RemoteAddr + host, _, err := net.SplitHostPort(addr) + if err == nil { + addr = host + } + accessLog := &accesslogs.AccessLog{ TeaVersion: teaconst.TeaVersion, RemoteAddr: this.requestRemoteAddr(), + RawRemoteAddr: addr, RemotePort: this.requestRemotePort(), RemoteUser: this.requestRemoteUser(), RequestURI: this.requestURI(), @@ -1469,6 +1488,7 @@ func (this *Request) log() { HasErrors: len(this.errors)> 0, Extend: &accesslogs.AccessLogExtend{}, Attrs: this.attrs, + Hostname: HOSTNAME, } // 日志和统计 @@ -1501,6 +1521,8 @@ func (this *Request) log() { if this.backend != nil { accessLog.BackendAddress = this.backend.Address accessLog.BackendId = this.backend.Id + accessLog.BackendScheme = this.backend.Scheme + accessLog.BackendCode = this.backend.Code } if this.fastcgi != nil { diff --git a/teawaf/checkpoints/cc.go b/teawaf/checkpoints/cc.go index d11d306..4cd9e3f 100644 --- a/teawaf/checkpoints/cc.go +++ b/teawaf/checkpoints/cc.go @@ -28,7 +28,7 @@ func (this *CCCheckpoint) Start() { if this.grid != nil { this.grid.Destroy() } - this.grid = teamemory.NewGrid(100) + this.grid = teamemory.NewGrid(32) } func (this *CCCheckpoint) RequestValue(req *requests.Request, param string, options map[string]string) (value interface{}, sysErr error, userErr error) { From 41248bec10828eb5b1c947125ab08ec17f999c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月29日 23:34:47 +0800 Subject: [PATCH 107/121] =?UTF-8?q?=E9=99=90=E5=88=B6=E5=90=84=E4=B8=AA?= =?UTF-8?q?=E5=86=85=E5=AD=98=E7=BC=93=E5=AD=98=E7=9A=84=E6=9D=A1=E7=9B=AE?= =?UTF-8?q?=E6=95=B0=E9=87=8F=EF=BC=8C=E9=98=B2=E6=AD=A2OOM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teageo/utils.go | 2 +- tealogs/accesslogs/access_log.go | 2 +- teamemory/cell.go | 9 ++++++++- teamemory/grid.go | 7 ++++++- teamemory/grid_test.go | 20 +++++++++++++------ teamemory/item.go | 9 +++++++-- teamemory/opt_limit_count.go | 11 ++++++++++ teawaf/checkpoints/cc.go | 2 +- teawaf/utils/utils.go | 10 +++++----- .../default/agents/board/scripts/engine.go | 2 +- 10 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 teamemory/opt_limit_count.go diff --git a/teageo/utils.go b/teageo/utils.go index 4887d2e..4d72031 100644 --- a/teageo/utils.go +++ b/teageo/utils.go @@ -9,7 +9,7 @@ import ( "strings" ) -var ip2cityGrid = teamemory.NewGrid(32) +var ip2cityGrid = teamemory.NewGrid(32, teamemory.NewLimitCountOpt(100_0000)) // 加载Geo-City数据库 func SetupDB() { diff --git a/tealogs/accesslogs/access_log.go b/tealogs/accesslogs/access_log.go index 6aa0732..9a39654 100644 --- a/tealogs/accesslogs/access_log.go +++ b/tealogs/accesslogs/access_log.go @@ -20,7 +20,7 @@ import ( "time" ) -var userAgentGrid = teamemory.NewGrid(32) +var userAgentGrid = teamemory.NewGrid(32, teamemory.NewLimitCountOpt(100_0000)) var charsetReg = regexp.MustCompile("(?i)charset\\s*=\\s*([\\w-]+)") var headerReg = regexp.MustCompile("([A-Z])") diff --git a/teamemory/cell.go b/teamemory/cell.go index 536e9e2..ce16539 100644 --- a/teamemory/cell.go +++ b/teamemory/cell.go @@ -7,7 +7,8 @@ import ( ) type Cell struct { - LimitSize int64 + LimitSize int64 + LimitCount int mapping map[uint64]*Item // key => item list *List // { item1, item2, ... } @@ -37,6 +38,12 @@ func (this *Cell) Write(hashKey uint64, item *Item) { } } + // limit count + if this.LimitCount> 0 && len(this.mapping)>= this.LimitCount { + this.locker.Unlock() + return + } + // trim memory size := item.Size() shouldTrim := false diff --git a/teamemory/grid.go b/teamemory/grid.go index cd8eef7..93146d9 100644 --- a/teamemory/grid.go +++ b/teamemory/grid.go @@ -24,7 +24,8 @@ type Grid struct { gzipLevel int - limitSize int64 + limitSize int64 + limitCount int } func NewGrid(countCells int, opt ...interface{}) *Grid { @@ -38,6 +39,8 @@ func NewGrid(countCells int, opt ...interface{}) *Grid { grid.gzipLevel = x.Level case *LimitSizeOpt: grid.limitSize = x.Size + case *LimitCountOpt: + grid.limitCount = x.Count case *RecycleIntervalOpt: grid.recycleInterval = x.Interval } @@ -52,6 +55,8 @@ func NewGrid(countCells int, opt ...interface{}) *Grid { for i := 0; i < countCells; i++ { cell := NewCell() cell.LimitSize = int64(math.Floor(float64(grid.limitSize) / float64(countCells))) + cell.LimitCount = int(math.Floor(float64(grid.limitCount)) / float64(countCells)) + cells = append(cells, cell) } grid.cells = cells diff --git a/teamemory/grid_test.go b/teamemory/grid_test.go index af50e16..c003442 100644 --- a/teamemory/grid_test.go +++ b/teamemory/grid_test.go @@ -27,7 +27,7 @@ func TestMemoryGrid_Write(t *testing.T) { grid.Delete([]byte("abc")) t.Log(grid.Read([]byte("abc"))) - for i := 0; i < 100; i ++ { + for i := 0; i < 100; i++ { grid.WriteInt64([]byte(fmt.Sprintf("%d", i)), 123, 1) } @@ -45,6 +45,14 @@ func TestMemoryGrid_Write(t *testing.T) { grid.Destroy() } +func TestMemoryGrid_Write_LimitCount(t *testing.T) { + grid := NewGrid(2, NewLimitCountOpt(10)) + for i := 0; i < 100; i++ { + grid.WriteInt64([]byte(strconv.Itoa(i)), int64(i), 30) + } + t.Log(grid.Stat().CountItems, "items") +} + func TestMemoryGrid_Compress(t *testing.T) { grid := NewGrid(5, NewCompressOpt(1)) grid.WriteString([]byte("hello"), strings.Repeat("abcd", 10240), 30) @@ -54,7 +62,7 @@ func TestMemoryGrid_Compress(t *testing.T) { func BenchmarkMemoryGrid_Performance(b *testing.B) { grid := NewGrid(1024) - for i := 0; i < b.N; i ++ { + for i := 0; i < b.N; i++ { grid.WriteInt64([]byte("key:"+strconv.Itoa(i)), int64(i), 3600) } } @@ -68,7 +76,7 @@ func TestMemoryGrid_Performance(t *testing.T) { s := []byte(strings.Repeat("abcd", 10*1024)) - for i := 0; i < 100000; i ++ { + for i := 0; i < 100000; i++ { grid.WriteBytes([]byte(fmt.Sprintf("key:%d_%d", i, 1)), s, 3600) item := grid.Read([]byte(fmt.Sprintf("key:%d_%d", i, 1))) if item != nil { @@ -97,10 +105,10 @@ func TestMemoryGrid_Performance_Concurrent(t *testing.T) { wg := sync.WaitGroup{} wg.Add(runtime.NumCPU()) - for c := 0; c < runtime.NumCPU(); c ++ { + for c := 0; c < runtime.NumCPU(); c++ { go func(c int) { defer wg.Done() - for i := 0; i < 50000; i ++ { + for i := 0; i < 50000; i++ { grid.WriteBytes([]byte(fmt.Sprintf("key:%d_%d", i, c)), s, 3600) item := grid.Read([]byte(fmt.Sprintf("key:%d_%d", i, c))) if item != nil { @@ -128,7 +136,7 @@ func TestMemoryGrid_CompressPerformance(t *testing.T) { now := time.Now() data := []byte(strings.Repeat("abcd", 1024)) - for i := 0; i < 100000; i ++ { + for i := 0; i < 100000; i++ { grid.WriteBytes([]byte(fmt.Sprintf("key:%d", i)), data, 3600) item := grid.Read([]byte(fmt.Sprintf("key:%d", i+100))) if item != nil { diff --git a/teamemory/item.go b/teamemory/item.go index 77dd9ef..d30bb89 100644 --- a/teamemory/item.go +++ b/teamemory/item.go @@ -9,7 +9,7 @@ import ( "unsafe" ) -type ItemType = int +type ItemType = int8 const ( ItemInt64 = 1 @@ -33,6 +33,8 @@ type Item struct { // linked list Prev *Item Next *Item + + size int64 } func NewItem(key []byte, dataType ItemType) *Item { @@ -79,5 +81,8 @@ func (this *Item) String() string { } func (this *Item) Size() int64 { - return int64(int(unsafe.Sizeof(*this)) + len(this.Key) + len(this.ValueBytes)) + if this.size == 0 { + this.size = int64(int(unsafe.Sizeof(*this)) + len(this.Key) + len(this.ValueBytes)) + } + return this.size } diff --git a/teamemory/opt_limit_count.go b/teamemory/opt_limit_count.go new file mode 100644 index 0000000..b2b8f25 --- /dev/null +++ b/teamemory/opt_limit_count.go @@ -0,0 +1,11 @@ +package teamemory + +type LimitCountOpt struct { + Count int +} + +func NewLimitCountOpt(count int) *LimitCountOpt { + return &LimitCountOpt{ + Count: count, + } +} diff --git a/teawaf/checkpoints/cc.go b/teawaf/checkpoints/cc.go index 4cd9e3f..d53f3ed 100644 --- a/teawaf/checkpoints/cc.go +++ b/teawaf/checkpoints/cc.go @@ -28,7 +28,7 @@ func (this *CCCheckpoint) Start() { if this.grid != nil { this.grid.Destroy() } - this.grid = teamemory.NewGrid(32) + this.grid = teamemory.NewGrid(32, teamemory.NewLimitCountOpt(1000_0000)) } func (this *CCCheckpoint) RequestValue(req *requests.Request, param string, options map[string]string) (value interface{}, sysErr error, userErr error) { diff --git a/teawaf/utils/utils.go b/teawaf/utils/utils.go index 888bbbf..21eaab7 100644 --- a/teawaf/utils/utils.go +++ b/teawaf/utils/utils.go @@ -9,7 +9,7 @@ import ( "strconv" ) -var grid = teamemory.NewGrid(32) +var grid = teamemory.NewGrid(32, teamemory.NewLimitCountOpt(1000_0000)) // 正则表达式匹配字符串,并缓存结果 func MatchStringCache(regex *regexp.Regexp, s string) bool { @@ -21,9 +21,9 @@ func MatchStringCache(regex *regexp.Regexp, s string) bool { } b := regex.MatchString(s) if b { - grid.WriteInt64(key, 1, 3600) + grid.WriteInt64(key, 1, 1800) } else { - grid.WriteInt64(key, 0, 3600) + grid.WriteInt64(key, 0, 1800) } return b } @@ -38,9 +38,9 @@ func MatchBytesCache(regex *regexp.Regexp, byteSlice []byte) bool { } b := regex.Match(byteSlice) if b { - grid.WriteInt64(key, 1, 3600) + grid.WriteInt64(key, 1, 1800) } else { - grid.WriteInt64(key, 0, 3600) + grid.WriteInt64(key, 0, 1800) } return b } diff --git a/teaweb/actions/default/agents/board/scripts/engine.go b/teaweb/actions/default/agents/board/scripts/engine.go index 4a118ae..8805ae3 100644 --- a/teaweb/actions/default/agents/board/scripts/engine.go +++ b/teaweb/actions/default/agents/board/scripts/engine.go @@ -22,7 +22,7 @@ import ( "time" ) -var engineCache = teamemory.NewGrid(1) +var engineCache = teamemory.NewGrid(1, teamemory.NewLimitCountOpt(1000)) var dayReg = regexp.MustCompile(`^(\d+)-(\d+)-(\d+)$`) // 脚本引擎 From 72309a6dcf82840322613d1c870295c8ac7811cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年5月29日 23:49:43 +0800 Subject: [PATCH 108/121] =?UTF-8?q?[waf]=E6=94=B9=E8=BF=9B=E6=AD=A3?= =?UTF-8?q?=E5=88=99=E8=A1=A8=E8=BE=BE=E5=BC=8F=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teawaf/utils/utils.go | 10 ++++++++++ teawaf/utils/utils_test.go | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/teawaf/utils/utils.go b/teawaf/utils/utils.go index 21eaab7..f0f6b65 100644 --- a/teawaf/utils/utils.go +++ b/teawaf/utils/utils.go @@ -13,6 +13,11 @@ var grid = teamemory.NewGrid(32, teamemory.NewLimitCountOpt(1000_0000)) // 正则表达式匹配字符串,并缓存结果 func MatchStringCache(regex *regexp.Regexp, s string) bool { + // 如果长度超过4096,大概率是不能重用的 + if len(s)> 4096 { + return regex.MatchString(s) + } + hash := siphash.Hash(0, 0, teautils.UnsafeStringToBytes(s)) key := []byte(fmt.Sprintf("%p_", regex) + strconv.FormatUint(hash, 10)) item := grid.Read(key) @@ -30,6 +35,11 @@ func MatchStringCache(regex *regexp.Regexp, s string) bool { // 正则表达式匹配字节slice,并缓存结果 func MatchBytesCache(regex *regexp.Regexp, byteSlice []byte) bool { + // 如果长度超过4096,大概率是不能重用的 + if len(byteSlice)> 4096 { + return regex.Match(byteSlice) + } + hash := siphash.Hash(0, 0, byteSlice) key := []byte(fmt.Sprintf("%p_", regex) + strconv.FormatUint(hash, 10)) item := grid.Read(key) diff --git a/teawaf/utils/utils_test.go b/teawaf/utils/utils_test.go index 90dffbb..26856f3 100644 --- a/teawaf/utils/utils_test.go +++ b/teawaf/utils/utils_test.go @@ -21,7 +21,7 @@ func TestMatchBytesCache(t *testing.T) { } func BenchmarkMatchStringCache(b *testing.B) { - data := strings.Repeat("HELLO", 1024*4) + data := strings.Repeat("HELLO", 512) regex := regexp.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`) for i := 0; i < b.N; i++ { @@ -30,7 +30,7 @@ func BenchmarkMatchStringCache(b *testing.B) { } func BenchmarkMatchStringCache_WithoutCache(b *testing.B) { - data := strings.Repeat("HELLO", 1024*4) + data := strings.Repeat("HELLO", 512) regex := regexp.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`) for i := 0; i < b.N; i++ { From db7764411b67eaed8574909851175b5bd12fe131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Tue, 2 Jun 2020 15:19:05 +0800 Subject: [PATCH 109/121] =?UTF-8?q?[proxy]=E6=97=A5=E5=BF=97=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E5=8C=96=E6=97=B6float=E7=9B=B4=E6=8E=A5=E8=BD=AC?= =?UTF-8?q?=E6=8D=A2=E4=B8=BA=E5=B8=A6=E7=82=B9=E7=9A=84=E5=B0=8F=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tealogs/accesslogs/access_log.go | 10 +++++++--- tealogs/accesslogs/access_log_test.go | 7 +++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tealogs/accesslogs/access_log.go b/tealogs/accesslogs/access_log.go index 9a39654..4934a03 100644 --- a/tealogs/accesslogs/access_log.go +++ b/tealogs/accesslogs/access_log.go @@ -183,10 +183,14 @@ func (this *AccessLog) Format(format string) string { if found { field := refValue.FieldByName(fieldName) if field.IsValid() { - if field.Kind() == reflect.String { + kind := field.Kind() + switch kind { + case reflect.String: return field.String() - } else { - return fmt.Sprintf("%#v", field.Interface()) + case reflect.Float32, reflect.Float64: + return fmt.Sprintf("%f", field.Interface()) + default: + return types.String(field.Interface()) } } diff --git a/tealogs/accesslogs/access_log_test.go b/tealogs/accesslogs/access_log_test.go index a129dc9..4b1ef69 100644 --- a/tealogs/accesslogs/access_log_test.go +++ b/tealogs/accesslogs/access_log_test.go @@ -280,6 +280,13 @@ func TestAccessLog_Format(t *testing.T) { t.Log(accessLog.Format(format)) } +func TestAccessLog_FormatFloat(t *testing.T) { + accessLog := &AccessLog{ + RequestTime: 7.964e-5, + } + t.Log(accessLog.Format("${requestTime}")) +} + func TestAccessLog_ParseGEO(t *testing.T) { teageo.SetupDB() From f5e3562fcc098fbb7f4935d8d78a3ce8fabdcde8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Tue, 2 Jun 2020 15:42:50 +0800 Subject: [PATCH 110/121] =?UTF-8?q?=E5=AF=B9=E5=86=85=E5=AD=98=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E5=A2=9E=E5=8A=A0=E5=A4=9A=E4=B8=AA=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teamemory/cell.go | 6 ++++++ teamemory/grid_test.go | 23 +++++++++++++++++++++++ teawaf/utils/utils_test.go | 22 ++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/teamemory/cell.go b/teamemory/cell.go index ce16539..4f06225 100644 --- a/teamemory/cell.go +++ b/teamemory/cell.go @@ -160,6 +160,11 @@ func (this *Cell) Range(f func(item *Item)) { func (this *Cell) Recycle() { this.locker.Lock() + if len(this.mapping) == 0 { + this.locker.Unlock() + return + } + timestamp := time.Now().Unix() for key, item := range this.mapping { if item.ExpireAt < timestamp { @@ -168,6 +173,7 @@ func (this *Cell) Recycle() { this.totalBytes -= item.Size() } } + this.locker.Unlock() } diff --git a/teamemory/grid_test.go b/teamemory/grid_test.go index c003442..b7cbf50 100644 --- a/teamemory/grid_test.go +++ b/teamemory/grid_test.go @@ -179,3 +179,26 @@ func TestMemoryGrid_Destroy(t *testing.T) { t.Fatal("looper != nil") } } + +func TestMemoryGrid_Recycle(t *testing.T) { + cell := NewCell() + timestamp := time.Now().Unix() + for i := 0; i < 300_0000; i++ { + cell.Write(uint64(i), &Item{ + ExpireAt: timestamp - 30, + }) + } + before := time.Now() + cell.Recycle() + t.Log(time.Since(before).Seconds()*1000, "ms") + t.Log(len(cell.mapping)) + + runtime.GC() + printMem(t) +} + +func printMem(t *testing.T) { + mem := &runtime.MemStats{} + runtime.ReadMemStats(mem) + t.Log(mem.Alloc/1024/1024, "M") +} diff --git a/teawaf/utils/utils_test.go b/teawaf/utils/utils_test.go index 26856f3..763a524 100644 --- a/teawaf/utils/utils_test.go +++ b/teawaf/utils/utils_test.go @@ -1,7 +1,10 @@ package utils import ( + "github.com/TeaWeb/code/teatesting" + "net/http" "regexp" + "strconv" "strings" "testing" ) @@ -20,6 +23,25 @@ func TestMatchBytesCache(t *testing.T) { t.Log(MatchBytesCache(regex, []byte("123"))) } +func TestMatchRemoteCache(t *testing.T) { + if teatesting.IsGlobal() { + return + } + client := http.Client{} + for i := 0; i < 200_0000; i++ { + req, err := http.NewRequest(http.MethodGet, "http://192.168.2.30:8882/?arg="+strconv.Itoa(i), nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("User-Agent", "GoTest/"+strconv.Itoa(i)) + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + } +} + func BenchmarkMatchStringCache(b *testing.B) { data := strings.Repeat("HELLO", 512) regex := regexp.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`) From 49710294deac22a65e4cbb923d372215a659d0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Thu, 4 Jun 2020 17:41:10 +0800 Subject: [PATCH 111/121] =?UTF-8?q?[log]=E6=96=87=E4=BB=B6=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E7=AD=96=E7=95=A5=E5=A2=9E=E5=8A=A0=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=88=9B=E5=BB=BA=E7=9B=AE=E5=BD=95=E9=80=89?= =?UTF-8?q?=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tealogs/storage_file.go | 23 +++++++++++++++---- .../actions/default/proxy/log/policies/add.go | 4 +++- .../default/proxy/log/policies/update.go | 4 +++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/tealogs/storage_file.go b/tealogs/storage_file.go index 694d25c..f40a732 100644 --- a/tealogs/storage_file.go +++ b/tealogs/storage_file.go @@ -5,6 +5,7 @@ import ( "github.com/TeaWeb/code/tealogs/accesslogs" "github.com/iwind/TeaGo/logs" "os" + "path/filepath" "sync" ) @@ -12,7 +13,8 @@ import ( type FileStorage struct { Storage `yaml:", inline"` - Path string `yaml:"path" json:"path"` // 文件路径,支持变量:${year|month|week|day|hour|minute|second} + Path string `yaml:"path" json:"path"` // 文件路径,支持变量:${year|month|week|day|hour|minute|second} + AutoCreate bool `yaml:"autoCreate" json:"autoCreate"` // 是否自动创建目录 writeLocker sync.Mutex @@ -52,10 +54,10 @@ func (this *FileStorage) Write(accessLogs []*accesslogs.AccessLog) error { } _, err = fp.Write(data) if err != nil { - this.Close() + _ = this.Close() break } - fp.WriteString("\n") + _, _ = fp.WriteString("\n") } return nil } @@ -87,7 +89,20 @@ func (this *FileStorage) fp() *os.File { // 关闭其他的文件 for _, f := range this.files { - f.Close() + _ = f.Close() + } + + // 是否创建文件目录 + if this.AutoCreate { + dir := filepath.Dir(path) + _, err := os.Stat(dir) + if os.IsNotExist(err) { + err = os.MkdirAll(dir, 0777) + if err != nil { + logs.Error(err) + return nil + } + } } // 打开新文件 diff --git a/teaweb/actions/default/proxy/log/policies/add.go b/teaweb/actions/default/proxy/log/policies/add.go index 48d47fd..aa39923 100644 --- a/teaweb/actions/default/proxy/log/policies/add.go +++ b/teaweb/actions/default/proxy/log/policies/add.go @@ -37,7 +37,8 @@ func (this *AddAction) RunPost(params struct { StorageType string // file - FilePath string + FilePath string + FileAutoCreate bool // es EsEndpoint string @@ -93,6 +94,7 @@ func (this *AddAction) RunPost(params struct { storage.Format = params.StorageFormat storage.Template = params.StorageTemplate storage.Path = params.FilePath + storage.AutoCreate = params.FileAutoCreate instance = storage case tealogs.StorageTypeES: params.Must. diff --git a/teaweb/actions/default/proxy/log/policies/update.go b/teaweb/actions/default/proxy/log/policies/update.go index d7bff91..b428f4a 100644 --- a/teaweb/actions/default/proxy/log/policies/update.go +++ b/teaweb/actions/default/proxy/log/policies/update.go @@ -56,7 +56,8 @@ func (this *UpdateAction) RunPost(params struct { StorageType string // file - FilePath string + FilePath string + FileAutoCreate bool // es EsEndpoint string @@ -117,6 +118,7 @@ func (this *UpdateAction) RunPost(params struct { storage.Format = params.StorageFormat storage.Template = params.StorageTemplate storage.Path = params.FilePath + storage.AutoCreate = params.FileAutoCreate instance = storage case tealogs.StorageTypeES: params.Must. From 02d511515bf7f865013f1a8cf06f6296c739c377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年6月11日 11:53:53 +0800 Subject: [PATCH 112/121] =?UTF-8?q?[waf]=E5=90=8C=E6=97=B6=E6=94=AF?= =?UTF-8?q?=E6=8C=81URL=E5=8F=82=E6=95=B0=E4=B8=AD=E7=9A=84+=E5=92=8C%20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teawaf/rule.go | 1 + 1 file changed, 1 insertion(+) diff --git a/teawaf/rule.go b/teawaf/rule.go index 2824a05..86e9efa 100644 --- a/teawaf/rule.go +++ b/teawaf/rule.go @@ -523,6 +523,7 @@ func (this *Rule) unescape(v string) string { v = strings.Replace(v, `!`, `(!|%21)`, -1) v = strings.Replace(v, `/`, `(/|%2F)`, -1) v = strings.Replace(v, `;`, `(;|%3B)`, -1) + v = strings.Replace(v, `\+`, `(\+|%20)`, -1) return v } From 34c7c0dfd263f95cfefd8097a4865dbf799d6304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年6月11日 11:58:39 +0800 Subject: [PATCH 113/121] =?UTF-8?q?[mongo]=E4=BC=98=E5=8C=96MongoDB?= =?UTF-8?q?=E7=9A=84=E5=85=A8=E5=B1=80=E7=9A=84group=E6=80=A7=E8=83=BD?= =?UTF-8?q?=EF=BC=88=E5=8F=AA=E7=BB=9F=E8=AE=A1=E5=89=8D10=E4=B8=87?= =?UTF-8?q?=E4=B8=AA=E6=95=B0=E6=8D=AE=EF=BC=89=EF=BC=8C=E4=BB=A5=E9=81=BF?= =?UTF-8?q?=E5=85=8D=E6=9F=A5=E8=AF=A2=E8=B6=85=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teadb/driver_mongo.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/teadb/driver_mongo.go b/teadb/driver_mongo.go index 367564e..5ccd47e 100644 --- a/teadb/driver_mongo.go +++ b/teadb/driver_mongo.go @@ -388,6 +388,9 @@ func (this *MongoDriver) Group(query *Query, field string, result map[string]Exp map[string]interface{}{ "$match": filter, }, + map[string]interface{}{ + "$limit": 100000, // 限制进入下一个pipeline的记录数量,以避免查询超时 + }, map[string]interface{}{ "$group": group, }, From bf79f8dd09efbb94b68a3f91c1aa97a2467cb890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年6月15日 16:16:52 +0800 Subject: [PATCH 114/121] =?UTF-8?q?[waf]MongoDB=E4=B8=AD=E7=9A=84attrs.waf?= =?UTF-8?q?=5Fid=E5=A2=9E=E5=8A=A0=E7=B4=A2=E5=BC=95=E4=BB=A5=E6=8F=90?= =?UTF-8?q?=E5=8D=87=E6=9F=A5=E8=AF=A2=E9=80=9F=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teadb/dao_access_log_mongo.go | 49 ++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/teadb/dao_access_log_mongo.go b/teadb/dao_access_log_mongo.go index 04e1ce3..cf79315 100644 --- a/teadb/dao_access_log_mongo.go +++ b/teadb/dao_access_log_mongo.go @@ -362,28 +362,35 @@ func (this *MongoAccessLogDAO) initTable(table string) { if isInitializedTable(table) { return } - for _, fields := range [][]*shared.IndexField{ - { - shared.NewIndexField("serverId", true), - }, - { - shared.NewIndexField("status", true), - shared.NewIndexField("serverId", true), - }, - { - shared.NewIndexField("remoteAddr", true), - shared.NewIndexField("serverId", true), - }, - { - shared.NewIndexField("hasErrors", true), - shared.NewIndexField("serverId", true), - }, - } { - err := this.createIndex(table, fields) - if err != nil { - logs.Error(err) + + // 异步执行,防止阻塞进程 + go func() { + for _, fields := range [][]*shared.IndexField{ + { + shared.NewIndexField("serverId", true), + }, + { + shared.NewIndexField("status", true), + shared.NewIndexField("serverId", true), + }, + { + shared.NewIndexField("remoteAddr", true), + shared.NewIndexField("serverId", true), + }, + { + shared.NewIndexField("hasErrors", true), + shared.NewIndexField("serverId", true), + }, + { + shared.NewIndexField("attrs.waf_id", true), + }, + } { + err := this.createIndex(table, fields) + if err != nil { + logs.Error(err) + } } - } + }() } func (this *MongoAccessLogDAO) createIndex(table string, fields []*shared.IndexField) error { From f5404621fbd218db952bdd23bd18b2b0121b6091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年6月21日 11:41:23 +0800 Subject: [PATCH 115/121] =?UTF-8?q?[proxy]=E4=BF=AE=E5=A4=8D=E8=B4=9F?= =?UTF-8?q?=E8=BD=BD=E5=9D=87=E8=A1=A1=E9=80=89=E9=A1=B9=E4=B8=8D=E8=B5=B7?= =?UTF-8?q?=E4=BD=9C=E7=94=A8=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconfigs/scheduling/scheduling_sticky.go | 2 -- teaconfigs/shared/request_call.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/teaconfigs/scheduling/scheduling_sticky.go b/teaconfigs/scheduling/scheduling_sticky.go index 651c1a2..060dc03 100644 --- a/teaconfigs/scheduling/scheduling_sticky.go +++ b/teaconfigs/scheduling/scheduling_sticky.go @@ -2,7 +2,6 @@ package scheduling import ( "github.com/TeaWeb/code/teaconfigs/shared" - "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" "math/rand" "net/http" @@ -71,7 +70,6 @@ func (this *StickyScheduling) Next(call *shared.RequestCall) CandidateInterface Path: "/", Expires: time.Now().AddDate(0, 1, 0), }) - logs.Println("set cookie", param, codes) }) } else { call.AddResponseCall(func(resp http.ResponseWriter) { diff --git a/teaconfigs/shared/request_call.go b/teaconfigs/shared/request_call.go index 2b86770..95d5756 100644 --- a/teaconfigs/shared/request_call.go +++ b/teaconfigs/shared/request_call.go @@ -25,7 +25,7 @@ func (this *RequestCall) Reset() { this.Formatter = nil this.Request = nil this.ResponseCallbacks = nil - this.Options = nil + this.Options = maps.Map{} } // 添加响应回调 From a32b329c8cedac70760ae14d7c2a0b0aa6196228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年6月21日 11:45:21 +0800 Subject: [PATCH 116/121] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaconst/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/teaconst/const.go b/teaconst/const.go index b2239b1..c52ef9b 100644 --- a/teaconst/const.go +++ b/teaconst/const.go @@ -1,7 +1,7 @@ package teaconst const ( - TeaVersion = "0.1.11" + TeaVersion = "0.1.12" TeaProcessName = "teaweb" // 进程名 TeaProductName = "TeaWeb" // 产品名 From 14c5fbd760c5830c8142dbc3778089cd56ebd6a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年6月21日 11:45:34 +0800 Subject: [PATCH 117/121] =?UTF-8?q?=E5=A2=9E=E5=8A=A0HTTP=20OPTIONS?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teatesting/server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/teatesting/server.go b/teatesting/server.go index 2162277..9a18728 100644 --- a/teatesting/server.go +++ b/teatesting/server.go @@ -223,6 +223,9 @@ func StartTestServer() { } logs.Println("[test]websocket closed") }). + Options("/options", func(req *http.Request, resp http.ResponseWriter) { + resp.Header().Set("AllowMethods", "GET, POST") + }). StartOn("127.0.0.1:9991") } From 347aa0e8100eb51e95a51072022b96baf870739c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年6月25日 10:43:46 +0800 Subject: [PATCH 118/121] =?UTF-8?q?[proxy]https=20host=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E4=B8=AD=E5=A2=9E=E5=8A=A0=E7=AB=AF=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/listener.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/teaproxy/listener.go b/teaproxy/listener.go index aefa97a..0fd6508 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -11,6 +11,7 @@ import ( "golang.org/x/net/http2" "net" "net/http" + "strings" "sync" "sync/atomic" "time" @@ -409,7 +410,13 @@ func (this *Listener) handleHTTP(writer http.ResponseWriter, rawRequest *http.Re if rawRequest.TLS != nil { serverName := rawRequest.TLS.ServerName if len(serverName)> 0 { - reqHost = serverName + // 端口 + index := strings.LastIndex(reqHost, ":") + if index>= 0 { + reqHost = serverName + reqHost[index:] + } else { + reqHost = serverName + } } } From be3a24d0e3ea831eb0ad11267d1e0f0c3b467cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: 2020年6月25日 12:27:57 +0800 Subject: [PATCH 119/121] =?UTF-8?q?[proxy]https=E5=9C=A8=E5=AF=B9=E6=96=B9?= =?UTF-8?q?=E8=AE=BF=E9=97=AE=E7=9A=84URL=E6=98=AFIP=E6=97=B6=E6=89=8D?= =?UTF-8?q?=E4=BC=9A=E4=BB=8ESNI=E4=B8=AD=E8=8E=B7=E5=8F=96=E5=9F=9F?= =?UTF-8?q?=E5=90=8D=EF=BC=8C=E5=90=A6=E5=88=99=E4=BD=BF=E7=94=A8URL?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E5=9F=9F=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teaproxy/listener.go | 35 ++++++++++++++++++++++++++--------- teaproxy/listener_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/teaproxy/listener.go b/teaproxy/listener.go index 0fd6508..cdbb4e9 100644 --- a/teaproxy/listener.go +++ b/teaproxy/listener.go @@ -407,15 +407,17 @@ func (this *Listener) handleHTTP(writer http.ResponseWriter, rawRequest *http.Re reqHost := rawRequest.Host // TLS域名 - if rawRequest.TLS != nil { - serverName := rawRequest.TLS.ServerName - if len(serverName)> 0 { - // 端口 - index := strings.LastIndex(reqHost, ":") - if index>= 0 { - reqHost = serverName + reqHost[index:] - } else { - reqHost = serverName + if this.isIP(reqHost) { + if rawRequest.TLS != nil { + serverName := rawRequest.TLS.ServerName + if len(serverName)> 0 { + // 端口 + index := strings.LastIndex(reqHost, ":") + if index>= 0 { + reqHost = serverName + reqHost[index:] + } else { + reqHost = serverName + } } } } @@ -748,3 +750,18 @@ func (this *Listener) connectTCPBackend(clientConn net.Conn, serverName string) delete(this.connectingTCPMap, clientConn) this.connectingTCPLocker.Unlock() } + +func (this *Listener) isIP(host string) bool { + // IPv6 + if strings.Index(host, "[")> -1 { + return true + } + + for _, b := range host { + if b>= 'a' && b <= 'z' { + return false + } + } + + return true +} diff --git a/teaproxy/listener_test.go b/teaproxy/listener_test.go index 17ff4c1..ab53cfd 100644 --- a/teaproxy/listener_test.go +++ b/teaproxy/listener_test.go @@ -5,8 +5,11 @@ import ( "github.com/TeaWeb/code/teaconfigs" "github.com/TeaWeb/code/teaconfigs/shared" "github.com/TeaWeb/code/teatesting" + "github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" + "runtime" + "strings" "testing" "time" ) @@ -331,3 +334,39 @@ func printListener(listener *Listener, t *testing.T) { }, t) } } + +func TestDetectIPOrDomain(t *testing.T) { + a := assert.NewAssertion(t) + a.IsTrue(testIsIP("192.168.1.101")) + a.IsTrue(testIsIP("192.168.1.102:1000")) + a.IsTrue(testIsIP("[1:2:3:4]:1000")) + a.IsTrue(!testIsIP("192.168.1.com")) + a.IsTrue(!testIsIP("192.168.1.com:12345")) + a.IsTrue(!testIsIP("local345:12345")) +} + +func BenchmarkIPOrDomain(b *testing.B) { + runtime.GOMAXPROCS(1) + + for i := 0; i < b.N; i++ { + _ = testIsIP("192.168.1.101") + _ = testIsIP("192.168.1.101:1000") + _ = testIsIP("www.example.com") + _ = testIsIP("www.example.com:12345") + } +} + +func testIsIP(host string) bool { + // IPv6 + if strings.Index(host, "[")> -1 { + return true + } + + for _, b := range host { + if b>= 'a' && b <= 'z' { + return false + } + } + + return true +} From f47ba712140eeae95156d0cdf99da66a3e6be7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Thu, 9 Jul 2020 14:29:06 +0800 Subject: [PATCH 120/121] =?UTF-8?q?redis=E4=BD=BF=E7=94=A8=E6=9C=80?= =?UTF-8?q?=E6=96=B0=E7=9A=84=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- teacache/redis.go | 19 ++++++++++--------- teatesting/require.go | 3 ++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/teacache/redis.go b/teacache/redis.go index 3816ce2..56a5a65 100644 --- a/teacache/redis.go +++ b/teacache/redis.go @@ -5,6 +5,7 @@ import ( "github.com/go-redis/redis" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" + "golang.org/x/net/context" "strings" "time" ) @@ -73,12 +74,12 @@ func (this *RedisManager) SetOptions(options map[string]interface{}) { } func (this *RedisManager) Write(key string, data []byte) error { - cmd := this.client.Set("TEA_CACHE_"+this.id+key, string(data), this.Life) + cmd := this.client.Set(context.Background(), "TEA_CACHE_"+this.id+key, string(data), this.Life) return cmd.Err() } func (this *RedisManager) Read(key string) (data []byte, err error) { - cmd := this.client.Get("TEA_CACHE_" + this.id + key) + cmd := this.client.Get(context.Background(), "TEA_CACHE_"+this.id+key) if cmd.Err() != nil { if cmd.Err() == redis.Nil { return nil, ErrNotFound @@ -91,7 +92,7 @@ func (this *RedisManager) Read(key string) (data []byte, err error) { // 删除 func (this *RedisManager) Delete(key string) error { - cmd := this.client.Del("TEA_CACHE_" + this.id + key) + cmd := this.client.Del(context.Background(), "TEA_CACHE_"+this.id+key) return cmd.Err() } @@ -111,7 +112,7 @@ func (this *RedisManager) DeletePrefixes(prefixes []string) (int, error) { loopCount++ var keys []string - keys, cursor, err = this.client.Scan(cursor, keyPrefix+"*", 10000).Result() + keys, cursor, err = this.client.Scan(context.Background(), cursor, keyPrefix+"*", 10000).Result() if err != nil { return count, err } @@ -120,7 +121,7 @@ func (this *RedisManager) DeletePrefixes(prefixes []string) (int, error) { realKey := key[keyPrefixLength:] for _, prefix := range prefixes { if strings.HasPrefix(realKey, prefix) || strings.HasPrefix("http://"+realKey, prefix) || strings.HasPrefix("https://"+realKey, prefix) { - err1 := this.client.Del(key).Err() + err1 := this.client.Del(context.Background(), key).Err() if err1 != nil { err = err1 break @@ -153,14 +154,14 @@ func (this *RedisManager) Stat() (size int64, countKeys int, err error) { loopCount++ var keys []string - keys, cursor, err = this.client.Scan(cursor, "TEA_CACHE_"+this.Id()+"*", 10000).Result() + keys, cursor, err = this.client.Scan(context.Background(), cursor, "TEA_CACHE_"+this.Id()+"*", 10000).Result() if err != nil { return } if len(keys)> 0 { countKeys += len(keys) for _, key := range keys { - val, _ := this.client.Get(key).Bytes() + val, _ := this.client.Get(context.Background(), key).Bytes() size += int64(len(val)) } } @@ -186,13 +187,13 @@ func (this *RedisManager) Clean() error { loopCount++ var keys []string - keys, cursor, err = this.client.Scan(cursor, "TEA_CACHE_"+this.Id()+"*", 10000).Result() + keys, cursor, err = this.client.Scan(context.Background(), cursor, "TEA_CACHE_"+this.Id()+"*", 10000).Result() if err != nil { return err } if len(keys)> 0 { for _, key := range keys { - err1 := this.client.Del(key).Err() + err1 := this.client.Del(context.Background(), key).Err() if err1 != nil { err = err1 break diff --git a/teatesting/require.go b/teatesting/require.go index 216b0b1..9757d98 100644 --- a/teatesting/require.go +++ b/teatesting/require.go @@ -1,6 +1,7 @@ package teatesting import ( + "context" "github.com/TeaWeb/code/teautils" "github.com/go-redis/redis" "github.com/iwind/TeaGo/logs" @@ -101,7 +102,7 @@ func RequireRedis() bool { Addr: "127.0.0.1:6379", DialTimeout: 5 * time.Second, }) - cmd := client.Ping() + cmd := client.Ping(context.Background()) return cmd.Err() == nil } From 3cbfdc6eef87799607d8b68ce648fc2b8c2228fd Mon Sep 17 00:00:00 2001 From: Liu Xiangchao Date: 2020年8月17日 10:04:55 +0800 Subject: [PATCH 121/121] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 593f4b5..85f5ba9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ +# 重大提示: 我们已经使用go mod重新构建了项目结构,在v0.2.0以后请使用 [https://github.com/TeaWeb/build](https://github.com/TeaWeb/build) 源码代替! + 这里是TeaWeb Go源码,以下是一些有用的链接: * [构建自己的二进制版本](http://teaos.cn/doc/main/Build.md) * [可以直接使用的二进制版本](http://teaos.cn/download) -* [实现自己的插件](http://teaos.cn/doc/plugins/Write.md) \ No newline at end of file +* [实现自己的插件](http://teaos.cn/doc/plugins/Write.md)