Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 22f5a97

Browse files
perf: Optimize website log reading (1Panel-dev#9755)
Refs 1Panel-dev#9383 
1 parent 90ba036 commit 22f5a97

File tree

4 files changed

+98
-9
lines changed

4 files changed

+98
-9
lines changed

‎agent/app/dto/response/file.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,12 @@ type FileWgetRes struct {
3838
}
3939

4040
type FileLineContent struct {
41-
Content string `json:"content"`
4241
End bool `json:"end"`
4342
Path string `json:"path"`
4443
Total int `json:"total"`
4544
TaskStatus string `json:"taskStatus"`
4645
Lines []string `json:"lines"`
46+
Scope string `json:"scope"`
4747
}
4848

4949
type FileExist struct {

‎agent/app/service/file.go‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -542,24 +542,48 @@ func (f *FileService) ReadLogByLine(req request.FileReadByLineReq) (*response.Fi
542542
logFilePath = path.Join(global.Dir.DataDir, fmt.Sprintf("apps/mariadb/%s/db/data/1Panel-slow.log", req.Name))
543543
}
544544

545-
lines, isEndOfFile, total, err := files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
545+
file, err := os.Open(logFilePath)
546546
if err != nil {
547547
return nil, err
548548
}
549-
if req.Latest && req.Page == 1 && len(lines) < 1000 && total > 1 {
550-
preLines, _, _, err := files.ReadFileByLine(logFilePath, total-1, req.PageSize, false)
549+
defer file.Close()
550+
stat, err := file.Stat()
551+
if err != nil {
552+
return nil, err
553+
}
554+
var (
555+
lines []string
556+
isEndOfFile bool
557+
total int
558+
scope string
559+
)
560+
561+
if stat.Size() > 500*1024*1024 {
562+
lines, err = files.TailFromEnd(logFilePath, req.PageSize)
563+
isEndOfFile = true
564+
scope = "tail"
565+
} else {
566+
lines, isEndOfFile, total, err = files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
551567
if err != nil {
552568
return nil, err
553569
}
554-
lines = append(preLines, lines...)
570+
if req.Latest && req.Page == 1 && len(lines) < 1000 && total > 1 {
571+
preLines, _, _, err := files.ReadFileByLine(logFilePath, total-1, req.PageSize, false)
572+
if err != nil {
573+
return nil, err
574+
}
575+
lines = append(preLines, lines...)
576+
}
577+
scope = "page"
555578
}
579+
556580
res := &response.FileLineContent{
557-
Content: strings.Join(lines, "\n"),
558581
End: isEndOfFile,
559582
Path: logFilePath,
560583
Total: total,
561584
TaskStatus: taskStatus,
562585
Lines: lines,
586+
Scope: scope,
563587
}
564588
return res, nil
565589
}

‎agent/utils/files/utils.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,66 @@ func countLines(path string) (int, error) {
101101
}
102102
}
103103

104+
func TailFromEnd(filename string, lines int) ([]string, error) {
105+
file, err := os.Open(filename)
106+
if err != nil {
107+
return nil, err
108+
}
109+
defer file.Close()
110+
111+
stat, err := file.Stat()
112+
if err != nil {
113+
return nil, err
114+
}
115+
fileSize := stat.Size()
116+
117+
bufSize := int64(4096)
118+
var result []string
119+
var leftover string
120+
121+
for offset := fileSize; offset > 0 && len(result) < lines; {
122+
readSize := bufSize
123+
if offset < bufSize {
124+
readSize = offset
125+
}
126+
offset -= readSize
127+
128+
buf := make([]byte, readSize)
129+
_, err := file.ReadAt(buf, offset)
130+
if err != nil && err != io.EOF {
131+
return nil, err
132+
}
133+
134+
data := string(buf) + leftover
135+
linesInChunk := strings.Split(data, "\n")
136+
137+
if offset > 0 {
138+
leftover = linesInChunk[0]
139+
linesInChunk = linesInChunk[1:]
140+
} else {
141+
leftover = ""
142+
}
143+
144+
for i := len(linesInChunk) - 1; i >= 0; i-- {
145+
if len(result) < lines {
146+
if !(i == len(linesInChunk)-1 && linesInChunk[i] == "" && len(result) == 0) {
147+
result = append([]string{linesInChunk[i]}, result...)
148+
}
149+
}
150+
}
151+
}
152+
153+
if leftover != "" && len(result) < lines {
154+
result = append([]string{leftover}, result...)
155+
}
156+
157+
if len(result) > lines {
158+
result = result[len(result)-lines:]
159+
}
160+
161+
return result, nil
162+
}
163+
104164
func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []string, isEndOfFile bool, total int, err error) {
105165
if !NewFileOp().Stat(filename) {
106166
return

‎frontend/src/components/log/file/index.vue‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<template>
22
<div v-loading="firstLoading">
33
<div v-if="defaultButton">
4+
<el-button icon="Refresh" @click="getContent(false)">{{ $t('commons.button.refresh') }}</el-button>
45
<el-checkbox
56
border
67
:disabled="isTailDisabled"
@@ -133,11 +134,10 @@ const end = ref(false);
133134
const lastLogs = ref([]);
134135
const maxPage = ref(0);
135136
const minPage = ref(0);
136-
let timer: NodeJS.Timer | null = null;
137+
let timer: ReturnType<typeofsetInterval> | null = null;
137138
const logPath = ref('');
138-
139+
const showTail =ref(false);
139140
const isTailDisabled = ref();
140-
141141
const firstLoading = ref(false);
142142
const logs = ref<string[]>([]);
143143
const logContainer = ref<HTMLElement | null>(null);
@@ -222,6 +222,10 @@ const getContent = async (pre: boolean) => {
222222
firstLoading.value = false;
223223
}
224224
225+
if (res.data.scope == 'tail') {
226+
showTail.value = false;
227+
}
228+
225229
if (res.data.taskStatus && res.data.taskStatus !== 'Executing') {
226230
isTailDisabled.value = true;
227231
}
@@ -342,6 +346,7 @@ const containerStyle = computed(() => ({
342346
}));
343347
344348
onMounted(async () => {
349+
showTail.value = props.showTail;
345350
logs.value = [];
346351
isTailDisabled.value = false;
347352
firstLoading.value = true;

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /