用最小的内存发送大文件 翻译+分析

原文:

https://medium.com/@owlwalks/sending-big-file-with-minimal-memory-in-golang-8f3fc280d2c

一般我们发送文件

 buf := new(bytes.Buffer)
 writer := multipart.NewWriter(buf)
 defer writer.Close()
 part, err := writer.CreateFormFile("myFile", "foo.txt")
 if err != nil {
 return err
 }
 file, err := os.Open(name)
 if err != nil {
 return err
 }
 defer file.Close()
 if _, err = io.Copy(part, file); err != nil {
 return err
 }
 http.Post(url, writer.FormDataContentType(), buf)

这样buf会读取文件的所有内容,加入文件非常大,内存占用就会比较大

优化方法

r, w := io.Pipe()
m := multipart.NewWriter(w)
go func() {
 defer w.Close()
 defer m.Close()
 part, err := m.CreateFormFile("myFile", "foo.txt")
 if err != nil {
 return
 }
 file, err := os.Open(name)
 if err != nil {
 return
 }
 defer file.Close()
 if _, err = io.Copy(part, file); err != nil {
 return
 }
}()
http.Post(url, m.FormDataContentType(), r)

上述是代码是从原处拷贝,下面分析下原因

net/http 中

func Post(url, contentType string, body io.Reader) (resp *Response, err error) {
] return DefaultClient.Post(url, contentType, body)
]}
func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) {
 req, err := NewRequest("POST", url, body)
 if err != nil {
 return nil, err
 }
 req.Header.Set("Content-Type", contentType)
 return c.Do(req)
 }
func NewRequest(method, url string, body io.Reader) (*Request, error) {
...
 if body != nil {
 switch v := body.(type) {
 case *bytes.Buffer:
 req.ContentLength = int64(v.Len())
 buf := v.Bytes()
 req.GetBody = func() (io.ReadCloser, error) {
 r := bytes.NewReader(buf)
 return ioutil.NopCloser(r), nil
 }
 case *bytes.Reader:
 req.ContentLength = int64(v.Len())
 snapshot := *v
 req.GetBody = func() (io.ReadCloser, error) {
 r := snapshot
 return ioutil.NopCloser(&r), nil
 }
 case *strings.Reader:
 req.ContentLength = int64(v.Len())
 snapshot := *v
 req.GetBody = func() (io.ReadCloser, error) {
 r := snapshot
 return ioutil.NopCloser(&r), nil
 }
 default:
 // This is where we'd set it to -1 (at least
 // if body != NoBody) to mean unknown, but
 // that broke people during the Go 1.8 testing
 // period. People depend on it being 0 I
 // guess. Maybe retry later. See Issue 18117.
 }
...
}

os中

func Pipe() (r *File, w *File, err error) {
 var p [2]int
 e := syscall.Pipe2(p[0:], syscall.O_CLOEXEC)
 // pipe2 was added in 2.6.27 and our minimum requirement is 2.6.23, so it
 // might not be implemented.
 if e == syscall.ENOSYS {
 // See ../syscall/exec.go for description of lock.
 syscall.ForkLock.RLock()
 e = syscall.Pipe(p[0:])
 if e != nil {
 syscall.ForkLock.RUnlock()
 return nil, nil, NewSyscallError("pipe", e)
 }
 syscall.CloseOnExec(p[0])
 syscall.CloseOnExec(p[1])
 syscall.ForkLock.RUnlock()
 } else if e != nil {
 return nil, nil, NewSyscallError("pipe2", e)
 }
 return newFile(uintptr(p[0]), "|0", kindPipe), newFile(uintptr(p[1]), "|1", kindPipe), nil
}

可见Pipe返回的类型在body的类型判断中进入了default的逻辑,而追溯post的方法会在此处写

func (t *transferWriter) writeBody(w io.Writer) error {
...
 if t.Body != nil {
 var body = transferBodyReader{t}
 if chunked(t.TransferEncoding) {
 if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse {
 w = &internal.FlushAfterChunkWriter{Writer: bw}
 }
 cw := internal.NewChunkedWriter(w)
 _, err = io.Copy(cw, body)
 if err == nil {
 err = cw.Close()
 }
 } else if t.ContentLength == -1 {
 ncopy, err = io.Copy(w, body)
 } else {
 ncopy, err = io.Copy(w, io.LimitReader(body, t.ContentLength))
 if err != nil {
 return err
 }
 var nextra int64
 nextra, err = io.Copy(ioutil.Discard, body)
 ncopy += nextra
 }
...
}

由于body不为nil,而且contentlength为0,所以进入了else的逻辑,也就形成了流式读取和流式写入,在大文件时候可以节省内存

有疑问加站长微信联系(非本文作者)

• 请尽量让自己的回复能够对别人有帮助
• 支持 Markdown 格式, **粗体**、~~删除线~~、`单行代码`
• 支持 @ 本站用户;支持表情(输入 : 提示),见 Emoji cheat sheet
• 图片支持拖拽、截图粘贴等方式上传