// Copyright 2011 Google Inc. All rights reserved. // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. package blobstore import ( "bytes" "encoding/base64" "fmt" "io" "mime/multipart" "mime/quotedprintable" "net/http" "net/textproto" "os" "strconv" "strings" "testing" "golang.org/x/text/encoding/htmlindex" "google.golang.org/appengine" "google.golang.org/appengine/internal/aetesting" pb "google.golang.org/appengine/internal/blobstore" ) const rbs = readBufferSize const charsetUTF8 = "utf-8" const charsetISO2022JP = "iso-2022-jp" const nonASCIIStr = "Hello, 世界" func min(x, y int) int { if x < y { return x } return y } func fakeFetchData(req *pb.FetchDataRequest, res *pb.FetchDataResponse) error { i0 := int(*req.StartIndex) i1 := int(*req.EndIndex + 1) // Blobstore's end-indices are inclusive; Go's are exclusive. bk := *req.BlobKey if i := strings.Index(bk, "."); i != -1 { // Strip everything past the ".". bk = bk[:i] } switch bk { case "a14p": const s = "abcdefghijklmnop" i0 := min(len(s), i0) i1 := min(len(s), i1) res.Data = []byte(s[i0:i1]) case "longBlob": res.Data = make([]byte, i1-i0) for i := range res.Data { res.Data[i] = 'A' + uint8(i0/rbs) i0++ } } return nil } // step is one step of a readerTest. // It consists of a Reader method to call, the method arguments // (lenp, offset, whence) and the expected results. type step struct { method string lenp int offset int64 whence int want string wantErr error } var readerTest = []struct { blobKey string step []step }{ {"noSuchBlobKey", []step{ {"Read", 8, 0, 0, "", io.EOF}, }}, {"a14p.0", []step{ // Test basic reads. {"Read", 1, 0, 0, "a", nil}, {"Read", 3, 0, 0, "bcd", nil}, {"Read", 1, 0, 0, "e", nil}, {"Read", 2, 0, 0, "fg", nil}, // Test Seek. {"Seek", 0, 2, os.SEEK_SET, "2", nil}, {"Read", 5, 0, 0, "cdefg", nil}, {"Seek", 0, 2, os.SEEK_CUR, "9", nil}, {"Read", 1, 0, 0, "j", nil}, // Test reads up to and past EOF. {"Read", 5, 0, 0, "klmno", nil}, {"Read", 5, 0, 0, "p", nil}, {"Read", 5, 0, 0, "", io.EOF}, // Test ReadAt. {"ReadAt", 4, 0, 0, "abcd", nil}, {"ReadAt", 4, 3, 0, "defg", nil}, {"ReadAt", 4, 12, 0, "mnop", nil}, {"ReadAt", 4, 13, 0, "nop", io.EOF}, {"ReadAt", 4, 99, 0, "", io.EOF}, }}, {"a14p.1", []step{ // Test Seek before any reads. {"Seek", 0, 2, os.SEEK_SET, "2", nil}, {"Read", 1, 0, 0, "c", nil}, // Test that ReadAt doesn't affect the Read offset. {"ReadAt", 3, 9, 0, "jkl", nil}, {"Read", 3, 0, 0, "def", nil}, }}, {"a14p.2", []step{ // Test ReadAt before any reads or seeks. {"ReadAt", 2, 14, 0, "op", nil}, }}, {"longBlob.0", []step{ // Test basic read. {"Read", 1, 0, 0, "A", nil}, // Test that Read returns early when the buffer is exhausted. {"Seek", 0, rbs - 2, os.SEEK_SET, strconv.Itoa(rbs - 2), nil}, {"Read", 5, 0, 0, "AA", nil}, {"Read", 3, 0, 0, "BBB", nil}, // Test that what we just read is still in the buffer. {"Seek", 0, rbs - 2, os.SEEK_SET, strconv.Itoa(rbs - 2), nil}, {"Read", 5, 0, 0, "AABBB", nil}, // Test ReadAt. {"ReadAt", 3, rbs - 4, 0, "AAA", nil}, {"ReadAt", 6, rbs - 4, 0, "AAAABB", nil}, {"ReadAt", 8, rbs - 4, 0, "AAAABBBB", nil}, {"ReadAt", 5, rbs - 4, 0, "AAAAB", nil}, {"ReadAt", 2, rbs - 4, 0, "AA", nil}, // Test seeking backwards from the Read offset. {"Seek", 0, 2*rbs - 8, os.SEEK_SET, strconv.Itoa(2*rbs - 8), nil}, {"Read", 1, 0, 0, "B", nil}, {"Read", 1, 0, 0, "B", nil}, {"Read", 1, 0, 0, "B", nil}, {"Read", 1, 0, 0, "B", nil}, {"Read", 8, 0, 0, "BBBBCCCC", nil}, }}, {"longBlob.1", []step{ // Test ReadAt with a slice larger than the buffer size. {"LargeReadAt", 2*rbs - 2, 0, 0, strconv.Itoa(2*rbs - 2), nil}, {"LargeReadAt", 2*rbs - 1, 0, 0, strconv.Itoa(2*rbs - 1), nil}, {"LargeReadAt", 2*rbs + 0, 0, 0, strconv.Itoa(2*rbs + 0), nil}, {"LargeReadAt", 2*rbs + 1, 0, 0, strconv.Itoa(2*rbs + 1), nil}, {"LargeReadAt", 2*rbs + 2, 0, 0, strconv.Itoa(2*rbs + 2), nil}, {"LargeReadAt", 2*rbs - 2, 1, 0, strconv.Itoa(2*rbs - 2), nil}, {"LargeReadAt", 2*rbs - 1, 1, 0, strconv.Itoa(2*rbs - 1), nil}, {"LargeReadAt", 2*rbs + 0, 1, 0, strconv.Itoa(2*rbs + 0), nil}, {"LargeReadAt", 2*rbs + 1, 1, 0, strconv.Itoa(2*rbs + 1), nil}, {"LargeReadAt", 2*rbs + 2, 1, 0, strconv.Itoa(2*rbs + 2), nil}, }}, } func TestReader(t *testing.T) { for _, rt := range readerTest { c := aetesting.FakeSingleContext(t, "blobstore", "FetchData", fakeFetchData) r := NewReader(c, appengine.BlobKey(rt.blobKey)) for i, step := range rt.step { var ( got string gotErr error n int offset int64 ) switch step.method { case "LargeReadAt": p := make([]byte, step.lenp) n, gotErr = r.ReadAt(p, step.offset) got = strconv.Itoa(n) case "Read": p := make([]byte, step.lenp) n, gotErr = r.Read(p) got = string(p[:n]) case "ReadAt": p := make([]byte, step.lenp) n, gotErr = r.ReadAt(p, step.offset) got = string(p[:n]) case "Seek": offset, gotErr = r.Seek(step.offset, step.whence) got = strconv.FormatInt(offset, 10) default: t.Fatalf("unknown method: %s", step.method) } if gotErr != step.wantErr { t.Fatalf("%s step %d: got error %v want %v", rt.blobKey, i, gotErr, step.wantErr) } if got != step.want { t.Fatalf("%s step %d: got %q want %q", rt.blobKey, i, got, step.want) } } } } // doPlainTextParseUploadTest tests ParseUpload's decoding of non-file form fields. // It ensures that MIME multipart parts with Content-Type not equal to // "message/external-body" (i.e. form fields that are not file uploads) are decoded // correctly according to the value of their Content-Transfer-Encoding header field. // If charset is not the empty string it will be set in the request's Content-Type // header field, and if encoding is not the empty string then the Content-Transfer-Encoding // header field will be set. func doPlainTextParseUploadTest(t *testing.T, charset string, encoding string, rawContent string, encodedContent string) { bodyBuf := &bytes.Buffer{} w := multipart.NewWriter(bodyBuf) fieldName := "foo" hdr := textproto.MIMEHeader{} hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", fieldName)) if charset != "" { hdr.Set("Content-Type", fmt.Sprintf("text/plain; charset=%q", charset)) } else { hdr.Set("Content-Type", "text/plain") } if encoding != "" { hdr.Set("Content-Transfer-Encoding", encoding) } pw, err := w.CreatePart(hdr) if err != nil { t.Fatalf("error creating part: %v", err) } pw.Write([]byte(encodedContent)) if err := w.Close(); err != nil { t.Fatalf("error closing multipart writer: %v\n", err) } req, err := http.NewRequest("POST", "/upload", bodyBuf) if err != nil { t.Fatalf("error creating request: %v", err) } req.Header.Set("Content-Type", w.FormDataContentType()) _, other, err := ParseUpload(req) if err != nil { t.Fatalf("error parsing upload: %v", err) } if other[fieldName][0] != rawContent { t.Errorf("got %q expected %q", other[fieldName][0], rawContent) } } func TestParseUploadUTF8Base64Encoding(t *testing.T) { encoded := base64.StdEncoding.EncodeToString([]byte(nonASCIIStr)) doPlainTextParseUploadTest(t, charsetUTF8, "base64", nonASCIIStr, encoded) } func TestParseUploadUTF8Base64EncodingMultiline(t *testing.T) { testStr := "words words words words words words words words words words words words" encoded := "d29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29y\r\nZHMgd29yZHMgd29yZHM=" doPlainTextParseUploadTest(t, charsetUTF8, "base64", testStr, encoded) } func TestParseUploadUTF8QuotedPrintableEncoding(t *testing.T) { var encoded bytes.Buffer writer := quotedprintable.NewWriter(&encoded) writer.Write([]byte(nonASCIIStr)) writer.Close() doPlainTextParseUploadTest(t, charsetUTF8, "quoted-printable", nonASCIIStr, encoded.String()) } func TestParseUploadISO2022JPBase64Encoding(t *testing.T) { testStr := "こんにちは" encoding, err := htmlindex.Get(charsetISO2022JP) if err != nil { t.Fatalf("error getting encoding: %v", err) } charsetEncoded, err := encoding.NewEncoder().String(testStr) if err != nil { t.Fatalf("error encoding string: %v", err) } base64Encoded := base64.StdEncoding.EncodeToString([]byte(charsetEncoded)) doPlainTextParseUploadTest(t, charsetISO2022JP, "base64", testStr, base64Encoded) } func TestParseUploadNoEncoding(t *testing.T) { doPlainTextParseUploadTest(t, "", "", "Hello", "Hello") }