Index: src/pkg/archive/tar/common.go =================================================================== --- a/src/pkg/archive/tar/common.go +++ b/src/pkg/archive/tar/common.go @@ -13,6 +13,7 @@ package tar import ( + "bytes" "errors" "fmt" "os" @@ -174,6 +175,23 @@ c_ISSOCK = 0140000 // Socket ) +// Keywords for the PAX Extended Header +const ( + paxAtime = "atime" + paxCharset = "charset" + paxComment = "comment" + paxCtime = "ctime" // please note that ctime is not a valid pax header. + paxGid = "gid" + paxGname = "gname" + paxLinkpath = "linkpath" + paxMtime = "mtime" + paxPath = "path" + paxSize = "size" + paxUid = "uid" + paxUname = "uname" + paxNone = "" +) + // FileInfoHeader creates a partially-populated Header from fi. // If fi describes a symlink, FileInfoHeader records link as the link target. // If fi describes a directory, a slash is appended to the name. @@ -257,3 +275,25 @@ b, *sp = s[0:n], s[n:] return } + +func isASCII(s string) bool { + for _, c := range s { + if c >= 0x80 { + return false + } + } + return true +} + +func toASCII(s string) string { + if isASCII(s) { + return s + } + var buf bytes.Buffer + for _, c := range s { + if c < 0x80 { + buf.WriteByte(byte(c)) + } + } + return buf.String() +} Index: src/pkg/archive/tar/reader.go =================================================================== --- a/src/pkg/archive/tar/reader.go +++ b/src/pkg/archive/tar/reader.go @@ -95,45 +95,45 @@ func mergePAX(hdr *Header, headers map[string]string) error { for k, v := range headers { switch k { - case "path": + case paxPath: hdr.Name = v - case "linkpath": + case paxLinkpath: hdr.Linkname = v - case "gname": + case paxGname: hdr.Gname = v - case "uname": + case paxUname: hdr.Uname = v - case "uid": + case paxUid: uid, err := strconv.ParseInt(v, 10, 0) if err != nil { return err } hdr.Uid = int(uid) - case "gid": + case paxGid: gid, err := strconv.ParseInt(v, 10, 0) if err != nil { return err } hdr.Gid = int(gid) - case "atime": + case paxAtime: t, err := parsePAXTime(v) if err != nil { return err } hdr.AccessTime = t - case "mtime": + case paxMtime: t, err := parsePAXTime(v) if err != nil { return err } hdr.ModTime = t - case "ctime": + case paxCtime: t, err := parsePAXTime(v) if err != nil { return err } hdr.ChangeTime = t - case "size": + case paxSize: size, err := strconv.ParseInt(v, 10, 0) if err != nil { return err Index: src/pkg/archive/tar/writer.go =================================================================== --- a/src/pkg/archive/tar/writer.go +++ b/src/pkg/archive/tar/writer.go @@ -24,6 +24,7 @@ ErrFieldTooLong = errors.New("archive/tar: header field too long") ErrWriteAfterClose = errors.New("archive/tar: write after close") errNameTooLong = errors.New("archive/tar: name too long") + errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values") ) // A Writer provides sequential writing of a tar archive in POSIX.1 format. @@ -37,6 +38,7 @@ pad int64 // amount of padding to write after current file entry closed bool usedBinary bool // whether the binary numeric field extension was used + preferPax bool // use pax header instead of binary numeric header } // NewWriter creates a new Writer writing to w. @@ -65,16 +67,23 @@ } // Write s into b, terminating it with a NUL if there is room. -func (tw *Writer) cString(b []byte, s string) { +// If the value is too long for the field and allowPax is true add a paxheader record instead +func (tw *Writer) cString(b []byte, s string, allowPax bool, paxKeyword string, paxHeaders map[string]string) { + needsPaxHeader := allowPax && len(s) > len(b) || !isASCII(s) + if needsPaxHeader { + paxHeaders[paxKeyword] = s + return + } if len(s) > len(b) { if tw.err == nil { tw.err = ErrFieldTooLong } return } - copy(b, s) - if len(s) < len(b) { - b[len(s)] = 0 + ascii := toASCII(s) + copy(b, ascii) + if len(ascii) < len(b) { + b[len(ascii)] = 0 } } @@ -85,17 +94,27 @@ for len(s)+1 < len(b) { s = "0" + s } - tw.cString(b, s) + tw.cString(b, s, false, paxNone, nil) } // Write x into b, either as octal or as binary (GNUtar/star extension). -func (tw *Writer) numeric(b []byte, x int64) { +// If the value is too long for the field and writingPax is enabled both for the field and the add a paxheader record instead +func (tw *Writer) numeric(b []byte, x int64, allowPax bool, paxKeyword string, paxHeaders map[string]string) { // Try octal first. s := strconv.FormatInt(x, 8) if len(s) < len(b) { tw.octal(b, x) return } + + // If it is too long for octal, and pax is preferred, use a pax header + if allowPax && tw.preferPax { + tw.octal(b, 0) + s := strconv.FormatInt(x, 10) + paxHeaders[paxKeyword] = s + return + } + // Too big: use binary (big-endian). tw.usedBinary = true for i := len(b) - 1; x > 0 && i >= 0; i-- { @@ -115,6 +134,15 @@ // WriteHeader calls Flush if it is not the first header. // Calling after a Close will return ErrWriteAfterClose. func (tw *Writer) WriteHeader(hdr *Header) error { + return tw.writeHeader(hdr, true) +} + +// WriteHeader writes hdr and prepares to accept the file's contents. +// WriteHeader calls Flush if it is not the first header. +// Calling after a Close will return ErrWriteAfterClose. +// As this method is called internally by writePax header to allow it to +// suppress writing the pax header. +func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { if tw.closed { return ErrWriteAfterClose } @@ -124,31 +152,21 @@ if tw.err != nil { return tw.err } - // Decide whether or not to use PAX extensions + + // a map to hold pax header records, if any are needed + paxHeaders := make(map[string]string) + // TODO(shanemhansen): we might want to use PAX headers for // subsecond time resolution, but for now let's just capture - // the long name/long symlink use case. - suffix := hdr.Name - prefix := "" - if len(hdr.Name) > fileNameSize || len(hdr.Linkname) > fileNameSize { - var err error - prefix, suffix, err = tw.splitUSTARLongName(hdr.Name) - // Either we were unable to pack the long name into ustar format - // or the link name is too long; use PAX headers. - if err == errNameTooLong || len(hdr.Linkname) > fileNameSize { - if err := tw.writePAXHeader(hdr); err != nil { - return err - } - } else if err != nil { - return err - } - } - tw.nb = int64(hdr.Size) - tw.pad = -tw.nb & (blockSize - 1) // blockSize is a power of two + // too long fields or non ascii characters header := make([]byte, blockSize) s := slicer(header) - tw.cString(s.next(fileNameSize), suffix) + + // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax + pathHeaderBytes := s.next(fileNameSize) + + tw.cString(pathHeaderBytes, hdr.Name, true, paxPath, paxHeaders) // Handle out of range ModTime carefully. var modTime int64 @@ -156,27 +174,55 @@ modTime = hdr.ModTime.Unix() } - tw.octal(s.next(8), hdr.Mode) // 100:108 - tw.numeric(s.next(8), int64(hdr.Uid)) // 108:116 - tw.numeric(s.next(8), int64(hdr.Gid)) // 116:124 - tw.numeric(s.next(12), hdr.Size) // 124:136 - tw.numeric(s.next(12), modTime) // 136:148 - s.next(8) // chksum (148:156) - s.next(1)[0] = hdr.Typeflag // 156:157 - tw.cString(s.next(100), hdr.Linkname) // linkname (157:257) - copy(s.next(8), []byte("ustar\x0000")) // 257:265 - tw.cString(s.next(32), hdr.Uname) // 265:297 - tw.cString(s.next(32), hdr.Gname) // 297:329 - tw.numeric(s.next(8), hdr.Devmajor) // 329:337 - tw.numeric(s.next(8), hdr.Devminor) // 337:345 - tw.cString(s.next(155), prefix) // 345:500 + tw.octal(s.next(8), hdr.Mode) // 100:108 + tw.numeric(s.next(8), int64(hdr.Uid), true, paxUid, paxHeaders) // 108:116 + tw.numeric(s.next(8), int64(hdr.Gid), true, paxGid, paxHeaders) // 116:124 + tw.numeric(s.next(12), hdr.Size, true, paxSize, paxHeaders) // 124:136 + tw.numeric(s.next(12), modTime, false, paxNone, nil) // 136:148 --- consider using pax for finer granularity + s.next(8) // chksum (148:156) + s.next(1)[0] = hdr.Typeflag // 156:157 + + tw.cString(s.next(100), hdr.Linkname, true, paxLinkpath, paxHeaders) + + copy(s.next(8), []byte("ustar\x0000")) // 257:265 + tw.cString(s.next(32), hdr.Uname, true, paxUname, paxHeaders) // 265:297 + tw.cString(s.next(32), hdr.Gname, true, paxGname, paxHeaders) // 297:329 + tw.numeric(s.next(8), hdr.Devmajor, false, paxNone, nil) // 329:337 + tw.numeric(s.next(8), hdr.Devminor, false, paxNone, nil) // 337:345 + + // keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax + prefixHeaderBytes := s.next(155) + tw.cString(prefixHeaderBytes, "", false, paxNone, nil) // 345:500 prefix + // Use the GNU magic instead of POSIX magic if we used any GNU extensions. if tw.usedBinary { copy(header[257:265], []byte("ustar \x00")) } - // Use the ustar magic if we used ustar long names. - if len(prefix) > 0 { - copy(header[257:265], []byte("ustar\000")) + + _, paxPathUsed := paxHeaders[paxPath] + // try to use a ustar header when only the name is too long + if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed { + suffix := hdr.Name + prefix := "" + if len(hdr.Name) > fileNameSize && isASCII(hdr.Name) { + var err error + prefix, suffix, err = tw.splitUSTARLongName(hdr.Name) + if err == nil { + // ok we can use a ustar long name instead of pax, now correct the fields + + // remove the path field from the pax header. this will suppress the pax header + delete(paxHeaders, paxPath) + + // update the path fields + tw.cString(pathHeaderBytes, suffix, false, paxNone, nil) + tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil) + + // Use the ustar magic if we used ustar long names. + if len(prefix) > 0 { + copy(header[257:265], []byte("ustar\000")) + } + } + } } // The chksum field is terminated by a NUL and a space. @@ -190,8 +236,18 @@ return tw.err } + if len(paxHeaders) > 0 { + if !allowPax { + return errInvalidHeader + } + if err := tw.writePAXHeader(hdr, paxHeaders); err != nil { + return err + } + } + tw.nb = int64(hdr.Size) + tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize + _, tw.err = tw.w.Write(header) - return tw.err } @@ -218,7 +274,7 @@ // writePaxHeader writes an extended pax header to the // archive. -func (tw *Writer) writePAXHeader(hdr *Header) error { +func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error { // Prepare extended header ext := new(Header) ext.Typeflag = TypeXHeader @@ -229,18 +285,23 @@ // with the current pid. pid := os.Getpid() dir, file := path.Split(hdr.Name) - ext.Name = path.Join(dir, - fmt.Sprintf("PaxHeaders.%d", pid), file)[0:100] + fullName := path.Join(dir, + fmt.Sprintf("PaxHeaders.%d", pid), file) + + ascii := toASCII(fullName) + if len(ascii) > 100 { + ascii = ascii[:100] + } + ext.Name = ascii // Construct the body var buf bytes.Buffer - if len(hdr.Name) > fileNameSize { - fmt.Fprint(&buf, paxHeader("path="+hdr.Name)) + + for k, v := range paxHeaders { + fmt.Fprint(&buf, paxHeader(k+"="+v)) } - if len(hdr.Linkname) > fileNameSize { - fmt.Fprint(&buf, paxHeader("linkpath="+hdr.Linkname)) - } + ext.Size = int64(len(buf.Bytes())) - if err := tw.WriteHeader(ext); err != nil { + if err := tw.writeHeader(ext, false); err != nil { return err } if _, err := tw.Write(buf.Bytes()); err != nil { Index: src/pkg/archive/tar/writer_test.go =================================================================== --- a/src/pkg/archive/tar/writer_test.go +++ b/src/pkg/archive/tar/writer_test.go @@ -243,15 +243,110 @@ } } +func TestPaxSymlink(t *testing.T) { + // Create an archive with a large linkname + fileinfo, err := os.Stat("testdata/small.txt") + if err != nil { + t.Fatal(err) + } + hdr, err := FileInfoHeader(fileinfo, "") + hdr.Typeflag = TypeSymlink + if err != nil { + t.Fatalf("os.Stat:1 %v", err) + } + // Force a PAX long linkname to be written + longLinkname := strings.Repeat("1234567890/1234567890", 10) + hdr.Linkname = longLinkname + + hdr.Size = 0 + var buf bytes.Buffer + writer := NewWriter(&buf) + if err := writer.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + // Simple test to make sure PAX extensions are in effect + if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { + t.Fatal("Expected at least one PAX header to be written.") + } + // Test that we can get a long name back out of the archive. + reader := NewReader(&buf) + hdr, err = reader.Next() + if err != nil { + t.Fatal(err) + } + if hdr.Linkname != longLinkname { + t.Fatal("Couldn't recover long link name") + } +} + +func TestPaxNonAscii(t *testing.T) { + // Create an archive with non ascii. These should trigger a pax header + // because pax headers have a defined utf-8 encoding. + fileinfo, err := os.Stat("testdata/small.txt") + if err != nil { + t.Fatal(err) + } + + hdr, err := FileInfoHeader(fileinfo, "") + if err != nil { + t.Fatalf("os.Stat:1 %v", err) + } + + // some sample data + chineseFilename := "文件名" + chineseGroupname := "組" + chineseUsername := "用戶名" + + hdr.Name = chineseFilename + hdr.Gname = chineseGroupname + hdr.Uname = chineseUsername + + contents := strings.Repeat(" ", int(hdr.Size)) + + var buf bytes.Buffer + writer := NewWriter(&buf) + if err := writer.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err = writer.Write([]byte(contents)); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + // Simple test to make sure PAX extensions are in effect + if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { + t.Fatal("Expected at least one PAX header to be written.") + } + // Test that we can get a long name back out of the archive. + reader := NewReader(&buf) + hdr, err = reader.Next() + if err != nil { + t.Fatal(err) + } + if hdr.Name != chineseFilename { + t.Fatal("Couldn't recover unicode name") + } + if hdr.Gname != chineseGroupname { + t.Fatal("Couldn't recover unicode group") + } + if hdr.Uname != chineseUsername { + t.Fatal("Couldn't recover unicode user") + } +} + func TestPAXHeader(t *testing.T) { medName := strings.Repeat("CD", 50) longName := strings.Repeat("AB", 100) paxTests := [][2]string{ - {"name=/etc/hosts", "19 name=/etc/hosts\n"}, + {paxPath + "=/etc/hosts", "19 path=/etc/hosts\n"}, {"a=b", "6 a=b\n"}, // Single digit length {"a=names", "11 a=names\n"}, // Test case involving carries - {"name=" + longName, fmt.Sprintf("210 name=%s\n", longName)}, - {"name=" + medName, fmt.Sprintf("110 name=%s\n", medName)}} + {paxPath + "=" + longName, fmt.Sprintf("210 path=%s\n", longName)}, + {paxPath + "=" + medName, fmt.Sprintf("110 path=%s\n", medName)}} for _, test := range paxTests { key, expected := test[0], test[1]