mirror of
https://github.com/OpenListTeam/OpenList.git
synced 2026-10-10 21:13:10 +08:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ac478842ad |
+1
-1
@@ -878,7 +878,7 @@ func (d *Yun139) newRequest(url string, method string, callback base.ReqCallback
|
||||
"X-Yun-Api-Version": "v1",
|
||||
"X-Yun-App-Channel": "10000034",
|
||||
"X-Yun-Channel-Source": "10000034",
|
||||
"X-Yun-Client-Info": "||13|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||",
|
||||
"X-Yun-Client-Info": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||",
|
||||
"X-Yun-Module-Type": "100",
|
||||
"X-Yun-Svc-Type": "1",
|
||||
})
|
||||
|
||||
+1
-10
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -367,14 +366,6 @@ func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error {
|
||||
return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200)
|
||||
}
|
||||
|
||||
func familyTransferTempName(srcName string) string {
|
||||
ext := path.Ext(srcName)
|
||||
if ext == "" {
|
||||
ext = ".transfer"
|
||||
}
|
||||
return fmt.Sprintf("0%s%s", uuid.NewString(), ext)
|
||||
}
|
||||
|
||||
func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) {
|
||||
overwrite := true
|
||||
isFamily := y.isFamily()
|
||||
@@ -408,7 +399,7 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
|
||||
srcName := stream.GetName()
|
||||
stream = &WrapFileStreamer{
|
||||
FileStreamer: stream,
|
||||
Name: familyTransferTempName(srcName),
|
||||
Name: fmt.Sprintf("0%s.transfer", uuid.NewString()),
|
||||
}
|
||||
|
||||
// 使用家庭云上传
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
package _189pc
|
||||
|
||||
import (
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFamilyTransferTempNamePreservesExtension(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
want string
|
||||
}{
|
||||
{name: "photo.jpg", want: ".jpg"},
|
||||
{name: "PHOTO.PNG", want: ".PNG"},
|
||||
{name: "archive.tar.gz", want: ".gz"},
|
||||
{name: "no-extension", want: ".transfer"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := familyTransferTempName(tt.name)
|
||||
if !strings.HasPrefix(got, "0") {
|
||||
t.Fatalf("temporary name %q must start with 0", got)
|
||||
}
|
||||
if ext := path.Ext(got); ext != tt.want {
|
||||
t.Fatalf("temporary name extension = %q, want %q", ext, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadProgressKeysIncludeExtension(t *testing.T) {
|
||||
jpgKeys := uploadProgressKeys("session", "same-md5", "photo.jpg")
|
||||
txtKeys := uploadProgressKeys("session", "same-md5", "photo.txt")
|
||||
retryKeys := uploadProgressKeys("session", "same-md5", "different-name.jpg")
|
||||
|
||||
if slices.Equal(jpgKeys, txtKeys) {
|
||||
t.Fatal("files with different extensions must not share upload progress")
|
||||
}
|
||||
if !slices.Equal(jpgKeys, retryKeys) {
|
||||
t.Fatal("files with the same session, MD5, and extension should share upload progress")
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -1204,10 +1203,6 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode
|
||||
return y.fastUpload(ctx, dstDir, file, up, isFamily, overwrite, generateTorrent)
|
||||
}
|
||||
|
||||
func uploadProgressKeys(sessionKey, fileMD5, fileName string) []string {
|
||||
return []string{sessionKey, fileMD5, path.Ext(fileName)}
|
||||
}
|
||||
|
||||
func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool, generateTorrent bool) (model.Obj, error) {
|
||||
var (
|
||||
cache = file.GetFile()
|
||||
@@ -1310,8 +1305,7 @@ func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file mode
|
||||
}
|
||||
|
||||
// 尝试恢复进度
|
||||
progressKeys := uploadProgressKeys(y.getTokenInfo().SessionKey, fileMd5Hex, file.GetName())
|
||||
uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, progressKeys...)
|
||||
uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.getTokenInfo().SessionKey, fileMd5Hex)
|
||||
if !ok {
|
||||
// step.2 预上传
|
||||
params := Params{
|
||||
@@ -1379,7 +1373,7 @@ func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file mode
|
||||
if err = threadG.Wait(); err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
uploadProgress.UploadParts = utils.SliceFilter(uploadProgress.UploadParts, func(s string) bool { return s != "" })
|
||||
base.SaveUploadProgress(y, uploadProgress, progressKeys...)
|
||||
base.SaveUploadProgress(y, uploadProgress, y.getTokenInfo().SessionKey, fileMd5Hex)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -52,20 +52,16 @@ func (d *Onedrive) InitReference(refStorage driver.Driver) error {
|
||||
}
|
||||
|
||||
func (d *Onedrive) Drop(ctx context.Context) error {
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
// RootFolderPath may change before this driver instance is reinitialized.
|
||||
d.root = nil
|
||||
d.ref = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Onedrive) GetRoot(ctx context.Context) (model.Obj, error) {
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
if d.root != nil {
|
||||
return d.root, nil
|
||||
}
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
root := &Object{
|
||||
ObjThumb: model.ObjThumb{
|
||||
Object: model.Object{
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
@@ -230,14 +229,9 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj
|
||||
return nil, errs.NotImplement
|
||||
}
|
||||
path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false)
|
||||
contentType, _ := ctx.Value(conf.DirectUploadContentTypeKey).(string)
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{
|
||||
Bucket: &d.Bucket,
|
||||
Key: &path,
|
||||
ContentType: &contentType,
|
||||
Bucket: &d.Bucket,
|
||||
Key: &path,
|
||||
})
|
||||
if req == nil {
|
||||
return nil, fmt.Errorf("failed to create PutObject request")
|
||||
@@ -249,7 +243,6 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj
|
||||
return &model.HttpDirectUploadInfo{
|
||||
UploadURL: link,
|
||||
Method: "PUT",
|
||||
Headers: map[string]string{"Content-Type": contentType},
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/pkg/sftp v1.13.11
|
||||
github.com/pquerna/otp v1.5.0
|
||||
github.com/quic-go/quic-go v0.61.0
|
||||
github.com/quic-go/quic-go v0.63.0
|
||||
github.com/rclone/rclone v1.75.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.8
|
||||
github.com/sirupsen/logrus v1.10.2
|
||||
|
||||
@@ -765,6 +765,8 @@ github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+
|
||||
github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk=
|
||||
github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNvA=
|
||||
github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs=
|
||||
github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A=
|
||||
github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
|
||||
github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE=
|
||||
github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo=
|
||||
github.com/rclone/rclone v1.74.4 h1:/c6LMO2kPQjJa6a/PBFcIYJHDvfNOAtfDBJy44mGMpc=
|
||||
|
||||
@@ -197,5 +197,4 @@ const (
|
||||
PathKey
|
||||
SharingIDKey
|
||||
SkipHookKey
|
||||
DirectUploadContentTypeKey
|
||||
)
|
||||
|
||||
@@ -1,271 +0,0 @@
|
||||
package net
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// scanETag determines if a syntactically valid ETag is present at s. If so,
|
||||
// the ETag and remaining text after consuming ETag is returned. Otherwise,
|
||||
// it returns "", "".
|
||||
func scanETag(s string) (etag string, remain string) {
|
||||
s = textproto.TrimString(s)
|
||||
start := 0
|
||||
if strings.HasPrefix(s, "W/") {
|
||||
start = 2
|
||||
}
|
||||
if len(s[start:]) < 2 || s[start] != '"' {
|
||||
return "", ""
|
||||
}
|
||||
// ETag is either W/"text" or "text".
|
||||
// See RFC 7232 2.3.
|
||||
for i := start + 1; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
// Character values allowed in ETags.
|
||||
case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
|
||||
case c == '"':
|
||||
return s[:i+1], s[i+1:]
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// etagStrongMatch reports whether a and b match using strong ETag comparison.
|
||||
// Assumes a and b are valid ETags.
|
||||
func etagStrongMatch(a, b string) bool {
|
||||
return a == b && a != "" && a[0] == '"'
|
||||
}
|
||||
|
||||
// etagWeakMatch reports whether a and b match using weak ETag comparison.
|
||||
// Assumes a and b are valid ETags.
|
||||
func etagWeakMatch(a, b string) bool {
|
||||
return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
|
||||
}
|
||||
|
||||
// condResult is the result of an HTTP request precondition check.
|
||||
// See https://tools.ietf.org/html/rfc7232 section 3.
|
||||
type condResult int
|
||||
|
||||
const (
|
||||
condNone condResult = iota
|
||||
condTrue
|
||||
condFalse
|
||||
)
|
||||
|
||||
func checkIfMatch(w http.ResponseWriter, r *http.Request, exists bool) condResult {
|
||||
values := r.Header.Values("If-Match")
|
||||
if len(values) == 0 {
|
||||
return condNone
|
||||
}
|
||||
im := strings.Join(values, ",")
|
||||
r.Header.Del("If-Match")
|
||||
if !exists {
|
||||
return condFalse
|
||||
}
|
||||
for {
|
||||
im = textproto.TrimString(im)
|
||||
if len(im) == 0 {
|
||||
break
|
||||
}
|
||||
if im[0] == ',' {
|
||||
im = im[1:]
|
||||
continue
|
||||
}
|
||||
if im[0] == '*' {
|
||||
return condTrue
|
||||
}
|
||||
etag, remain := scanETag(im)
|
||||
if etag == "" {
|
||||
break
|
||||
}
|
||||
if etagStrongMatch(etag, w.Header().Get("Etag")) {
|
||||
return condTrue
|
||||
}
|
||||
im = remain
|
||||
}
|
||||
|
||||
return condFalse
|
||||
}
|
||||
|
||||
func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult {
|
||||
ius := r.Header.Get("If-Unmodified-Since")
|
||||
if ius == "" {
|
||||
return condNone
|
||||
}
|
||||
r.Header.Del("If-Unmodified-Since")
|
||||
if isZeroTime(modtime) {
|
||||
return condNone
|
||||
}
|
||||
t, err := http.ParseTime(ius)
|
||||
if err != nil {
|
||||
return condNone
|
||||
}
|
||||
|
||||
// The Last-Modified header truncates sub-second precision so
|
||||
// the modtime needs to be truncated too.
|
||||
modtime = modtime.Truncate(time.Second)
|
||||
if ret := modtime.Compare(t); ret <= 0 {
|
||||
return condTrue
|
||||
}
|
||||
return condFalse
|
||||
}
|
||||
|
||||
func checkIfNoneMatch(w http.ResponseWriter, r *http.Request, exists bool) condResult {
|
||||
values := r.Header.Values("If-None-Match")
|
||||
if len(values) == 0 {
|
||||
return condNone
|
||||
}
|
||||
inm := strings.Join(values, ",")
|
||||
r.Header.Del("If-None-Match")
|
||||
if !exists {
|
||||
return condTrue
|
||||
}
|
||||
buf := inm
|
||||
for {
|
||||
buf = textproto.TrimString(buf)
|
||||
if len(buf) == 0 {
|
||||
break
|
||||
}
|
||||
if buf[0] == ',' {
|
||||
buf = buf[1:]
|
||||
continue
|
||||
}
|
||||
if buf[0] == '*' {
|
||||
return condFalse
|
||||
}
|
||||
etag, remain := scanETag(buf)
|
||||
if etag == "" {
|
||||
break
|
||||
}
|
||||
if etagWeakMatch(etag, w.Header().Get("Etag")) {
|
||||
return condFalse
|
||||
}
|
||||
buf = remain
|
||||
}
|
||||
return condTrue
|
||||
}
|
||||
|
||||
func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult {
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
return condNone
|
||||
}
|
||||
ims := r.Header.Get("If-Modified-Since")
|
||||
if ims == "" {
|
||||
return condNone
|
||||
}
|
||||
r.Header.Del("If-Modified-Since")
|
||||
if isZeroTime(modtime) {
|
||||
return condNone
|
||||
}
|
||||
t, err := http.ParseTime(ims)
|
||||
if err != nil {
|
||||
return condNone
|
||||
}
|
||||
// The Last-Modified header truncates sub-second precision so
|
||||
// the modtime needs to be truncated too.
|
||||
modtime = modtime.Truncate(time.Second)
|
||||
if ret := modtime.Compare(t); ret <= 0 {
|
||||
return condFalse
|
||||
}
|
||||
return condTrue
|
||||
}
|
||||
|
||||
func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult {
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
return condNone
|
||||
}
|
||||
ir := r.Header.Get("If-Range")
|
||||
if ir == "" {
|
||||
return condNone
|
||||
}
|
||||
r.Header.Del("If-Range")
|
||||
etag, _ := scanETag(ir)
|
||||
if etag != "" {
|
||||
if etagStrongMatch(etag, w.Header().Get("Etag")) {
|
||||
return condTrue
|
||||
}
|
||||
return condFalse
|
||||
}
|
||||
// The If-Range value is typically the ETag value, but it may also be
|
||||
// the modtime date. See golang.org/issue/8367.
|
||||
if modtime.IsZero() {
|
||||
return condFalse
|
||||
}
|
||||
t, err := http.ParseTime(ir)
|
||||
if err != nil {
|
||||
return condFalse
|
||||
}
|
||||
if t.Unix() == modtime.Unix() {
|
||||
return condTrue
|
||||
}
|
||||
return condFalse
|
||||
}
|
||||
|
||||
var unixEpochTime = time.Unix(0, 0)
|
||||
|
||||
// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0).
|
||||
func isZeroTime(t time.Time) bool {
|
||||
return t.IsZero() || t.Equal(unixEpochTime)
|
||||
}
|
||||
|
||||
func setLastModified(w http.ResponseWriter, modtime time.Time) {
|
||||
if !isZeroTime(modtime) {
|
||||
w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
|
||||
}
|
||||
}
|
||||
|
||||
func writeNotModified(w http.ResponseWriter) {
|
||||
// RFC 7232 section 4.1:
|
||||
// a sender SHOULD NOT generate representation metadata other than the
|
||||
// above listed fields unless said metadata exists for the purpose of
|
||||
// guiding cache updates (e.g., Last-Modified might be useful if the
|
||||
// response does not have an ETag field).
|
||||
h := w.Header()
|
||||
delete(h, "Content-Type")
|
||||
delete(h, "Content-Length")
|
||||
delete(h, "Content-Encoding")
|
||||
if h.Get("Etag") != "" {
|
||||
delete(h, "Last-Modified")
|
||||
}
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
}
|
||||
|
||||
// CheckPreconditions evaluates request preconditions and reports whether a precondition
|
||||
// resulted in sending StatusNotModified or StatusPreconditionFailed. The caller must
|
||||
// set the current ETag response header and indicate whether the representation exists.
|
||||
func CheckPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time, exists bool) (done bool, rangeHeader string) {
|
||||
// Evaluate preconditions in the order specified by RFC 9110 section 13.2.2.
|
||||
ch := checkIfMatch(w, r, exists)
|
||||
if ch == condNone {
|
||||
ch = checkIfUnmodifiedSince(r, modtime)
|
||||
}
|
||||
if ch == condFalse {
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
return true, ""
|
||||
}
|
||||
switch checkIfNoneMatch(w, r, exists) {
|
||||
case condFalse:
|
||||
if r.Method == "GET" || r.Method == "HEAD" {
|
||||
writeNotModified(w)
|
||||
return true, ""
|
||||
}
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
return true, ""
|
||||
case condNone:
|
||||
if checkIfModifiedSince(r, modtime) == condFalse {
|
||||
writeNotModified(w)
|
||||
return true, ""
|
||||
}
|
||||
}
|
||||
|
||||
rangeHeader = r.Header.Get("Range")
|
||||
if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse {
|
||||
rangeHeader = ""
|
||||
}
|
||||
return false, rangeHeader
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
package net
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCheckPreconditions(t *testing.T) {
|
||||
modified := time.Date(2026, time.January, 2, 3, 4, 5, 123456789, time.UTC)
|
||||
date := modified.Format(http.TimeFormat)
|
||||
older := modified.Add(-time.Second).Format(http.TimeFormat)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
method string
|
||||
exists bool
|
||||
etag string
|
||||
header http.Header
|
||||
status int
|
||||
}{
|
||||
{"unconditional create", "PUT", false, "", nil, 0},
|
||||
{"unconditional overwrite", "PUT", true, `"current"`, nil, 0},
|
||||
{"create only missing", "PUT", false, "", http.Header{"If-None-Match": {"*"}}, 0},
|
||||
{"create only existing", "PUT", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 412},
|
||||
{"create only existing without etag", "PUT", true, "", http.Header{"If-None-Match": {"*"}}, 412},
|
||||
{"match missing", "PUT", false, "", http.Header{"If-Match": {"*"}}, 412},
|
||||
{"match existing", "PUT", true, `"current"`, http.Header{"If-Match": {"*"}}, 0},
|
||||
{"match existing without etag", "PUT", true, "", http.Header{"If-Match": {"*"}}, 0},
|
||||
{"match current", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}}, 0},
|
||||
{"match stale", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412},
|
||||
{"match missing tag", "PUT", false, "", http.Header{"If-Match": {`"current"`}}, 412},
|
||||
{"match weak request", "PUT", true, `"current"`, http.Header{"If-Match": {`W/"current"`}}, 412},
|
||||
{"match weak representation", "PUT", true, `W/"current"`, http.Header{"If-Match": {`"current"`}}, 412},
|
||||
{"match list", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale", "current"`}}, 0},
|
||||
{"match multiple lines", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`, `"current"`}}, 0},
|
||||
{"match empty list members", "PUT", true, `"current"`, http.Header{"If-Match": {`, , "current",`}}, 0},
|
||||
{"match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-Match": {`"stale", "one,two"`}}, 0},
|
||||
{"malformed match", "PUT", true, `"current"`, http.Header{"If-Match": {"current"}}, 412},
|
||||
{"empty match", "PUT", true, `"current"`, http.Header{"If-Match": {""}}, 412},
|
||||
{"none match current", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"current"`}}, 412},
|
||||
{"none match stale", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}}, 0},
|
||||
{"none match missing tag", "PUT", false, "", http.Header{"If-None-Match": {`"current"`}}, 0},
|
||||
{"none match weak request", "PUT", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 412},
|
||||
{"none match weak representation", "PUT", true, `W/"current"`, http.Header{"If-None-Match": {`"current"`}}, 412},
|
||||
{"none match multiple lines", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`, `"current"`}}, 412},
|
||||
{"none match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-None-Match": {`"stale", "one,two"`}}, 412},
|
||||
{"both conditions must pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"current"`}}, 412},
|
||||
{"both conditions pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"stale"`}}, 0},
|
||||
{"both wildcards missing", "PUT", false, "", http.Header{"If-Match": {"*"}, "If-None-Match": {"*"}}, 412},
|
||||
{"unmodified since older", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {older}}, 412},
|
||||
{"unmodified since same second", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {date}}, 0},
|
||||
{"unmodified since missing", "PUT", false, "", http.Header{"If-Unmodified-Since": {older}}, 0},
|
||||
{"invalid unmodified since", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {"invalid"}}, 0},
|
||||
{"match overrides unmodified since", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-Unmodified-Since": {older}}, 0},
|
||||
{"ignore modified since on put", "PUT", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 0},
|
||||
{"get not modified", "GET", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 304},
|
||||
{"head not modified", "HEAD", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 304},
|
||||
{"get stale match", "GET", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412},
|
||||
{"get modified since", "GET", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 304},
|
||||
{"none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}, "If-Modified-Since": {date}}, 0},
|
||||
{"empty none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {""}, "If-Modified-Since": {date}}, 0},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(tt.method, "/file.txt", nil)
|
||||
for name, values := range tt.header {
|
||||
for _, value := range values {
|
||||
r.Header.Add(name, value)
|
||||
}
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
if tt.etag != "" {
|
||||
w.Header().Set("Etag", tt.etag)
|
||||
}
|
||||
var modTime time.Time
|
||||
if tt.exists {
|
||||
modTime = modified
|
||||
}
|
||||
done, _ := CheckPreconditions(w, r, modTime, tt.exists)
|
||||
if done != (tt.status != 0) {
|
||||
t.Fatalf("done = %v, want status %d", done, tt.status)
|
||||
}
|
||||
if done && w.Code != tt.status {
|
||||
t.Errorf("status = %d, want %d", w.Code, tt.status)
|
||||
}
|
||||
if w.Body.Len() != 0 {
|
||||
t.Errorf("unexpected response body: %q", w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckPreconditionsIfRange(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
etag string
|
||||
want string
|
||||
}{
|
||||
{`"current"`, "bytes=0-3"},
|
||||
{`"stale"`, ""},
|
||||
{`W/"current"`, ""},
|
||||
} {
|
||||
t.Run(tt.etag, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodGet, "/file.txt", nil)
|
||||
r.Header.Set("Range", "bytes=0-3")
|
||||
r.Header.Set("If-Range", tt.etag)
|
||||
w := httptest.NewRecorder()
|
||||
w.Header().Set("Etag", `"current"`)
|
||||
done, got := CheckPreconditions(w, r, time.Time{}, true)
|
||||
if done || got != tt.want {
|
||||
t.Fatalf("CheckPreconditions() = (%v, %q), want (false, %q)", done, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+12
-10
@@ -49,7 +49,7 @@ import (
|
||||
// ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range.
|
||||
func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, rangeReader model.RangeReaderIF) (err error) {
|
||||
setLastModified(w, modTime)
|
||||
done, rangeReq := CheckPreconditions(w, r, modTime, true)
|
||||
done, rangeReq := checkPreconditions(w, r, modTime)
|
||||
if done {
|
||||
return nil
|
||||
}
|
||||
@@ -302,18 +302,20 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea
|
||||
}
|
||||
// TODO clean header with blocklist or passlist
|
||||
res.Header.Del("set-cookie")
|
||||
var reader io.Reader
|
||||
if res.StatusCode >= 400 {
|
||||
defer res.Body.Close()
|
||||
var reader io.Reader = res.Body
|
||||
if res.Header.Get("Content-Encoding") == "gzip" {
|
||||
gzipReader, err := gzip.NewReader(res.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http request [%s] failure,status: %w decode gzip response: %v", URL, HttpStatusCodeError(res.StatusCode), err)
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
reader = gzipReader
|
||||
// 根据 Content-Encoding 判断 Body 是否压缩
|
||||
switch res.Header.Get("Content-Encoding") {
|
||||
case "gzip":
|
||||
// 使用gzip.NewReader解压缩
|
||||
reader, _ = gzip.NewReader(res.Body)
|
||||
defer reader.(*gzip.Reader).Close()
|
||||
default:
|
||||
// 没有Content-Encoding,直接读取
|
||||
reader = res.Body
|
||||
}
|
||||
all, _ := io.ReadAll(reader)
|
||||
_ = res.Body.Close()
|
||||
msg := string(all)
|
||||
log.Debugln(msg)
|
||||
return nil, fmt.Errorf("http request [%s] failure,status: %w response:%s", URL, HttpStatusCodeError(res.StatusCode), msg)
|
||||
|
||||
@@ -4,7 +4,10 @@ import (
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
@@ -15,6 +18,260 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// scanETag determines if a syntactically valid ETag is present at s. If so,
|
||||
// the ETag and remaining text after consuming ETag is returned. Otherwise,
|
||||
// it returns "", "".
|
||||
func scanETag(s string) (etag string, remain string) {
|
||||
s = textproto.TrimString(s)
|
||||
start := 0
|
||||
if strings.HasPrefix(s, "W/") {
|
||||
start = 2
|
||||
}
|
||||
if len(s[start:]) < 2 || s[start] != '"' {
|
||||
return "", ""
|
||||
}
|
||||
// ETag is either W/"text" or "text".
|
||||
// See RFC 7232 2.3.
|
||||
for i := start + 1; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
// Character values allowed in ETags.
|
||||
case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
|
||||
case c == '"':
|
||||
return s[:i+1], s[i+1:]
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// etagStrongMatch reports whether a and b match using strong ETag comparison.
|
||||
// Assumes a and b are valid ETags.
|
||||
func etagStrongMatch(a, b string) bool {
|
||||
return a == b && a != "" && a[0] == '"'
|
||||
}
|
||||
|
||||
// etagWeakMatch reports whether a and b match using weak ETag comparison.
|
||||
// Assumes a and b are valid ETags.
|
||||
func etagWeakMatch(a, b string) bool {
|
||||
return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
|
||||
}
|
||||
|
||||
// condResult is the result of an HTTP request precondition check.
|
||||
// See https://tools.ietf.org/html/rfc7232 section 3.
|
||||
type condResult int
|
||||
|
||||
const (
|
||||
condNone condResult = iota
|
||||
condTrue
|
||||
condFalse
|
||||
)
|
||||
|
||||
func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult {
|
||||
im := r.Header.Get("If-Match")
|
||||
if im == "" {
|
||||
return condNone
|
||||
}
|
||||
r.Header.Del("If-Match")
|
||||
for {
|
||||
im = textproto.TrimString(im)
|
||||
if len(im) == 0 {
|
||||
break
|
||||
}
|
||||
if im[0] == ',' {
|
||||
im = im[1:]
|
||||
continue
|
||||
}
|
||||
if im[0] == '*' {
|
||||
return condTrue
|
||||
}
|
||||
etag, remain := scanETag(im)
|
||||
if etag == "" {
|
||||
break
|
||||
}
|
||||
if etagStrongMatch(etag, w.Header().Get("Etag")) {
|
||||
return condTrue
|
||||
}
|
||||
im = remain
|
||||
}
|
||||
|
||||
return condFalse
|
||||
}
|
||||
|
||||
func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult {
|
||||
ius := r.Header.Get("If-Unmodified-Since")
|
||||
if ius == "" {
|
||||
return condNone
|
||||
}
|
||||
r.Header.Del("If-Unmodified-Since")
|
||||
if isZeroTime(modtime) {
|
||||
return condNone
|
||||
}
|
||||
t, err := http.ParseTime(ius)
|
||||
if err != nil {
|
||||
return condNone
|
||||
}
|
||||
|
||||
// The Last-Modified header truncates sub-second precision so
|
||||
// the modtime needs to be truncated too.
|
||||
modtime = modtime.Truncate(time.Second)
|
||||
if ret := modtime.Compare(t); ret <= 0 {
|
||||
return condTrue
|
||||
}
|
||||
return condFalse
|
||||
}
|
||||
|
||||
func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult {
|
||||
inm := r.Header.Get("If-None-Match")
|
||||
if inm == "" {
|
||||
return condNone
|
||||
}
|
||||
r.Header.Del("If-None-Match")
|
||||
buf := inm
|
||||
for {
|
||||
buf = textproto.TrimString(buf)
|
||||
if len(buf) == 0 {
|
||||
break
|
||||
}
|
||||
if buf[0] == ',' {
|
||||
buf = buf[1:]
|
||||
continue
|
||||
}
|
||||
if buf[0] == '*' {
|
||||
return condFalse
|
||||
}
|
||||
etag, remain := scanETag(buf)
|
||||
if etag == "" {
|
||||
break
|
||||
}
|
||||
if etagWeakMatch(etag, w.Header().Get("Etag")) {
|
||||
return condFalse
|
||||
}
|
||||
buf = remain
|
||||
}
|
||||
return condTrue
|
||||
}
|
||||
|
||||
func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult {
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
return condNone
|
||||
}
|
||||
ims := r.Header.Get("If-Modified-Since")
|
||||
if ims == "" {
|
||||
return condNone
|
||||
}
|
||||
r.Header.Del("If-Modified-Since")
|
||||
if isZeroTime(modtime) {
|
||||
return condNone
|
||||
}
|
||||
t, err := http.ParseTime(ims)
|
||||
if err != nil {
|
||||
return condNone
|
||||
}
|
||||
// The Last-Modified header truncates sub-second precision so
|
||||
// the modtime needs to be truncated too.
|
||||
modtime = modtime.Truncate(time.Second)
|
||||
if ret := modtime.Compare(t); ret <= 0 {
|
||||
return condFalse
|
||||
}
|
||||
return condTrue
|
||||
}
|
||||
|
||||
func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult {
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
return condNone
|
||||
}
|
||||
ir := r.Header.Get("If-Range")
|
||||
if ir == "" {
|
||||
return condNone
|
||||
}
|
||||
r.Header.Del("If-Range")
|
||||
etag, _ := scanETag(ir)
|
||||
if etag != "" {
|
||||
if etagStrongMatch(etag, w.Header().Get("Etag")) {
|
||||
return condTrue
|
||||
}
|
||||
return condFalse
|
||||
}
|
||||
// The If-Range value is typically the ETag value, but it may also be
|
||||
// the modtime date. See golang.org/issue/8367.
|
||||
if modtime.IsZero() {
|
||||
return condFalse
|
||||
}
|
||||
t, err := http.ParseTime(ir)
|
||||
if err != nil {
|
||||
return condFalse
|
||||
}
|
||||
if t.Unix() == modtime.Unix() {
|
||||
return condTrue
|
||||
}
|
||||
return condFalse
|
||||
}
|
||||
|
||||
var unixEpochTime = time.Unix(0, 0)
|
||||
|
||||
// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0).
|
||||
func isZeroTime(t time.Time) bool {
|
||||
return t.IsZero() || t.Equal(unixEpochTime)
|
||||
}
|
||||
|
||||
func setLastModified(w http.ResponseWriter, modtime time.Time) {
|
||||
if !isZeroTime(modtime) {
|
||||
w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
|
||||
}
|
||||
}
|
||||
|
||||
func writeNotModified(w http.ResponseWriter) {
|
||||
// RFC 7232 section 4.1:
|
||||
// a sender SHOULD NOT generate representation metadata other than the
|
||||
// above listed fields unless said metadata exists for the purpose of
|
||||
// guiding cache updates (e.g., Last-Modified might be useful if the
|
||||
// response does not have an ETag field).
|
||||
h := w.Header()
|
||||
delete(h, "Content-Type")
|
||||
delete(h, "Content-Length")
|
||||
delete(h, "Content-Encoding")
|
||||
if h.Get("Etag") != "" {
|
||||
delete(h, "Last-Modified")
|
||||
}
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
}
|
||||
|
||||
// checkPreconditions evaluates request preconditions and reports whether a precondition
|
||||
// resulted in sending StatusNotModified or StatusPreconditionFailed.
|
||||
func checkPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time) (done bool, rangeHeader string) {
|
||||
// This function carefully follows RFC 7232 section 6.
|
||||
ch := checkIfMatch(w, r)
|
||||
if ch == condNone {
|
||||
ch = checkIfUnmodifiedSince(r, modtime)
|
||||
}
|
||||
if ch == condFalse {
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
return true, ""
|
||||
}
|
||||
switch checkIfNoneMatch(w, r) {
|
||||
case condFalse:
|
||||
if r.Method == "GET" || r.Method == "HEAD" {
|
||||
writeNotModified(w)
|
||||
return true, ""
|
||||
}
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
return true, ""
|
||||
case condNone:
|
||||
if checkIfModifiedSince(r, modtime) == condFalse {
|
||||
writeNotModified(w)
|
||||
return true, ""
|
||||
}
|
||||
}
|
||||
|
||||
rangeHeader = r.Header.Get("Range")
|
||||
if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse {
|
||||
rangeHeader = ""
|
||||
}
|
||||
return false, rangeHeader
|
||||
}
|
||||
|
||||
func sumRangesSize(ranges []http_range.Range) (size int64) {
|
||||
for _, ra := range ranges {
|
||||
size += ra.Length
|
||||
|
||||
@@ -237,16 +237,14 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error {
|
||||
return nil
|
||||
}
|
||||
storageDriver, err := GetStorageByMountPath(oldStorage.MountPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get storage driver")
|
||||
}
|
||||
// Storage settings may change which objects are exposed at the same mount
|
||||
// path, so cached entries must be discarded before reinitializing the driver.
|
||||
Cache.DeleteDirectoryTree(storageDriver, "/")
|
||||
Cache.InvalidateStorageDetails(storageDriver)
|
||||
if oldStorage.MountPath != storage.MountPath {
|
||||
// mount path renamed, need to drop the storage
|
||||
storagesMap.Delete(oldStorage.MountPath)
|
||||
Cache.DeleteDirectoryTree(storageDriver, "/")
|
||||
Cache.InvalidateStorageDetails(storageDriver)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get storage driver")
|
||||
}
|
||||
err = storageDriver.Drop(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
func TestGetEtagStableAcrossCacheRefresh(t *testing.T) {
|
||||
uploaded := &model.Object{Size: 4, Modified: time.Unix(1700000000, 123456789)}
|
||||
before := GetEtag(uploaded, uploaded.Size)
|
||||
refreshed := &model.Object{Size: uploaded.Size, Modified: time.Unix(1700000000, 0)}
|
||||
if got := GetEtag(refreshed, refreshed.Size); got != before {
|
||||
t.Fatalf("refreshed file ETag = %q, want %q", got, before)
|
||||
}
|
||||
refreshed.Modified = refreshed.Modified.Add(time.Second)
|
||||
if got := GetEtag(refreshed, refreshed.Size); got == before {
|
||||
t.Errorf("changed modification second retained ETag %q", got)
|
||||
}
|
||||
refreshed.Modified = uploaded.Modified
|
||||
if got := GetEtag(refreshed, refreshed.Size+1); got == before {
|
||||
t.Errorf("changed size retained ETag %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetEtagContentHash(t *testing.T) {
|
||||
file := &model.Object{
|
||||
Size: 4,
|
||||
HashInfo: utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("data"))),
|
||||
}
|
||||
before := GetEtag(file, file.Size)
|
||||
file.Modified = time.Unix(1700000000, 100)
|
||||
if got := GetEtag(file, file.Size); got != before {
|
||||
t.Errorf("unchanged content ETag = %q, want %q", got, before)
|
||||
}
|
||||
file.HashInfo = utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("next")))
|
||||
if got := GetEtag(file, file.Size); got == before {
|
||||
t.Errorf("changed content retained ETag %q", got)
|
||||
}
|
||||
}
|
||||
@@ -93,8 +93,7 @@ func GetEtag(file model.Obj, size int64) string {
|
||||
if len(hash) > 0 {
|
||||
return fmt.Sprintf(`"%s"`, hash)
|
||||
}
|
||||
// Storage backends may not preserve sub-second timestamps across cache
|
||||
// refreshes, so only use the reproducible second precision here.
|
||||
// 参考nginx
|
||||
return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), size)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package handles
|
||||
|
||||
import (
|
||||
"context"
|
||||
"mime"
|
||||
"net/url"
|
||||
stdpath "path"
|
||||
|
||||
@@ -17,11 +15,10 @@ import (
|
||||
)
|
||||
|
||||
type FsGetDirectUploadInfoReq struct {
|
||||
Path string `json:"path" form:"path"`
|
||||
FileName string `json:"file_name" form:"file_name"`
|
||||
FileSize int64 `json:"file_size" form:"file_size"`
|
||||
ContentType string `json:"content_type" form:"content_type"`
|
||||
Tool string `json:"tool" form:"tool"`
|
||||
Path string `json:"path" form:"path"`
|
||||
FileName string `json:"file_name" form:"file_name"`
|
||||
FileSize int64 `json:"file_size" form:"file_size"`
|
||||
Tool string `json:"tool" form:"tool"`
|
||||
}
|
||||
|
||||
// FsGetDirectUploadInfo returns the direct upload info if supported by the driver
|
||||
@@ -94,14 +91,7 @@ func FsGetDirectUploadInfo(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if req.ContentType != "" {
|
||||
if _, _, err := mime.ParseMediaType(req.ContentType); err != nil {
|
||||
common.ErrorResp(c, err, 400)
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx := context.WithValue(c, conf.DirectUploadContentTypeKey, req.ContentType)
|
||||
directUploadInfo, err := fs.GetDirectUploadInfo(ctx, req.Tool, path, req.FileName, req.FileSize, overwrite)
|
||||
directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize, overwrite)
|
||||
if err != nil {
|
||||
if !overwrite && errs.IsObjectAlreadyExists(err) {
|
||||
common.ErrorStrResp(c, "file exists", 403)
|
||||
|
||||
+4
-20
@@ -10,7 +10,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/gofakes3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -128,13 +127,9 @@ func (b *s3Backend) walkPage(
|
||||
return page.addContent(item), nil
|
||||
}
|
||||
|
||||
// Emit entries in the order their keys have in a flat S3 keyspace. A
|
||||
// directory sorts as if it carried its trailing slash, so "b.txt" comes
|
||||
// before "b/..." as it does in a real bucket. Without this a listing
|
||||
// resumed from a marker could silently skip or repeat keys.
|
||||
dirEntries = slices.Clone(dirEntries)
|
||||
sort.Slice(dirEntries, func(i, j int) bool {
|
||||
return listSortKey(dirEntries[i]) < listSortKey(dirEntries[j])
|
||||
return dirEntries[i].GetName() < dirEntries[j].GetName()
|
||||
})
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
@@ -148,16 +143,14 @@ func (b *s3Backend) walkPage(
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
// S3 common prefixes include the delimiter, e.g. "base/20261009T020008/".
|
||||
// Clients such as barman-cloud ignore prefixes without it.
|
||||
subtreePrefix := objectPath + "/"
|
||||
if addPrefix {
|
||||
// response.AddPrefix(gofakes3.URLEncode(subtreePrefix))
|
||||
if !page.addPrefix(subtreePrefix) {
|
||||
// response.AddPrefix(gofakes3.URLEncode(objectPath))
|
||||
if !page.addPrefix(objectPath) {
|
||||
return false, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
subtreePrefix := objectPath + "/"
|
||||
// A marker beyond this subtree lets us avoid an upstream directory read.
|
||||
if subtreePrefix <= page.marker && !strings.HasPrefix(page.marker, subtreePrefix) {
|
||||
continue
|
||||
@@ -182,12 +175,3 @@ func (b *s3Backend) walkPage(
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// listSortKey returns the name used to order directory entries so that the
|
||||
// emitted keys follow S3 lexicographic order.
|
||||
func listSortKey(entry model.Obj) string {
|
||||
if entry.IsDir() {
|
||||
return entry.GetName() + "/"
|
||||
}
|
||||
return entry.GetName()
|
||||
}
|
||||
|
||||
+5
-70
@@ -4,9 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
@@ -125,10 +123,10 @@ func TestListPagePaginatesContentsAndCommonPrefixesTogether(t *testing.T) {
|
||||
if len(first.Contents) != 1 || first.Contents[0].Key != "data/a" {
|
||||
t.Fatalf("first contents = %#v, want data/a", first.Contents)
|
||||
}
|
||||
if len(first.CommonPrefixes) != 1 || first.CommonPrefixes[0].Prefix != "data/b/" {
|
||||
t.Fatalf("first prefixes = %#v, want data/b/", first.CommonPrefixes)
|
||||
if len(first.CommonPrefixes) != 1 || first.CommonPrefixes[0].Prefix != "data/b" {
|
||||
t.Fatalf("first prefixes = %#v, want data/b", first.CommonPrefixes)
|
||||
}
|
||||
if !first.IsTruncated || first.NextMarker != "data/b/" {
|
||||
if !first.IsTruncated || first.NextMarker != "data/b" {
|
||||
t.Fatalf("first page truncated = %v, next marker = %q", first.IsTruncated, first.NextMarker)
|
||||
}
|
||||
|
||||
@@ -143,8 +141,8 @@ func TestListPagePaginatesContentsAndCommonPrefixesTogether(t *testing.T) {
|
||||
if len(second.Contents) != 1 || second.Contents[0].Key != "data/c" {
|
||||
t.Fatalf("second contents = %#v, want data/c", second.Contents)
|
||||
}
|
||||
if len(second.CommonPrefixes) != 1 || second.CommonPrefixes[0].Prefix != "data/d/" {
|
||||
t.Fatalf("second prefixes = %#v, want data/d/", second.CommonPrefixes)
|
||||
if len(second.CommonPrefixes) != 1 || second.CommonPrefixes[0].Prefix != "data/d" {
|
||||
t.Fatalf("second prefixes = %#v, want data/d", second.CommonPrefixes)
|
||||
}
|
||||
if second.IsTruncated || second.NextMarker != "" {
|
||||
t.Fatalf("second page truncated = %v, next marker = %q", second.IsTruncated, second.NextMarker)
|
||||
@@ -281,66 +279,3 @@ func TestListPageDoesNotReorderDirectoryCacheEntries(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPageMixedDirectoryAndFilePaginatesInKeyOrder(t *testing.T) {
|
||||
tree := map[string][]model.Obj{
|
||||
"bucket/data": {
|
||||
&model.Object{Name: "c", Size: 1},
|
||||
&model.Object{Name: "b", IsFolder: true},
|
||||
&model.Object{Name: "b.txt", Size: 1},
|
||||
&model.Object{Name: "b-c", IsFolder: true},
|
||||
&model.Object{Name: "a", Size: 1},
|
||||
},
|
||||
"bucket/data/b": {&model.Object{Name: "y", Size: 1}, &model.Object{Name: "x", Size: 1}},
|
||||
"bucket/data/b-c": {&model.Object{Name: "k", Size: 1}},
|
||||
}
|
||||
b := &s3Backend{
|
||||
listDir: func(_ context.Context, dir string) ([]model.Obj, error) {
|
||||
return tree[dir], nil
|
||||
},
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
delimiter bool
|
||||
want []string
|
||||
}{
|
||||
{"delimiter", true, []string{"data/a", "data/b-c/", "data/b.txt", "data/b/", "data/c"}},
|
||||
{"recursive", false, []string{"data/a", "data/b-c/k", "data/b.txt", "data/b/x", "data/b/y", "data/c"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if !sort.StringsAreSorted(tt.want) {
|
||||
t.Fatalf("expectation is not in key order: %v", tt.want)
|
||||
}
|
||||
for maxKeys := int64(1); maxKeys <= int64(len(tt.want))+1; maxKeys++ {
|
||||
var got []string
|
||||
page := gofakes3.ListBucketPage{MaxKeys: maxKeys}
|
||||
for i := 0; i <= len(tt.want); i++ {
|
||||
list, err := b.listPage(context.Background(), "bucket", "data", "", tt.delimiter, page)
|
||||
if err != nil {
|
||||
t.Fatalf("MaxKeys=%d: listPage() error = %v", maxKeys, err)
|
||||
}
|
||||
var keys []string
|
||||
for _, item := range list.Contents {
|
||||
keys = append(keys, item.Key)
|
||||
}
|
||||
for _, prefix := range list.CommonPrefixes {
|
||||
if !strings.HasSuffix(prefix.Prefix, "/") {
|
||||
t.Fatalf("common prefix %q does not end with /", prefix.Prefix)
|
||||
}
|
||||
keys = append(keys, prefix.Prefix)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
got = append(got, keys...)
|
||||
if !list.IsTruncated {
|
||||
break
|
||||
}
|
||||
page.Marker, page.HasMarker = list.NextMarker, true
|
||||
}
|
||||
if !slices.Equal(got, tt.want) {
|
||||
t.Fatalf("MaxKeys=%d: listing = %v, want %v", maxKeys, got, tt.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+5
-32
@@ -355,6 +355,8 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
|
||||
return status, err
|
||||
}
|
||||
defer release()
|
||||
// TODO(rost): Support the If-Match, If-None-Match headers? See bradfitz'
|
||||
// comments in http.checkEtag.
|
||||
ctx := r.Context()
|
||||
user := ctx.Value(conf.UserKey).(*model.User)
|
||||
reqPath, err = user.JoinPath(reqPath)
|
||||
@@ -392,30 +394,6 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
|
||||
if !common.CanWrite(user, parentMeta, parentPath) {
|
||||
return http.StatusForbidden, errs.PermissionDenied
|
||||
}
|
||||
fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
|
||||
exists := err == nil
|
||||
if err != nil && !errs.IsObjectNotFound(err) {
|
||||
if errs.IsNotFoundError(err) {
|
||||
return http.StatusNotFound, err
|
||||
}
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
var modTime time.Time
|
||||
if exists {
|
||||
if fi.IsDir() {
|
||||
return http.StatusMethodNotAllowed, nil
|
||||
}
|
||||
etag, err := findETag(ctx, h.LockSystem, reqPath, fi)
|
||||
if err != nil {
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
w.Header().Set("Etag", etag)
|
||||
modTime = fi.ModTime()
|
||||
}
|
||||
if done, _ := net.CheckPreconditions(w, r, modTime, exists); done {
|
||||
return 0, nil
|
||||
}
|
||||
w.Header().Del("Etag")
|
||||
fsStream := &stream.FileStream{
|
||||
Obj: &obj,
|
||||
Reader: r.Body,
|
||||
@@ -433,21 +411,16 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
|
||||
if err != nil {
|
||||
return http.StatusMethodNotAllowed, err
|
||||
}
|
||||
fi, err = fs.Get(ctx, reqPath, &fs.GetArgs{})
|
||||
fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
|
||||
if err != nil {
|
||||
return http.StatusInternalServerError, err
|
||||
fi = &obj
|
||||
}
|
||||
etag, err := findETag(ctx, h.LockSystem, reqPath, fi)
|
||||
if err != nil {
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
w.Header().Set("Etag", etag)
|
||||
if exists {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
return 0, nil
|
||||
return http.StatusCreated, nil
|
||||
}
|
||||
|
||||
func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status int, err error) {
|
||||
|
||||
Reference in New Issue
Block a user