fix(net): handle gzip reader error on error response decoding (#3179)

- Defer closing response body when handling HTTP error responses
- Check error when initializing `gzip.Reader` to prevent panics from invalid gzip streams
- Avoid unsafe type assertion during gzip reader cleanup

drivers/139/util.go

@@ -878,7 +878,7 @@ func (d *Yun139) newRequest(url string, method string, callback base.ReqCallback
 		"X-Yun-Api-Version":    "v1",
 		"X-Yun-App-Channel":    "10000034",
 		"X-Yun-Channel-Source": "10000034",
-		"X-Yun-Client-Info":    "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||",
+		"X-Yun-Client-Info":    "||13|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||",
 		"X-Yun-Module-Type":    "100",
 		"X-Yun-Svc-Type":       "1",
 	})

drivers/189pc/driver.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -366,6 +367,14 @@ func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error {
 	return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200)
 }
 
+func familyTransferTempName(srcName string) string {
+	ext := path.Ext(srcName)
+	if ext == "" {
+		ext = ".transfer"
+	}
+	return fmt.Sprintf("0%s%s", uuid.NewString(), ext)
+}
+
 func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) {
 	overwrite := true
 	isFamily := y.isFamily()
@@ -399,7 +408,7 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
 		srcName := stream.GetName()
 		stream = &WrapFileStreamer{
 			FileStreamer: stream,
-			Name:         fmt.Sprintf("0%s.transfer", uuid.NewString()),
+			Name:         familyTransferTempName(srcName),
 		}
 
 		// 使用家庭云上传

drivers/189pc/driver_test.go

@@ -0,0 +1,45 @@
+package _189pc
+
+import (
+	"path"
+	"slices"
+	"strings"
+	"testing"
+)
+
+func TestFamilyTransferTempNamePreservesExtension(t *testing.T) {
+	tests := []struct {
+		name string
+		want string
+	}{
+		{name: "photo.jpg", want: ".jpg"},
+		{name: "PHOTO.PNG", want: ".PNG"},
+		{name: "archive.tar.gz", want: ".gz"},
+		{name: "no-extension", want: ".transfer"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := familyTransferTempName(tt.name)
+			if !strings.HasPrefix(got, "0") {
+				t.Fatalf("temporary name %q must start with 0", got)
+			}
+			if ext := path.Ext(got); ext != tt.want {
+				t.Fatalf("temporary name extension = %q, want %q", ext, tt.want)
+			}
+		})
+	}
+}
+
+func TestUploadProgressKeysIncludeExtension(t *testing.T) {
+	jpgKeys := uploadProgressKeys("session", "same-md5", "photo.jpg")
+	txtKeys := uploadProgressKeys("session", "same-md5", "photo.txt")
+	retryKeys := uploadProgressKeys("session", "same-md5", "different-name.jpg")
+
+	if slices.Equal(jpgKeys, txtKeys) {
+		t.Fatal("files with different extensions must not share upload progress")
+	}
+	if !slices.Equal(jpgKeys, retryKeys) {
+		t.Fatal("files with the same session, MD5, and extension should share upload progress")
+	}
+}

drivers/189pc/utils.go

@@ -14,6 +14,7 @@ import (
 	"net/http/cookiejar"
 	"net/url"
 	"os"
+	"path"
 	"regexp"
 	"sort"
 	"strconv"
@@ -1203,6 +1204,10 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode
 	return y.fastUpload(ctx, dstDir, file, up, isFamily, overwrite, generateTorrent)
 }
 
+func uploadProgressKeys(sessionKey, fileMD5, fileName string) []string {
+	return []string{sessionKey, fileMD5, path.Ext(fileName)}
+}
+
 func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool, generateTorrent bool) (model.Obj, error) {
 	var (
 		cache = file.GetFile()
@@ -1305,7 +1310,8 @@ func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file mode
 	}
 
 	// 尝试恢复进度
-	uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.getTokenInfo().SessionKey, fileMd5Hex)
+	progressKeys := uploadProgressKeys(y.getTokenInfo().SessionKey, fileMd5Hex, file.GetName())
+	uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, progressKeys...)
 	if !ok {
 		// step.2 预上传
 		params := Params{
@@ -1373,7 +1379,7 @@ func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file mode
 		if err = threadG.Wait(); err != nil {
 			if errors.Is(err, context.Canceled) {
 				uploadProgress.UploadParts = utils.SliceFilter(uploadProgress.UploadParts, func(s string) bool { return s != "" })
-				base.SaveUploadProgress(y, uploadProgress, y.getTokenInfo().SessionKey, fileMd5Hex)
+				base.SaveUploadProgress(y, uploadProgress, progressKeys...)
 			}
 			return nil, err
 		}

drivers/onedrive/driver.go

@@ -52,16 +52,20 @@ func (d *Onedrive) InitReference(refStorage driver.Driver) error {
 }
 
 func (d *Onedrive) Drop(ctx context.Context) error {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	// RootFolderPath may change before this driver instance is reinitialized.
+	d.root = nil
 	d.ref = nil
 	return nil
 }
 
 func (d *Onedrive) GetRoot(ctx context.Context) (model.Obj, error) {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
 	if d.root != nil {
 		return d.root, nil
 	}
-	d.mutex.Lock()
-	defer d.mutex.Unlock()
 	root := &Object{
 		ObjThumb: model.ObjThumb{
 			Object: model.Object{

drivers/s3/driver.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/OpenListTeam/OpenList/v4/internal/conf"
 	"github.com/OpenListTeam/OpenList/v4/internal/driver"
 	"github.com/OpenListTeam/OpenList/v4/internal/errs"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -229,9 +230,14 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj
 		return nil, errs.NotImplement
 	}
 	path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false)
+	contentType, _ := ctx.Value(conf.DirectUploadContentTypeKey).(string)
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
 	req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{
-		Bucket: &d.Bucket,
-		Key:    &path,
+		Bucket:      &d.Bucket,
+		Key:         &path,
+		ContentType: &contentType,
 	})
 	if req == nil {
 		return nil, fmt.Errorf("failed to create PutObject request")
@@ -243,6 +249,7 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj
 	return &model.HttpDirectUploadInfo{
 		UploadURL: link,
 		Method:    "PUT",
+		Headers:   map[string]string{"Content-Type": contentType},
 	}, nil
 }
 

go.mod

@@ -36,7 +36,6 @@ require (
 	github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
 	github.com/disintegration/imaging v1.6.2
 	github.com/dlclark/regexp2 v1.12.0
-	github.com/dlclark/regexp2/v2 v2.8.4
 	github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564
 	github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6
 	github.com/foxxorcat/mopan-sdk-go v0.1.6

go.sum

@@ -341,7 +341,6 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8=
 github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/dlclark/regexp2/v2 v2.8.4/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
 github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
 github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=

internal/conf/const.go

@@ -197,4 +197,5 @@ const (
 	PathKey
 	SharingIDKey
 	SkipHookKey
+	DirectUploadContentTypeKey
 )

internal/net/preconditions_test.go

@@ -0,0 +1,114 @@
+package net
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestCheckPreconditions(t *testing.T) {
+	modified := time.Date(2026, time.January, 2, 3, 4, 5, 123456789, time.UTC)
+	date := modified.Format(http.TimeFormat)
+	older := modified.Add(-time.Second).Format(http.TimeFormat)
+	for _, tt := range []struct {
+		name   string
+		method string
+		exists bool
+		etag   string
+		header http.Header
+		status int
+	}{
+		{"unconditional create", "PUT", false, "", nil, 0},
+		{"unconditional overwrite", "PUT", true, `"current"`, nil, 0},
+		{"create only missing", "PUT", false, "", http.Header{"If-None-Match": {"*"}}, 0},
+		{"create only existing", "PUT", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 412},
+		{"create only existing without etag", "PUT", true, "", http.Header{"If-None-Match": {"*"}}, 412},
+		{"match missing", "PUT", false, "", http.Header{"If-Match": {"*"}}, 412},
+		{"match existing", "PUT", true, `"current"`, http.Header{"If-Match": {"*"}}, 0},
+		{"match existing without etag", "PUT", true, "", http.Header{"If-Match": {"*"}}, 0},
+		{"match current", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}}, 0},
+		{"match stale", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412},
+		{"match missing tag", "PUT", false, "", http.Header{"If-Match": {`"current"`}}, 412},
+		{"match weak request", "PUT", true, `"current"`, http.Header{"If-Match": {`W/"current"`}}, 412},
+		{"match weak representation", "PUT", true, `W/"current"`, http.Header{"If-Match": {`"current"`}}, 412},
+		{"match list", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale", "current"`}}, 0},
+		{"match multiple lines", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`, `"current"`}}, 0},
+		{"match empty list members", "PUT", true, `"current"`, http.Header{"If-Match": {`, , "current",`}}, 0},
+		{"match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-Match": {`"stale", "one,two"`}}, 0},
+		{"malformed match", "PUT", true, `"current"`, http.Header{"If-Match": {"current"}}, 412},
+		{"empty match", "PUT", true, `"current"`, http.Header{"If-Match": {""}}, 412},
+		{"none match current", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"current"`}}, 412},
+		{"none match stale", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}}, 0},
+		{"none match missing tag", "PUT", false, "", http.Header{"If-None-Match": {`"current"`}}, 0},
+		{"none match weak request", "PUT", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 412},
+		{"none match weak representation", "PUT", true, `W/"current"`, http.Header{"If-None-Match": {`"current"`}}, 412},
+		{"none match multiple lines", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`, `"current"`}}, 412},
+		{"none match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-None-Match": {`"stale", "one,two"`}}, 412},
+		{"both conditions must pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"current"`}}, 412},
+		{"both conditions pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"stale"`}}, 0},
+		{"both wildcards missing", "PUT", false, "", http.Header{"If-Match": {"*"}, "If-None-Match": {"*"}}, 412},
+		{"unmodified since older", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {older}}, 412},
+		{"unmodified since same second", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {date}}, 0},
+		{"unmodified since missing", "PUT", false, "", http.Header{"If-Unmodified-Since": {older}}, 0},
+		{"invalid unmodified since", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {"invalid"}}, 0},
+		{"match overrides unmodified since", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-Unmodified-Since": {older}}, 0},
+		{"ignore modified since on put", "PUT", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 0},
+		{"get not modified", "GET", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 304},
+		{"head not modified", "HEAD", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 304},
+		{"get stale match", "GET", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412},
+		{"get modified since", "GET", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 304},
+		{"none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}, "If-Modified-Since": {date}}, 0},
+		{"empty none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {""}, "If-Modified-Since": {date}}, 0},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			r := httptest.NewRequest(tt.method, "/file.txt", nil)
+			for name, values := range tt.header {
+				for _, value := range values {
+					r.Header.Add(name, value)
+				}
+			}
+			w := httptest.NewRecorder()
+			if tt.etag != "" {
+				w.Header().Set("Etag", tt.etag)
+			}
+			var modTime time.Time
+			if tt.exists {
+				modTime = modified
+			}
+			done, _ := CheckPreconditions(w, r, modTime, tt.exists)
+			if done != (tt.status != 0) {
+				t.Fatalf("done = %v, want status %d", done, tt.status)
+			}
+			if done && w.Code != tt.status {
+				t.Errorf("status = %d, want %d", w.Code, tt.status)
+			}
+			if w.Body.Len() != 0 {
+				t.Errorf("unexpected response body: %q", w.Body.String())
+			}
+		})
+	}
+}
+
+func TestCheckPreconditionsIfRange(t *testing.T) {
+	for _, tt := range []struct {
+		etag string
+		want string
+	}{
+		{`"current"`, "bytes=0-3"},
+		{`"stale"`, ""},
+		{`W/"current"`, ""},
+	} {
+		t.Run(tt.etag, func(t *testing.T) {
+			r := httptest.NewRequest(http.MethodGet, "/file.txt", nil)
+			r.Header.Set("Range", "bytes=0-3")
+			r.Header.Set("If-Range", tt.etag)
+			w := httptest.NewRecorder()
+			w.Header().Set("Etag", `"current"`)
+			done, got := CheckPreconditions(w, r, time.Time{}, true)
+			if done || got != tt.want {
+				t.Fatalf("CheckPreconditions() = (%v, %q), want (false, %q)", done, got, tt.want)
+			}
+		})
+	}
+}

internal/net/serve.go

@@ -49,7 +49,7 @@ import (
 // ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range.
 func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, rangeReader model.RangeReaderIF) (err error) {
 	setLastModified(w, modTime)
-	done, rangeReq := checkPreconditions(w, r, modTime)
+	done, rangeReq := CheckPreconditions(w, r, modTime, true)
 	if done {
 		return nil
 	}
@@ -302,20 +302,18 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea
 	}
 	// TODO clean header with blocklist or passlist
 	res.Header.Del("set-cookie")
-	var reader io.Reader
 	if res.StatusCode >= 400 {
-		// 根据 Content-Encoding 判断 Body 是否压缩
-		switch res.Header.Get("Content-Encoding") {
-		case "gzip":
-			// 使用gzip.NewReader解压缩
-			reader, _ = gzip.NewReader(res.Body)
-			defer reader.(*gzip.Reader).Close()
-		default:
-			// 没有Content-Encoding，直接读取
-			reader = res.Body
+		defer res.Body.Close()
+		var reader io.Reader = res.Body
+		if res.Header.Get("Content-Encoding") == "gzip" {
+			gzipReader, err := gzip.NewReader(res.Body)
+			if err != nil {
+				return nil, fmt.Errorf("http request [%s] failure,status: %w decode gzip response: %v", URL, HttpStatusCodeError(res.StatusCode), err)
+			}
+			defer gzipReader.Close()
+			reader = gzipReader
 		}
 		all, _ := io.ReadAll(reader)
-		_ = res.Body.Close()
 		msg := string(all)
 		log.Debugln(msg)
 		return nil, fmt.Errorf("http request [%s] failure,status: %w response:%s", URL, HttpStatusCodeError(res.StatusCode), msg)

internal/op/storage.go

@@ -237,14 +237,16 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error {
 		return nil
 	}
 	storageDriver, err := GetStorageByMountPath(oldStorage.MountPath)
+	if err != nil {
+		return errors.WithMessage(err, "failed get storage driver")
+	}
+	// Storage settings may change which objects are exposed at the same mount
+	// path, so cached entries must be discarded before reinitializing the driver.
+	Cache.DeleteDirectoryTree(storageDriver, "/")
+	Cache.InvalidateStorageDetails(storageDriver)
 	if oldStorage.MountPath != storage.MountPath {
 		// mount path renamed, need to drop the storage
 		storagesMap.Delete(oldStorage.MountPath)
-		Cache.DeleteDirectoryTree(storageDriver, "/")
-		Cache.InvalidateStorageDetails(storageDriver)
-	}
-	if err != nil {
-		return errors.WithMessage(err, "failed get storage driver")
 	}
 	err = storageDriver.Drop(ctx)
 	if err != nil {

server/common/etag_test.go

@@ -0,0 +1,42 @@
+package common
+
+import (
+	"testing"
+	"time"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+)
+
+func TestGetEtagStableAcrossCacheRefresh(t *testing.T) {
+	uploaded := &model.Object{Size: 4, Modified: time.Unix(1700000000, 123456789)}
+	before := GetEtag(uploaded, uploaded.Size)
+	refreshed := &model.Object{Size: uploaded.Size, Modified: time.Unix(1700000000, 0)}
+	if got := GetEtag(refreshed, refreshed.Size); got != before {
+		t.Fatalf("refreshed file ETag = %q, want %q", got, before)
+	}
+	refreshed.Modified = refreshed.Modified.Add(time.Second)
+	if got := GetEtag(refreshed, refreshed.Size); got == before {
+		t.Errorf("changed modification second retained ETag %q", got)
+	}
+	refreshed.Modified = uploaded.Modified
+	if got := GetEtag(refreshed, refreshed.Size+1); got == before {
+		t.Errorf("changed size retained ETag %q", got)
+	}
+}
+
+func TestGetEtagContentHash(t *testing.T) {
+	file := &model.Object{
+		Size:     4,
+		HashInfo: utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("data"))),
+	}
+	before := GetEtag(file, file.Size)
+	file.Modified = time.Unix(1700000000, 100)
+	if got := GetEtag(file, file.Size); got != before {
+		t.Errorf("unchanged content ETag = %q, want %q", got, before)
+	}
+	file.HashInfo = utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("next")))
+	if got := GetEtag(file, file.Size); got == before {
+		t.Errorf("changed content retained ETag %q", got)
+	}
+}

server/common/proxy.go

@@ -93,7 +93,8 @@ func GetEtag(file model.Obj, size int64) string {
 	if len(hash) > 0 {
 		return fmt.Sprintf(`"%s"`, hash)
 	}
-	// 参考nginx
+	// Storage backends may not preserve sub-second timestamps across cache
+	// refreshes, so only use the reproducible second precision here.
 	return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), size)
 }
 

server/handles/direct_upload.go

@@ -1,6 +1,8 @@
 package handles
 
 import (
+	"context"
+	"mime"
 	"net/url"
 	stdpath "path"
 
@@ -15,10 +17,11 @@ import (
 )
 
 type FsGetDirectUploadInfoReq struct {
-	Path     string `json:"path" form:"path"`
-	FileName string `json:"file_name" form:"file_name"`
-	FileSize int64  `json:"file_size" form:"file_size"`
-	Tool     string `json:"tool" form:"tool"`
+	Path        string `json:"path" form:"path"`
+	FileName    string `json:"file_name" form:"file_name"`
+	FileSize    int64  `json:"file_size" form:"file_size"`
+	ContentType string `json:"content_type" form:"content_type"`
+	Tool        string `json:"tool" form:"tool"`
 }
 
 // FsGetDirectUploadInfo returns the direct upload info if supported by the driver
@@ -91,7 +94,14 @@ func FsGetDirectUploadInfo(c *gin.Context) {
 			return
 		}
 	}
-	directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize, overwrite)
+	if req.ContentType != "" {
+		if _, _, err := mime.ParseMediaType(req.ContentType); err != nil {
+			common.ErrorResp(c, err, 400)
+			return
+		}
+	}
+	ctx := context.WithValue(c, conf.DirectUploadContentTypeKey, req.ContentType)
+	directUploadInfo, err := fs.GetDirectUploadInfo(ctx, req.Tool, path, req.FileName, req.FileSize, overwrite)
 	if err != nil {
 		if !overwrite && errs.IsObjectAlreadyExists(err) {
 			common.ErrorStrResp(c, "file exists", 403)

server/s3/list.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
 	"github.com/OpenListTeam/gofakes3"
 	log "github.com/sirupsen/logrus"
 )
@@ -127,9 +128,13 @@ func (b *s3Backend) walkPage(
 		return page.addContent(item), nil
 	}
 
+	// Emit entries in the order their keys have in a flat S3 keyspace. A
+	// directory sorts as if it carried its trailing slash, so "b.txt" comes
+	// before "b/..." as it does in a real bucket. Without this a listing
+	// resumed from a marker could silently skip or repeat keys.
 	dirEntries = slices.Clone(dirEntries)
 	sort.Slice(dirEntries, func(i, j int) bool {
-		return dirEntries[i].GetName() < dirEntries[j].GetName()
+		return listSortKey(dirEntries[i]) < listSortKey(dirEntries[j])
 	})
 
 	for _, entry := range dirEntries {
@@ -143,14 +148,16 @@ func (b *s3Backend) walkPage(
 		}
 
 		if entry.IsDir() {
+			// S3 common prefixes include the delimiter, e.g. "base/20261009T020008/".
+			// Clients such as barman-cloud ignore prefixes without it.
+			subtreePrefix := objectPath + "/"
 			if addPrefix {
-				// response.AddPrefix(gofakes3.URLEncode(objectPath))
-				if !page.addPrefix(objectPath) {
+				// response.AddPrefix(gofakes3.URLEncode(subtreePrefix))
+				if !page.addPrefix(subtreePrefix) {
 					return false, nil
 				}
 				continue
 			}
-			subtreePrefix := objectPath + "/"
 			// A marker beyond this subtree lets us avoid an upstream directory read.
 			if subtreePrefix <= page.marker && !strings.HasPrefix(page.marker, subtreePrefix) {
 				continue
@@ -175,3 +182,12 @@ func (b *s3Backend) walkPage(
 	}
 	return true, nil
 }
+
+// listSortKey returns the name used to order directory entries so that the
+// emitted keys follow S3 lexicographic order.
+func listSortKey(entry model.Obj) string {
+	if entry.IsDir() {
+		return entry.GetName() + "/"
+	}
+	return entry.GetName()
+}

server/s3/list_test.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -123,10 +125,10 @@ func TestListPagePaginatesContentsAndCommonPrefixesTogether(t *testing.T) {
 	if len(first.Contents) != 1 || first.Contents[0].Key != "data/a" {
 		t.Fatalf("first contents = %#v, want data/a", first.Contents)
 	}
-	if len(first.CommonPrefixes) != 1 || first.CommonPrefixes[0].Prefix != "data/b" {
-		t.Fatalf("first prefixes = %#v, want data/b", first.CommonPrefixes)
+	if len(first.CommonPrefixes) != 1 || first.CommonPrefixes[0].Prefix != "data/b/" {
+		t.Fatalf("first prefixes = %#v, want data/b/", first.CommonPrefixes)
 	}
-	if !first.IsTruncated || first.NextMarker != "data/b" {
+	if !first.IsTruncated || first.NextMarker != "data/b/" {
 		t.Fatalf("first page truncated = %v, next marker = %q", first.IsTruncated, first.NextMarker)
 	}
 
@@ -141,8 +143,8 @@ func TestListPagePaginatesContentsAndCommonPrefixesTogether(t *testing.T) {
 	if len(second.Contents) != 1 || second.Contents[0].Key != "data/c" {
 		t.Fatalf("second contents = %#v, want data/c", second.Contents)
 	}
-	if len(second.CommonPrefixes) != 1 || second.CommonPrefixes[0].Prefix != "data/d" {
-		t.Fatalf("second prefixes = %#v, want data/d", second.CommonPrefixes)
+	if len(second.CommonPrefixes) != 1 || second.CommonPrefixes[0].Prefix != "data/d/" {
+		t.Fatalf("second prefixes = %#v, want data/d/", second.CommonPrefixes)
 	}
 	if second.IsTruncated || second.NextMarker != "" {
 		t.Fatalf("second page truncated = %v, next marker = %q", second.IsTruncated, second.NextMarker)
@@ -279,3 +281,66 @@ func TestListPageDoesNotReorderDirectoryCacheEntries(t *testing.T) {
 		}
 	}
 }
+
+func TestListPageMixedDirectoryAndFilePaginatesInKeyOrder(t *testing.T) {
+	tree := map[string][]model.Obj{
+		"bucket/data": {
+			&model.Object{Name: "c", Size: 1},
+			&model.Object{Name: "b", IsFolder: true},
+			&model.Object{Name: "b.txt", Size: 1},
+			&model.Object{Name: "b-c", IsFolder: true},
+			&model.Object{Name: "a", Size: 1},
+		},
+		"bucket/data/b":   {&model.Object{Name: "y", Size: 1}, &model.Object{Name: "x", Size: 1}},
+		"bucket/data/b-c": {&model.Object{Name: "k", Size: 1}},
+	}
+	b := &s3Backend{
+		listDir: func(_ context.Context, dir string) ([]model.Obj, error) {
+			return tree[dir], nil
+		},
+	}
+	tests := []struct {
+		name      string
+		delimiter bool
+		want      []string
+	}{
+		{"delimiter", true, []string{"data/a", "data/b-c/", "data/b.txt", "data/b/", "data/c"}},
+		{"recursive", false, []string{"data/a", "data/b-c/k", "data/b.txt", "data/b/x", "data/b/y", "data/c"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if !sort.StringsAreSorted(tt.want) {
+				t.Fatalf("expectation is not in key order: %v", tt.want)
+			}
+			for maxKeys := int64(1); maxKeys <= int64(len(tt.want))+1; maxKeys++ {
+				var got []string
+				page := gofakes3.ListBucketPage{MaxKeys: maxKeys}
+				for i := 0; i <= len(tt.want); i++ {
+					list, err := b.listPage(context.Background(), "bucket", "data", "", tt.delimiter, page)
+					if err != nil {
+						t.Fatalf("MaxKeys=%d: listPage() error = %v", maxKeys, err)
+					}
+					var keys []string
+					for _, item := range list.Contents {
+						keys = append(keys, item.Key)
+					}
+					for _, prefix := range list.CommonPrefixes {
+						if !strings.HasSuffix(prefix.Prefix, "/") {
+							t.Fatalf("common prefix %q does not end with /", prefix.Prefix)
+						}
+						keys = append(keys, prefix.Prefix)
+					}
+					sort.Strings(keys)
+					got = append(got, keys...)
+					if !list.IsTruncated {
+						break
+					}
+					page.Marker, page.HasMarker = list.NextMarker, true
+				}
+				if !slices.Equal(got, tt.want) {
+					t.Fatalf("MaxKeys=%d: listing = %v, want %v", maxKeys, got, tt.want)
+				}
+			}
+		})
+	}
+}

server/webdav/webdav.go

@@ -355,8 +355,6 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
 		return status, err
 	}
 	defer release()
-	// TODO(rost): Support the If-Match, If-None-Match headers? See bradfitz'
-	// comments in http.checkEtag.
 	ctx := r.Context()
 	user := ctx.Value(conf.UserKey).(*model.User)
 	reqPath, err = user.JoinPath(reqPath)
@@ -394,6 +392,30 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
 	if !common.CanWrite(user, parentMeta, parentPath) {
 		return http.StatusForbidden, errs.PermissionDenied
 	}
+	fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
+	exists := err == nil
+	if err != nil && !errs.IsObjectNotFound(err) {
+		if errs.IsNotFoundError(err) {
+			return http.StatusNotFound, err
+		}
+		return http.StatusInternalServerError, err
+	}
+	var modTime time.Time
+	if exists {
+		if fi.IsDir() {
+			return http.StatusMethodNotAllowed, nil
+		}
+		etag, err := findETag(ctx, h.LockSystem, reqPath, fi)
+		if err != nil {
+			return http.StatusInternalServerError, err
+		}
+		w.Header().Set("Etag", etag)
+		modTime = fi.ModTime()
+	}
+	if done, _ := net.CheckPreconditions(w, r, modTime, exists); done {
+		return 0, nil
+	}
+	w.Header().Del("Etag")
 	fsStream := &stream.FileStream{
 		Obj:      &obj,
 		Reader:   r.Body,
@@ -411,16 +433,21 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
 	if err != nil {
 		return http.StatusMethodNotAllowed, err
 	}
-	fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
+	fi, err = fs.Get(ctx, reqPath, &fs.GetArgs{})
 	if err != nil {
-		fi = &obj
+		return http.StatusInternalServerError, err
 	}
 	etag, err := findETag(ctx, h.LockSystem, reqPath, fi)
 	if err != nil {
 		return http.StatusInternalServerError, err
 	}
 	w.Header().Set("Etag", etag)
-	return http.StatusCreated, nil
+	if exists {
+		w.WriteHeader(http.StatusNoContent)
+	} else {
+		w.WriteHeader(http.StatusCreated)
+	}
+	return 0, nil
 }
 
 func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status int, err error) {