Compare commits

...

3 Commits

Author SHA1 Message Date
jyxjjj 136e107463 build(deps): update gofakes3 to merged commit
- Pin gofakes3 to the master merge commit for conditional object writes
- Replace the previous PR head checksums with the merged version checksums

Co-authored-by: Codex <267193182+codex@users.noreply.github.com>
Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com>
2026-09-20 01:05:06 +08:00
jyxjjj 7b4d4454a8 fix(s3): return NoSuchKey for missing copy sources
- Translate missing source errors during both CopyObject reads
- Add regression coverage for copying a nonexistent source key

Co-authored-by: Codex <267193182+codex@users.noreply.github.com>
Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com>
2026-09-20 01:01:01 +08:00
jyxjjj 23164ac7f2 fix(s3): enforce conditional writes with valid object ETags
- Pin gofakes3 with conditional PUT, multipart completion, and copy support
- Derive object ETags from content and preserve matching multipart validators
- Keep conditional requests in the S3 handler instead of redirecting them
- Format modification times in UTC and propagate storage errors
- Close content readers and discard metadata after successful deletion
- Add regression tests for object hashes, metadata, and conditional redirects

Co-authored-by: Codex <267193182+codex@users.noreply.github.com>
Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com>
2026-09-10 23:32:20 +08:00
9 changed files with 239 additions and 32 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ require (
github.com/KarpelesLab/reflink v1.0.2
github.com/KirCute/zip v1.0.1
github.com/OpenListTeam/go-cache v0.1.0
github.com/OpenListTeam/gofakes3 v0.8.1
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911144636-404e34e0f5af
github.com/OpenListTeam/sftpd-openlist v1.0.1
github.com/OpenListTeam/tache v0.2.2
github.com/OpenListTeam/times v0.1.0
+2 -2
View File
@@ -51,8 +51,8 @@ github.com/OpenListTeam/115-sdk-go v0.2.6 h1:ehXyStvncvn4qRBuknor3kyGZtUmHc0+stj
github.com/OpenListTeam/115-sdk-go v0.2.6/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU=
github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI=
github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4=
github.com/OpenListTeam/gofakes3 v0.8.1 h1:uihJ7Zgb4qIafFcXhcm71BzxCyGRIqBVJYg4YOUa6uY=
github.com/OpenListTeam/gofakes3 v0.8.1/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U=
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911144636-404e34e0f5af h1:pcORNNaOgbc28g9YliwPc5mpEmGf/dKyZjkXhHZgvVo=
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911144636-404e34e0f5af/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U=
github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs=
github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s=
github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef h1:67uGHancMF/abMrnkc8abVUWQiG73Wk5d8CKt3RzkFo=
+65 -24
View File
File diff suppressed because it is too large Load Diff
+17
View File
@@ -0,0 +1,17 @@
package s3
import (
"context"
"testing"
"github.com/OpenListTeam/gofakes3"
)
func TestCopyObjectMissingSourceReturnsNoSuchKey(t *testing.T) {
b, _ := setupMultipartBackend(t)
_, err := b.CopyObject(context.Background(), "mp", "missing.txt", "mp", "copy.txt", nil)
if code := s3ErrorCode(err); code != gofakes3.ErrNoSuchKey {
t.Fatalf("CopyObject() error = %v, want NoSuchKey", code)
}
}
+73
View File
@@ -0,0 +1,73 @@
package s3
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
type objectMetadata struct {
headers map[string]string
hash []byte
etag string
}
// Only reuse upload metadata when it still describes the current content.
func (b *s3Backend) loadMetadata(name string, hash []byte) (map[string]string, string) {
if value, ok := b.meta.Load(name); ok {
metadata := value.(objectMetadata)
if bytes.Equal(metadata.hash, hash) {
return metadata.headers, metadata.etag
}
}
return nil, ""
}
// Metadata alone cannot identify content: clients can preserve both file size
// and modification time when overwriting an object. Use the driver's MD5 when
// available, otherwise hash the complete content, including for HEAD and ranges.
func getObjectHash(ctx context.Context, name string, obj model.Obj) ([]byte, error) {
if value := obj.GetHash().GetHash(utils.MD5); value != "" {
hash, err := hex.DecodeString(value)
if err != nil || len(hash) != md5.Size {
return nil, fmt.Errorf("invalid object MD5: %q", value)
}
return hash, nil
}
link, file, err := fs.Link(ctx, name, model.LinkArgs{})
if err != nil {
return nil, err
}
defer link.Close()
size := link.ContentLength
if size <= 0 {
size = file.GetSize()
}
ranges, err := stream.GetRangeReaderFromLink(size, link)
if err != nil {
return nil, err
}
reader, err := ranges.RangeRead(ctx, http_range.Range{Length: -1})
if err != nil {
return nil, err
}
defer reader.Close()
hash := md5.New()
n, err := utils.CopyWithBuffer(hash, reader)
if err != nil {
return nil, err
}
if n != size {
return nil, io.ErrUnexpectedEOF
}
return hash.Sum(nil), nil
}
+64
View File
@@ -0,0 +1,64 @@
package s3
import (
"context"
"crypto/md5"
"encoding/hex"
"net/http/httptest"
"sync"
"testing"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
func TestObjectHashFromDriver(t *testing.T) {
sum := md5.Sum([]byte("content"))
obj := &model.Object{HashInfo: utils.NewHashInfo(utils.MD5, hex.EncodeToString(sum[:]))}
hash, err := getObjectHash(context.Background(), "object", obj)
if err != nil || hex.EncodeToString(hash) != hex.EncodeToString(sum[:]) {
t.Fatalf("hash = %x, err = %v", hash, err)
}
for _, value := range []string{"invalid", "ab"} {
obj.HashInfo = utils.NewHashInfo(utils.MD5, value)
if _, err := getObjectHash(context.Background(), "object", obj); err == nil {
t.Fatalf("accepted invalid MD5 %q", value)
}
}
}
func TestMultipartMetadataMatchesCurrentContent(t *testing.T) {
b := &s3Backend{meta: new(sync.Map)}
oldHash := md5.Sum([]byte("old"))
newHash := md5.Sum([]byte("new"))
b.meta.Store("object", objectMetadata{
headers: map[string]string{"Content-Type": "text/plain"},
hash: oldHash[:],
etag: `"multipart-2"`,
})
meta, etag := b.loadMetadata("object", oldHash[:])
if etag != `"multipart-2"` || meta["Content-Type"] != "text/plain" {
t.Fatal("lost metadata for unchanged multipart content")
}
meta, etag = b.loadMetadata("object", newHash[:])
if meta != nil || etag != "" {
t.Fatal("reused a multipart validator after a same-size content change")
}
}
func TestConditionalRequestsDoNotRedirect(t *testing.T) {
for _, method := range []string{"GET", "PUT"} {
for _, name := range []string{"If-Match", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since", "If-Range"} {
for _, value := range []string{"*", ""} {
r := httptest.NewRequest(method, "/bucket/object", nil)
r.Header.Set(name, value)
if url, ok := directObjectURL(r, nil); ok || url != "" {
t.Fatalf("redirected %s with %s", method, name)
}
if url, ok := directUploadURL(r, nil); ok || url != "" {
t.Fatalf("redirected %s with %s", method, name)
}
}
}
}
}
+3 -3
View File
@@ -234,7 +234,9 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucket, object
defer combined.Close()
err := b.putStream(ctx, bucket, object, state.meta, combined, total)
sum := md5.Sum(concat)
etag := fmt.Sprintf("%q", fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:]), len(ordered)))
err := b.putStream(ctx, bucket, object, state.meta, combined, total, etag)
if err != nil {
// Leave the upload in place so the client may retry completion, per
// the gofakes3 MultipartBackend contract.
@@ -244,8 +246,6 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucket, object
// Success: drop bookkeeping and clean up part files.
b.removeUpload(uploadID)
sum := md5.Sum(concat)
etag := fmt.Sprintf("%q", fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:]), len(ordered)))
log.Debugf("s3 multipart: completed upload %s -> %s/%s (%d bytes)", uploadID, bucket, object, total)
return "", etag, nil
}
+12
View File
@@ -0,0 +1,12 @@
package s3
import "net/http"
func hasPreconditions(r *http.Request) bool {
for _, name := range []string{"If-Match", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since", "If-Range"} {
if _, ok := r.Header[name]; ok {
return true
}
}
return false
}
+2 -2
View File
@@ -38,7 +38,7 @@ func redirectHandler(next http.Handler, authPairs map[string]string) http.Handle
}
func directObjectURL(r *http.Request, authPairs map[string]string) (string, bool) {
if r.Method != http.MethodGet {
if r.Method != http.MethodGet || hasPreconditions(r) {
return "", false
}
if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) {
@@ -75,7 +75,7 @@ func directObjectURL(r *http.Request, authPairs map[string]string) (string, bool
}
func directUploadURL(r *http.Request, authPairs map[string]string) (string, bool) {
if r.Method != http.MethodPut || r.ContentLength < 0 {
if r.Method != http.MethodPut || r.ContentLength < 0 || hasPreconditions(r) {
return "", false
}
if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) {