mirror of
https://github.com/OpenListTeam/OpenList.git
synced 2026-10-10 21:13:10 +08:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 136e107463 | |||
| 7b4d4454a8 | |||
| 23164ac7f2 |
@@ -10,7 +10,7 @@ require (
|
||||
github.com/KarpelesLab/reflink v1.0.2
|
||||
github.com/KirCute/zip v1.0.1
|
||||
github.com/OpenListTeam/go-cache v0.1.0
|
||||
github.com/OpenListTeam/gofakes3 v0.8.1
|
||||
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911144636-404e34e0f5af
|
||||
github.com/OpenListTeam/sftpd-openlist v1.0.1
|
||||
github.com/OpenListTeam/tache v0.2.2
|
||||
github.com/OpenListTeam/times v0.1.0
|
||||
|
||||
@@ -51,8 +51,8 @@ github.com/OpenListTeam/115-sdk-go v0.2.6 h1:ehXyStvncvn4qRBuknor3kyGZtUmHc0+stj
|
||||
github.com/OpenListTeam/115-sdk-go v0.2.6/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU=
|
||||
github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI=
|
||||
github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4=
|
||||
github.com/OpenListTeam/gofakes3 v0.8.1 h1:uihJ7Zgb4qIafFcXhcm71BzxCyGRIqBVJYg4YOUa6uY=
|
||||
github.com/OpenListTeam/gofakes3 v0.8.1/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U=
|
||||
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911144636-404e34e0f5af h1:pcORNNaOgbc28g9YliwPc5mpEmGf/dKyZjkXhHZgvVo=
|
||||
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911144636-404e34e0f5af/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U=
|
||||
github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs=
|
||||
github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s=
|
||||
github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef h1:67uGHancMF/abMrnkc8abVUWQiG73Wk5d8CKt3RzkFo=
|
||||
|
||||
+65
-24
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,17 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/OpenListTeam/gofakes3"
|
||||
)
|
||||
|
||||
func TestCopyObjectMissingSourceReturnsNoSuchKey(t *testing.T) {
|
||||
b, _ := setupMultipartBackend(t)
|
||||
|
||||
_, err := b.CopyObject(context.Background(), "mp", "missing.txt", "mp", "copy.txt", nil)
|
||||
if code := s3ErrorCode(err); code != gofakes3.ErrNoSuchKey {
|
||||
t.Fatalf("CopyObject() error = %v, want NoSuchKey", code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
type objectMetadata struct {
|
||||
headers map[string]string
|
||||
hash []byte
|
||||
etag string
|
||||
}
|
||||
|
||||
// Only reuse upload metadata when it still describes the current content.
|
||||
func (b *s3Backend) loadMetadata(name string, hash []byte) (map[string]string, string) {
|
||||
if value, ok := b.meta.Load(name); ok {
|
||||
metadata := value.(objectMetadata)
|
||||
if bytes.Equal(metadata.hash, hash) {
|
||||
return metadata.headers, metadata.etag
|
||||
}
|
||||
}
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
// Metadata alone cannot identify content: clients can preserve both file size
|
||||
// and modification time when overwriting an object. Use the driver's MD5 when
|
||||
// available, otherwise hash the complete content, including for HEAD and ranges.
|
||||
func getObjectHash(ctx context.Context, name string, obj model.Obj) ([]byte, error) {
|
||||
if value := obj.GetHash().GetHash(utils.MD5); value != "" {
|
||||
hash, err := hex.DecodeString(value)
|
||||
if err != nil || len(hash) != md5.Size {
|
||||
return nil, fmt.Errorf("invalid object MD5: %q", value)
|
||||
}
|
||||
return hash, nil
|
||||
}
|
||||
link, file, err := fs.Link(ctx, name, model.LinkArgs{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer link.Close()
|
||||
size := link.ContentLength
|
||||
if size <= 0 {
|
||||
size = file.GetSize()
|
||||
}
|
||||
ranges, err := stream.GetRangeReaderFromLink(size, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := ranges.RangeRead(ctx, http_range.Range{Length: -1})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
hash := md5.New()
|
||||
n, err := utils.CopyWithBuffer(hash, reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n != size {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
return hash.Sum(nil), nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
func TestObjectHashFromDriver(t *testing.T) {
|
||||
sum := md5.Sum([]byte("content"))
|
||||
obj := &model.Object{HashInfo: utils.NewHashInfo(utils.MD5, hex.EncodeToString(sum[:]))}
|
||||
hash, err := getObjectHash(context.Background(), "object", obj)
|
||||
if err != nil || hex.EncodeToString(hash) != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("hash = %x, err = %v", hash, err)
|
||||
}
|
||||
for _, value := range []string{"invalid", "ab"} {
|
||||
obj.HashInfo = utils.NewHashInfo(utils.MD5, value)
|
||||
if _, err := getObjectHash(context.Background(), "object", obj); err == nil {
|
||||
t.Fatalf("accepted invalid MD5 %q", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartMetadataMatchesCurrentContent(t *testing.T) {
|
||||
b := &s3Backend{meta: new(sync.Map)}
|
||||
oldHash := md5.Sum([]byte("old"))
|
||||
newHash := md5.Sum([]byte("new"))
|
||||
b.meta.Store("object", objectMetadata{
|
||||
headers: map[string]string{"Content-Type": "text/plain"},
|
||||
hash: oldHash[:],
|
||||
etag: `"multipart-2"`,
|
||||
})
|
||||
meta, etag := b.loadMetadata("object", oldHash[:])
|
||||
if etag != `"multipart-2"` || meta["Content-Type"] != "text/plain" {
|
||||
t.Fatal("lost metadata for unchanged multipart content")
|
||||
}
|
||||
meta, etag = b.loadMetadata("object", newHash[:])
|
||||
if meta != nil || etag != "" {
|
||||
t.Fatal("reused a multipart validator after a same-size content change")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionalRequestsDoNotRedirect(t *testing.T) {
|
||||
for _, method := range []string{"GET", "PUT"} {
|
||||
for _, name := range []string{"If-Match", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since", "If-Range"} {
|
||||
for _, value := range []string{"*", ""} {
|
||||
r := httptest.NewRequest(method, "/bucket/object", nil)
|
||||
r.Header.Set(name, value)
|
||||
if url, ok := directObjectURL(r, nil); ok || url != "" {
|
||||
t.Fatalf("redirected %s with %s", method, name)
|
||||
}
|
||||
if url, ok := directUploadURL(r, nil); ok || url != "" {
|
||||
t.Fatalf("redirected %s with %s", method, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -234,7 +234,9 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucket, object
|
||||
|
||||
defer combined.Close()
|
||||
|
||||
err := b.putStream(ctx, bucket, object, state.meta, combined, total)
|
||||
sum := md5.Sum(concat)
|
||||
etag := fmt.Sprintf("%q", fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:]), len(ordered)))
|
||||
err := b.putStream(ctx, bucket, object, state.meta, combined, total, etag)
|
||||
if err != nil {
|
||||
// Leave the upload in place so the client may retry completion, per
|
||||
// the gofakes3 MultipartBackend contract.
|
||||
@@ -244,8 +246,6 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucket, object
|
||||
// Success: drop bookkeeping and clean up part files.
|
||||
b.removeUpload(uploadID)
|
||||
|
||||
sum := md5.Sum(concat)
|
||||
etag := fmt.Sprintf("%q", fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:]), len(ordered)))
|
||||
log.Debugf("s3 multipart: completed upload %s -> %s/%s (%d bytes)", uploadID, bucket, object, total)
|
||||
return "", etag, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package s3
|
||||
|
||||
import "net/http"
|
||||
|
||||
func hasPreconditions(r *http.Request) bool {
|
||||
for _, name := range []string{"If-Match", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since", "If-Range"} {
|
||||
if _, ok := r.Header[name]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func redirectHandler(next http.Handler, authPairs map[string]string) http.Handle
|
||||
}
|
||||
|
||||
func directObjectURL(r *http.Request, authPairs map[string]string) (string, bool) {
|
||||
if r.Method != http.MethodGet {
|
||||
if r.Method != http.MethodGet || hasPreconditions(r) {
|
||||
return "", false
|
||||
}
|
||||
if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) {
|
||||
@@ -75,7 +75,7 @@ func directObjectURL(r *http.Request, authPairs map[string]string) (string, bool
|
||||
}
|
||||
|
||||
func directUploadURL(r *http.Request, authPairs map[string]string) (string, bool) {
|
||||
if r.Method != http.MethodPut || r.ContentLength < 0 {
|
||||
if r.Method != http.MethodPut || r.ContentLength < 0 || hasPreconditions(r) {
|
||||
return "", false
|
||||
}
|
||||
if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) {
|
||||
|
||||
Reference in New Issue
Block a user