Compare commits

..

1 Commits

Author SHA1 Message Date
renovate[bot] ac478842ad fix(deps): update module github.com/quic-go/quic-go to v0.63.0 2026-10-05 18:39:56 +00:00
20 changed files with 305 additions and 664 deletions
+1 -1
View File
@@ -878,7 +878,7 @@ func (d *Yun139) newRequest(url string, method string, callback base.ReqCallback
"X-Yun-Api-Version": "v1",
"X-Yun-App-Channel": "10000034",
"X-Yun-Channel-Source": "10000034",
"X-Yun-Client-Info": "||13|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||",
"X-Yun-Client-Info": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||",
"X-Yun-Module-Type": "100",
"X-Yun-Svc-Type": "1",
})
+1 -10
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"net/http"
"path"
"strconv"
"strings"
"time"
@@ -367,14 +366,6 @@ func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error {
return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200)
}
func familyTransferTempName(srcName string) string {
ext := path.Ext(srcName)
if ext == "" {
ext = ".transfer"
}
return fmt.Sprintf("0%s%s", uuid.NewString(), ext)
}
func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) {
overwrite := true
isFamily := y.isFamily()
@@ -408,7 +399,7 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
srcName := stream.GetName()
stream = &WrapFileStreamer{
FileStreamer: stream,
Name: familyTransferTempName(srcName),
Name: fmt.Sprintf("0%s.transfer", uuid.NewString()),
}
// 使用家庭云上传
-45
View File
@@ -1,45 +0,0 @@
package _189pc
import (
"path"
"slices"
"strings"
"testing"
)
func TestFamilyTransferTempNamePreservesExtension(t *testing.T) {
tests := []struct {
name string
want string
}{
{name: "photo.jpg", want: ".jpg"},
{name: "PHOTO.PNG", want: ".PNG"},
{name: "archive.tar.gz", want: ".gz"},
{name: "no-extension", want: ".transfer"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := familyTransferTempName(tt.name)
if !strings.HasPrefix(got, "0") {
t.Fatalf("temporary name %q must start with 0", got)
}
if ext := path.Ext(got); ext != tt.want {
t.Fatalf("temporary name extension = %q, want %q", ext, tt.want)
}
})
}
}
func TestUploadProgressKeysIncludeExtension(t *testing.T) {
jpgKeys := uploadProgressKeys("session", "same-md5", "photo.jpg")
txtKeys := uploadProgressKeys("session", "same-md5", "photo.txt")
retryKeys := uploadProgressKeys("session", "same-md5", "different-name.jpg")
if slices.Equal(jpgKeys, txtKeys) {
t.Fatal("files with different extensions must not share upload progress")
}
if !slices.Equal(jpgKeys, retryKeys) {
t.Fatal("files with the same session, MD5, and extension should share upload progress")
}
}
+2 -8
View File
@@ -14,7 +14,6 @@ import (
"net/http/cookiejar"
"net/url"
"os"
"path"
"regexp"
"sort"
"strconv"
@@ -1204,10 +1203,6 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode
return y.fastUpload(ctx, dstDir, file, up, isFamily, overwrite, generateTorrent)
}
func uploadProgressKeys(sessionKey, fileMD5, fileName string) []string {
return []string{sessionKey, fileMD5, path.Ext(fileName)}
}
func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool, generateTorrent bool) (model.Obj, error) {
var (
cache = file.GetFile()
@@ -1310,8 +1305,7 @@ func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file mode
}
// 尝试恢复进度
progressKeys := uploadProgressKeys(y.getTokenInfo().SessionKey, fileMd5Hex, file.GetName())
uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, progressKeys...)
uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.getTokenInfo().SessionKey, fileMd5Hex)
if !ok {
// step.2 预上传
params := Params{
@@ -1379,7 +1373,7 @@ func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file mode
if err = threadG.Wait(); err != nil {
if errors.Is(err, context.Canceled) {
uploadProgress.UploadParts = utils.SliceFilter(uploadProgress.UploadParts, func(s string) bool { return s != "" })
base.SaveUploadProgress(y, uploadProgress, progressKeys...)
base.SaveUploadProgress(y, uploadProgress, y.getTokenInfo().SessionKey, fileMd5Hex)
}
return nil, err
}
+2 -6
View File
@@ -52,20 +52,16 @@ func (d *Onedrive) InitReference(refStorage driver.Driver) error {
}
func (d *Onedrive) Drop(ctx context.Context) error {
d.mutex.Lock()
defer d.mutex.Unlock()
// RootFolderPath may change before this driver instance is reinitialized.
d.root = nil
d.ref = nil
return nil
}
func (d *Onedrive) GetRoot(ctx context.Context) (model.Obj, error) {
d.mutex.Lock()
defer d.mutex.Unlock()
if d.root != nil {
return d.root, nil
}
d.mutex.Lock()
defer d.mutex.Unlock()
root := &Object{
ObjThumb: model.ObjThumb{
Object: model.Object{
+2 -9
View File
@@ -9,7 +9,6 @@ import (
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -230,14 +229,9 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj
return nil, errs.NotImplement
}
path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false)
contentType, _ := ctx.Value(conf.DirectUploadContentTypeKey).(string)
if contentType == "" {
contentType = "application/octet-stream"
}
req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{
Bucket: &d.Bucket,
Key: &path,
ContentType: &contentType,
Bucket: &d.Bucket,
Key: &path,
})
if req == nil {
return nil, fmt.Errorf("failed to create PutObject request")
@@ -249,7 +243,6 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj
return &model.HttpDirectUploadInfo{
UploadURL: link,
Method: "PUT",
Headers: map[string]string{"Content-Type": contentType},
}, nil
}
+1 -1
View File
@@ -63,7 +63,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/pkg/sftp v1.13.11
github.com/pquerna/otp v1.5.0
github.com/quic-go/quic-go v0.61.0
github.com/quic-go/quic-go v0.63.0
github.com/rclone/rclone v1.75.1
github.com/shirou/gopsutil/v4 v4.26.8
github.com/sirupsen/logrus v1.10.2
+2
View File
@@ -765,6 +765,8 @@ github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+
github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk=
github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNvA=
github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs=
github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A=
github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE=
github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo=
github.com/rclone/rclone v1.74.4 h1:/c6LMO2kPQjJa6a/PBFcIYJHDvfNOAtfDBJy44mGMpc=
-1
View File
@@ -197,5 +197,4 @@ const (
PathKey
SharingIDKey
SkipHookKey
DirectUploadContentTypeKey
)
-271
View File
@@ -1,271 +0,0 @@
package net
import (
"net/http"
"net/textproto"
"strings"
"time"
)
// scanETag determines if a syntactically valid ETag is present at s. If so,
// the ETag and remaining text after consuming ETag is returned. Otherwise,
// it returns "", "".
func scanETag(s string) (etag string, remain string) {
s = textproto.TrimString(s)
start := 0
if strings.HasPrefix(s, "W/") {
start = 2
}
if len(s[start:]) < 2 || s[start] != '"' {
return "", ""
}
// ETag is either W/"text" or "text".
// See RFC 7232 2.3.
for i := start + 1; i < len(s); i++ {
c := s[i]
switch {
// Character values allowed in ETags.
case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
case c == '"':
return s[:i+1], s[i+1:]
default:
return "", ""
}
}
return "", ""
}
// etagStrongMatch reports whether a and b match using strong ETag comparison.
// Assumes a and b are valid ETags.
func etagStrongMatch(a, b string) bool {
return a == b && a != "" && a[0] == '"'
}
// etagWeakMatch reports whether a and b match using weak ETag comparison.
// Assumes a and b are valid ETags.
func etagWeakMatch(a, b string) bool {
return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
}
// condResult is the result of an HTTP request precondition check.
// See https://tools.ietf.org/html/rfc7232 section 3.
type condResult int
const (
condNone condResult = iota
condTrue
condFalse
)
func checkIfMatch(w http.ResponseWriter, r *http.Request, exists bool) condResult {
values := r.Header.Values("If-Match")
if len(values) == 0 {
return condNone
}
im := strings.Join(values, ",")
r.Header.Del("If-Match")
if !exists {
return condFalse
}
for {
im = textproto.TrimString(im)
if len(im) == 0 {
break
}
if im[0] == ',' {
im = im[1:]
continue
}
if im[0] == '*' {
return condTrue
}
etag, remain := scanETag(im)
if etag == "" {
break
}
if etagStrongMatch(etag, w.Header().Get("Etag")) {
return condTrue
}
im = remain
}
return condFalse
}
func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult {
ius := r.Header.Get("If-Unmodified-Since")
if ius == "" {
return condNone
}
r.Header.Del("If-Unmodified-Since")
if isZeroTime(modtime) {
return condNone
}
t, err := http.ParseTime(ius)
if err != nil {
return condNone
}
// The Last-Modified header truncates sub-second precision so
// the modtime needs to be truncated too.
modtime = modtime.Truncate(time.Second)
if ret := modtime.Compare(t); ret <= 0 {
return condTrue
}
return condFalse
}
func checkIfNoneMatch(w http.ResponseWriter, r *http.Request, exists bool) condResult {
values := r.Header.Values("If-None-Match")
if len(values) == 0 {
return condNone
}
inm := strings.Join(values, ",")
r.Header.Del("If-None-Match")
if !exists {
return condTrue
}
buf := inm
for {
buf = textproto.TrimString(buf)
if len(buf) == 0 {
break
}
if buf[0] == ',' {
buf = buf[1:]
continue
}
if buf[0] == '*' {
return condFalse
}
etag, remain := scanETag(buf)
if etag == "" {
break
}
if etagWeakMatch(etag, w.Header().Get("Etag")) {
return condFalse
}
buf = remain
}
return condTrue
}
func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult {
if r.Method != "GET" && r.Method != "HEAD" {
return condNone
}
ims := r.Header.Get("If-Modified-Since")
if ims == "" {
return condNone
}
r.Header.Del("If-Modified-Since")
if isZeroTime(modtime) {
return condNone
}
t, err := http.ParseTime(ims)
if err != nil {
return condNone
}
// The Last-Modified header truncates sub-second precision so
// the modtime needs to be truncated too.
modtime = modtime.Truncate(time.Second)
if ret := modtime.Compare(t); ret <= 0 {
return condFalse
}
return condTrue
}
func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult {
if r.Method != "GET" && r.Method != "HEAD" {
return condNone
}
ir := r.Header.Get("If-Range")
if ir == "" {
return condNone
}
r.Header.Del("If-Range")
etag, _ := scanETag(ir)
if etag != "" {
if etagStrongMatch(etag, w.Header().Get("Etag")) {
return condTrue
}
return condFalse
}
// The If-Range value is typically the ETag value, but it may also be
// the modtime date. See golang.org/issue/8367.
if modtime.IsZero() {
return condFalse
}
t, err := http.ParseTime(ir)
if err != nil {
return condFalse
}
if t.Unix() == modtime.Unix() {
return condTrue
}
return condFalse
}
var unixEpochTime = time.Unix(0, 0)
// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0).
func isZeroTime(t time.Time) bool {
return t.IsZero() || t.Equal(unixEpochTime)
}
func setLastModified(w http.ResponseWriter, modtime time.Time) {
if !isZeroTime(modtime) {
w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
}
}
func writeNotModified(w http.ResponseWriter) {
// RFC 7232 section 4.1:
// a sender SHOULD NOT generate representation metadata other than the
// above listed fields unless said metadata exists for the purpose of
// guiding cache updates (e.g., Last-Modified might be useful if the
// response does not have an ETag field).
h := w.Header()
delete(h, "Content-Type")
delete(h, "Content-Length")
delete(h, "Content-Encoding")
if h.Get("Etag") != "" {
delete(h, "Last-Modified")
}
w.WriteHeader(http.StatusNotModified)
}
// CheckPreconditions evaluates request preconditions and reports whether a precondition
// resulted in sending StatusNotModified or StatusPreconditionFailed. The caller must
// set the current ETag response header and indicate whether the representation exists.
func CheckPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time, exists bool) (done bool, rangeHeader string) {
// Evaluate preconditions in the order specified by RFC 9110 section 13.2.2.
ch := checkIfMatch(w, r, exists)
if ch == condNone {
ch = checkIfUnmodifiedSince(r, modtime)
}
if ch == condFalse {
w.WriteHeader(http.StatusPreconditionFailed)
return true, ""
}
switch checkIfNoneMatch(w, r, exists) {
case condFalse:
if r.Method == "GET" || r.Method == "HEAD" {
writeNotModified(w)
return true, ""
}
w.WriteHeader(http.StatusPreconditionFailed)
return true, ""
case condNone:
if checkIfModifiedSince(r, modtime) == condFalse {
writeNotModified(w)
return true, ""
}
}
rangeHeader = r.Header.Get("Range")
if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse {
rangeHeader = ""
}
return false, rangeHeader
}
-114
View File
@@ -1,114 +0,0 @@
package net
import (
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestCheckPreconditions(t *testing.T) {
modified := time.Date(2026, time.January, 2, 3, 4, 5, 123456789, time.UTC)
date := modified.Format(http.TimeFormat)
older := modified.Add(-time.Second).Format(http.TimeFormat)
for _, tt := range []struct {
name string
method string
exists bool
etag string
header http.Header
status int
}{
{"unconditional create", "PUT", false, "", nil, 0},
{"unconditional overwrite", "PUT", true, `"current"`, nil, 0},
{"create only missing", "PUT", false, "", http.Header{"If-None-Match": {"*"}}, 0},
{"create only existing", "PUT", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 412},
{"create only existing without etag", "PUT", true, "", http.Header{"If-None-Match": {"*"}}, 412},
{"match missing", "PUT", false, "", http.Header{"If-Match": {"*"}}, 412},
{"match existing", "PUT", true, `"current"`, http.Header{"If-Match": {"*"}}, 0},
{"match existing without etag", "PUT", true, "", http.Header{"If-Match": {"*"}}, 0},
{"match current", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}}, 0},
{"match stale", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412},
{"match missing tag", "PUT", false, "", http.Header{"If-Match": {`"current"`}}, 412},
{"match weak request", "PUT", true, `"current"`, http.Header{"If-Match": {`W/"current"`}}, 412},
{"match weak representation", "PUT", true, `W/"current"`, http.Header{"If-Match": {`"current"`}}, 412},
{"match list", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale", "current"`}}, 0},
{"match multiple lines", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`, `"current"`}}, 0},
{"match empty list members", "PUT", true, `"current"`, http.Header{"If-Match": {`, , "current",`}}, 0},
{"match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-Match": {`"stale", "one,two"`}}, 0},
{"malformed match", "PUT", true, `"current"`, http.Header{"If-Match": {"current"}}, 412},
{"empty match", "PUT", true, `"current"`, http.Header{"If-Match": {""}}, 412},
{"none match current", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"current"`}}, 412},
{"none match stale", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}}, 0},
{"none match missing tag", "PUT", false, "", http.Header{"If-None-Match": {`"current"`}}, 0},
{"none match weak request", "PUT", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 412},
{"none match weak representation", "PUT", true, `W/"current"`, http.Header{"If-None-Match": {`"current"`}}, 412},
{"none match multiple lines", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`, `"current"`}}, 412},
{"none match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-None-Match": {`"stale", "one,two"`}}, 412},
{"both conditions must pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"current"`}}, 412},
{"both conditions pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"stale"`}}, 0},
{"both wildcards missing", "PUT", false, "", http.Header{"If-Match": {"*"}, "If-None-Match": {"*"}}, 412},
{"unmodified since older", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {older}}, 412},
{"unmodified since same second", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {date}}, 0},
{"unmodified since missing", "PUT", false, "", http.Header{"If-Unmodified-Since": {older}}, 0},
{"invalid unmodified since", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {"invalid"}}, 0},
{"match overrides unmodified since", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-Unmodified-Since": {older}}, 0},
{"ignore modified since on put", "PUT", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 0},
{"get not modified", "GET", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 304},
{"head not modified", "HEAD", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 304},
{"get stale match", "GET", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412},
{"get modified since", "GET", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 304},
{"none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}, "If-Modified-Since": {date}}, 0},
{"empty none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {""}, "If-Modified-Since": {date}}, 0},
} {
t.Run(tt.name, func(t *testing.T) {
r := httptest.NewRequest(tt.method, "/file.txt", nil)
for name, values := range tt.header {
for _, value := range values {
r.Header.Add(name, value)
}
}
w := httptest.NewRecorder()
if tt.etag != "" {
w.Header().Set("Etag", tt.etag)
}
var modTime time.Time
if tt.exists {
modTime = modified
}
done, _ := CheckPreconditions(w, r, modTime, tt.exists)
if done != (tt.status != 0) {
t.Fatalf("done = %v, want status %d", done, tt.status)
}
if done && w.Code != tt.status {
t.Errorf("status = %d, want %d", w.Code, tt.status)
}
if w.Body.Len() != 0 {
t.Errorf("unexpected response body: %q", w.Body.String())
}
})
}
}
func TestCheckPreconditionsIfRange(t *testing.T) {
for _, tt := range []struct {
etag string
want string
}{
{`"current"`, "bytes=0-3"},
{`"stale"`, ""},
{`W/"current"`, ""},
} {
t.Run(tt.etag, func(t *testing.T) {
r := httptest.NewRequest(http.MethodGet, "/file.txt", nil)
r.Header.Set("Range", "bytes=0-3")
r.Header.Set("If-Range", tt.etag)
w := httptest.NewRecorder()
w.Header().Set("Etag", `"current"`)
done, got := CheckPreconditions(w, r, time.Time{}, true)
if done || got != tt.want {
t.Fatalf("CheckPreconditions() = (%v, %q), want (false, %q)", done, got, tt.want)
}
})
}
}
+12 -10
View File
@@ -49,7 +49,7 @@ import (
// ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range.
func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, rangeReader model.RangeReaderIF) (err error) {
setLastModified(w, modTime)
done, rangeReq := CheckPreconditions(w, r, modTime, true)
done, rangeReq := checkPreconditions(w, r, modTime)
if done {
return nil
}
@@ -302,18 +302,20 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea
}
// TODO clean header with blocklist or passlist
res.Header.Del("set-cookie")
var reader io.Reader
if res.StatusCode >= 400 {
defer res.Body.Close()
var reader io.Reader = res.Body
if res.Header.Get("Content-Encoding") == "gzip" {
gzipReader, err := gzip.NewReader(res.Body)
if err != nil {
return nil, fmt.Errorf("http request [%s] failure,status: %w decode gzip response: %v", URL, HttpStatusCodeError(res.StatusCode), err)
}
defer gzipReader.Close()
reader = gzipReader
// 根据 Content-Encoding 判断 Body 是否压缩
switch res.Header.Get("Content-Encoding") {
case "gzip":
// 使用gzip.NewReader解压缩
reader, _ = gzip.NewReader(res.Body)
defer reader.(*gzip.Reader).Close()
default:
// 没有Content-Encoding,直接读取
reader = res.Body
}
all, _ := io.ReadAll(reader)
_ = res.Body.Close()
msg := string(all)
log.Debugln(msg)
return nil, fmt.Errorf("http request [%s] failure,status: %w response:%s", URL, HttpStatusCodeError(res.StatusCode), msg)
+257
View File
@@ -4,7 +4,10 @@ import (
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
@@ -15,6 +18,260 @@ import (
log "github.com/sirupsen/logrus"
)
// scanETag determines if a syntactically valid ETag is present at s. If so,
// the ETag and remaining text after consuming ETag is returned. Otherwise,
// it returns "", "".
func scanETag(s string) (etag string, remain string) {
s = textproto.TrimString(s)
start := 0
if strings.HasPrefix(s, "W/") {
start = 2
}
if len(s[start:]) < 2 || s[start] != '"' {
return "", ""
}
// ETag is either W/"text" or "text".
// See RFC 7232 2.3.
for i := start + 1; i < len(s); i++ {
c := s[i]
switch {
// Character values allowed in ETags.
case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
case c == '"':
return s[:i+1], s[i+1:]
default:
return "", ""
}
}
return "", ""
}
// etagStrongMatch reports whether a and b match using strong ETag comparison.
// Assumes a and b are valid ETags.
func etagStrongMatch(a, b string) bool {
return a == b && a != "" && a[0] == '"'
}
// etagWeakMatch reports whether a and b match using weak ETag comparison.
// Assumes a and b are valid ETags.
func etagWeakMatch(a, b string) bool {
return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
}
// condResult is the result of an HTTP request precondition check.
// See https://tools.ietf.org/html/rfc7232 section 3.
type condResult int
const (
condNone condResult = iota
condTrue
condFalse
)
func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult {
im := r.Header.Get("If-Match")
if im == "" {
return condNone
}
r.Header.Del("If-Match")
for {
im = textproto.TrimString(im)
if len(im) == 0 {
break
}
if im[0] == ',' {
im = im[1:]
continue
}
if im[0] == '*' {
return condTrue
}
etag, remain := scanETag(im)
if etag == "" {
break
}
if etagStrongMatch(etag, w.Header().Get("Etag")) {
return condTrue
}
im = remain
}
return condFalse
}
func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult {
ius := r.Header.Get("If-Unmodified-Since")
if ius == "" {
return condNone
}
r.Header.Del("If-Unmodified-Since")
if isZeroTime(modtime) {
return condNone
}
t, err := http.ParseTime(ius)
if err != nil {
return condNone
}
// The Last-Modified header truncates sub-second precision so
// the modtime needs to be truncated too.
modtime = modtime.Truncate(time.Second)
if ret := modtime.Compare(t); ret <= 0 {
return condTrue
}
return condFalse
}
func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult {
inm := r.Header.Get("If-None-Match")
if inm == "" {
return condNone
}
r.Header.Del("If-None-Match")
buf := inm
for {
buf = textproto.TrimString(buf)
if len(buf) == 0 {
break
}
if buf[0] == ',' {
buf = buf[1:]
continue
}
if buf[0] == '*' {
return condFalse
}
etag, remain := scanETag(buf)
if etag == "" {
break
}
if etagWeakMatch(etag, w.Header().Get("Etag")) {
return condFalse
}
buf = remain
}
return condTrue
}
func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult {
if r.Method != "GET" && r.Method != "HEAD" {
return condNone
}
ims := r.Header.Get("If-Modified-Since")
if ims == "" {
return condNone
}
r.Header.Del("If-Modified-Since")
if isZeroTime(modtime) {
return condNone
}
t, err := http.ParseTime(ims)
if err != nil {
return condNone
}
// The Last-Modified header truncates sub-second precision so
// the modtime needs to be truncated too.
modtime = modtime.Truncate(time.Second)
if ret := modtime.Compare(t); ret <= 0 {
return condFalse
}
return condTrue
}
func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult {
if r.Method != "GET" && r.Method != "HEAD" {
return condNone
}
ir := r.Header.Get("If-Range")
if ir == "" {
return condNone
}
r.Header.Del("If-Range")
etag, _ := scanETag(ir)
if etag != "" {
if etagStrongMatch(etag, w.Header().Get("Etag")) {
return condTrue
}
return condFalse
}
// The If-Range value is typically the ETag value, but it may also be
// the modtime date. See golang.org/issue/8367.
if modtime.IsZero() {
return condFalse
}
t, err := http.ParseTime(ir)
if err != nil {
return condFalse
}
if t.Unix() == modtime.Unix() {
return condTrue
}
return condFalse
}
var unixEpochTime = time.Unix(0, 0)
// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0).
func isZeroTime(t time.Time) bool {
return t.IsZero() || t.Equal(unixEpochTime)
}
func setLastModified(w http.ResponseWriter, modtime time.Time) {
if !isZeroTime(modtime) {
w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
}
}
func writeNotModified(w http.ResponseWriter) {
// RFC 7232 section 4.1:
// a sender SHOULD NOT generate representation metadata other than the
// above listed fields unless said metadata exists for the purpose of
// guiding cache updates (e.g., Last-Modified might be useful if the
// response does not have an ETag field).
h := w.Header()
delete(h, "Content-Type")
delete(h, "Content-Length")
delete(h, "Content-Encoding")
if h.Get("Etag") != "" {
delete(h, "Last-Modified")
}
w.WriteHeader(http.StatusNotModified)
}
// checkPreconditions evaluates request preconditions and reports whether a precondition
// resulted in sending StatusNotModified or StatusPreconditionFailed.
func checkPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time) (done bool, rangeHeader string) {
// This function carefully follows RFC 7232 section 6.
ch := checkIfMatch(w, r)
if ch == condNone {
ch = checkIfUnmodifiedSince(r, modtime)
}
if ch == condFalse {
w.WriteHeader(http.StatusPreconditionFailed)
return true, ""
}
switch checkIfNoneMatch(w, r) {
case condFalse:
if r.Method == "GET" || r.Method == "HEAD" {
writeNotModified(w)
return true, ""
}
w.WriteHeader(http.StatusPreconditionFailed)
return true, ""
case condNone:
if checkIfModifiedSince(r, modtime) == condFalse {
writeNotModified(w)
return true, ""
}
}
rangeHeader = r.Header.Get("Range")
if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse {
rangeHeader = ""
}
return false, rangeHeader
}
func sumRangesSize(ranges []http_range.Range) (size int64) {
for _, ra := range ranges {
size += ra.Length
+5 -7
View File
@@ -237,16 +237,14 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error {
return nil
}
storageDriver, err := GetStorageByMountPath(oldStorage.MountPath)
if err != nil {
return errors.WithMessage(err, "failed get storage driver")
}
// Storage settings may change which objects are exposed at the same mount
// path, so cached entries must be discarded before reinitializing the driver.
Cache.DeleteDirectoryTree(storageDriver, "/")
Cache.InvalidateStorageDetails(storageDriver)
if oldStorage.MountPath != storage.MountPath {
// mount path renamed, need to drop the storage
storagesMap.Delete(oldStorage.MountPath)
Cache.DeleteDirectoryTree(storageDriver, "/")
Cache.InvalidateStorageDetails(storageDriver)
}
if err != nil {
return errors.WithMessage(err, "failed get storage driver")
}
err = storageDriver.Drop(ctx)
if err != nil {
-42
View File
@@ -1,42 +0,0 @@
package common
import (
"testing"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
func TestGetEtagStableAcrossCacheRefresh(t *testing.T) {
uploaded := &model.Object{Size: 4, Modified: time.Unix(1700000000, 123456789)}
before := GetEtag(uploaded, uploaded.Size)
refreshed := &model.Object{Size: uploaded.Size, Modified: time.Unix(1700000000, 0)}
if got := GetEtag(refreshed, refreshed.Size); got != before {
t.Fatalf("refreshed file ETag = %q, want %q", got, before)
}
refreshed.Modified = refreshed.Modified.Add(time.Second)
if got := GetEtag(refreshed, refreshed.Size); got == before {
t.Errorf("changed modification second retained ETag %q", got)
}
refreshed.Modified = uploaded.Modified
if got := GetEtag(refreshed, refreshed.Size+1); got == before {
t.Errorf("changed size retained ETag %q", got)
}
}
func TestGetEtagContentHash(t *testing.T) {
file := &model.Object{
Size: 4,
HashInfo: utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("data"))),
}
before := GetEtag(file, file.Size)
file.Modified = time.Unix(1700000000, 100)
if got := GetEtag(file, file.Size); got != before {
t.Errorf("unchanged content ETag = %q, want %q", got, before)
}
file.HashInfo = utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("next")))
if got := GetEtag(file, file.Size); got == before {
t.Errorf("changed content retained ETag %q", got)
}
}
+1 -2
View File
@@ -93,8 +93,7 @@ func GetEtag(file model.Obj, size int64) string {
if len(hash) > 0 {
return fmt.Sprintf(`"%s"`, hash)
}
// Storage backends may not preserve sub-second timestamps across cache
// refreshes, so only use the reproducible second precision here.
// 参考nginx
return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), size)
}
+5 -15
View File
@@ -1,8 +1,6 @@
package handles
import (
"context"
"mime"
"net/url"
stdpath "path"
@@ -17,11 +15,10 @@ import (
)
type FsGetDirectUploadInfoReq struct {
Path string `json:"path" form:"path"`
FileName string `json:"file_name" form:"file_name"`
FileSize int64 `json:"file_size" form:"file_size"`
ContentType string `json:"content_type" form:"content_type"`
Tool string `json:"tool" form:"tool"`
Path string `json:"path" form:"path"`
FileName string `json:"file_name" form:"file_name"`
FileSize int64 `json:"file_size" form:"file_size"`
Tool string `json:"tool" form:"tool"`
}
// FsGetDirectUploadInfo returns the direct upload info if supported by the driver
@@ -94,14 +91,7 @@ func FsGetDirectUploadInfo(c *gin.Context) {
return
}
}
if req.ContentType != "" {
if _, _, err := mime.ParseMediaType(req.ContentType); err != nil {
common.ErrorResp(c, err, 400)
return
}
}
ctx := context.WithValue(c, conf.DirectUploadContentTypeKey, req.ContentType)
directUploadInfo, err := fs.GetDirectUploadInfo(ctx, req.Tool, path, req.FileName, req.FileSize, overwrite)
directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize, overwrite)
if err != nil {
if !overwrite && errs.IsObjectAlreadyExists(err) {
common.ErrorStrResp(c, "file exists", 403)
+4 -20
View File
@@ -10,7 +10,6 @@ import (
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/gofakes3"
log "github.com/sirupsen/logrus"
)
@@ -128,13 +127,9 @@ func (b *s3Backend) walkPage(
return page.addContent(item), nil
}
// Emit entries in the order their keys have in a flat S3 keyspace. A
// directory sorts as if it carried its trailing slash, so "b.txt" comes
// before "b/..." as it does in a real bucket. Without this a listing
// resumed from a marker could silently skip or repeat keys.
dirEntries = slices.Clone(dirEntries)
sort.Slice(dirEntries, func(i, j int) bool {
return listSortKey(dirEntries[i]) < listSortKey(dirEntries[j])
return dirEntries[i].GetName() < dirEntries[j].GetName()
})
for _, entry := range dirEntries {
@@ -148,16 +143,14 @@ func (b *s3Backend) walkPage(
}
if entry.IsDir() {
// S3 common prefixes include the delimiter, e.g. "base/20261009T020008/".
// Clients such as barman-cloud ignore prefixes without it.
subtreePrefix := objectPath + "/"
if addPrefix {
// response.AddPrefix(gofakes3.URLEncode(subtreePrefix))
if !page.addPrefix(subtreePrefix) {
// response.AddPrefix(gofakes3.URLEncode(objectPath))
if !page.addPrefix(objectPath) {
return false, nil
}
continue
}
subtreePrefix := objectPath + "/"
// A marker beyond this subtree lets us avoid an upstream directory read.
if subtreePrefix <= page.marker && !strings.HasPrefix(page.marker, subtreePrefix) {
continue
@@ -182,12 +175,3 @@ func (b *s3Backend) walkPage(
}
return true, nil
}
// listSortKey returns the name used to order directory entries so that the
// emitted keys follow S3 lexicographic order.
func listSortKey(entry model.Obj) string {
if entry.IsDir() {
return entry.GetName() + "/"
}
return entry.GetName()
}
+5 -70
View File
@@ -4,9 +4,7 @@ import (
"context"
"errors"
"fmt"
"slices"
"sort"
"strings"
"testing"
"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -125,10 +123,10 @@ func TestListPagePaginatesContentsAndCommonPrefixesTogether(t *testing.T) {
if len(first.Contents) != 1 || first.Contents[0].Key != "data/a" {
t.Fatalf("first contents = %#v, want data/a", first.Contents)
}
if len(first.CommonPrefixes) != 1 || first.CommonPrefixes[0].Prefix != "data/b/" {
t.Fatalf("first prefixes = %#v, want data/b/", first.CommonPrefixes)
if len(first.CommonPrefixes) != 1 || first.CommonPrefixes[0].Prefix != "data/b" {
t.Fatalf("first prefixes = %#v, want data/b", first.CommonPrefixes)
}
if !first.IsTruncated || first.NextMarker != "data/b/" {
if !first.IsTruncated || first.NextMarker != "data/b" {
t.Fatalf("first page truncated = %v, next marker = %q", first.IsTruncated, first.NextMarker)
}
@@ -143,8 +141,8 @@ func TestListPagePaginatesContentsAndCommonPrefixesTogether(t *testing.T) {
if len(second.Contents) != 1 || second.Contents[0].Key != "data/c" {
t.Fatalf("second contents = %#v, want data/c", second.Contents)
}
if len(second.CommonPrefixes) != 1 || second.CommonPrefixes[0].Prefix != "data/d/" {
t.Fatalf("second prefixes = %#v, want data/d/", second.CommonPrefixes)
if len(second.CommonPrefixes) != 1 || second.CommonPrefixes[0].Prefix != "data/d" {
t.Fatalf("second prefixes = %#v, want data/d", second.CommonPrefixes)
}
if second.IsTruncated || second.NextMarker != "" {
t.Fatalf("second page truncated = %v, next marker = %q", second.IsTruncated, second.NextMarker)
@@ -281,66 +279,3 @@ func TestListPageDoesNotReorderDirectoryCacheEntries(t *testing.T) {
}
}
}
func TestListPageMixedDirectoryAndFilePaginatesInKeyOrder(t *testing.T) {
tree := map[string][]model.Obj{
"bucket/data": {
&model.Object{Name: "c", Size: 1},
&model.Object{Name: "b", IsFolder: true},
&model.Object{Name: "b.txt", Size: 1},
&model.Object{Name: "b-c", IsFolder: true},
&model.Object{Name: "a", Size: 1},
},
"bucket/data/b": {&model.Object{Name: "y", Size: 1}, &model.Object{Name: "x", Size: 1}},
"bucket/data/b-c": {&model.Object{Name: "k", Size: 1}},
}
b := &s3Backend{
listDir: func(_ context.Context, dir string) ([]model.Obj, error) {
return tree[dir], nil
},
}
tests := []struct {
name string
delimiter bool
want []string
}{
{"delimiter", true, []string{"data/a", "data/b-c/", "data/b.txt", "data/b/", "data/c"}},
{"recursive", false, []string{"data/a", "data/b-c/k", "data/b.txt", "data/b/x", "data/b/y", "data/c"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if !sort.StringsAreSorted(tt.want) {
t.Fatalf("expectation is not in key order: %v", tt.want)
}
for maxKeys := int64(1); maxKeys <= int64(len(tt.want))+1; maxKeys++ {
var got []string
page := gofakes3.ListBucketPage{MaxKeys: maxKeys}
for i := 0; i <= len(tt.want); i++ {
list, err := b.listPage(context.Background(), "bucket", "data", "", tt.delimiter, page)
if err != nil {
t.Fatalf("MaxKeys=%d: listPage() error = %v", maxKeys, err)
}
var keys []string
for _, item := range list.Contents {
keys = append(keys, item.Key)
}
for _, prefix := range list.CommonPrefixes {
if !strings.HasSuffix(prefix.Prefix, "/") {
t.Fatalf("common prefix %q does not end with /", prefix.Prefix)
}
keys = append(keys, prefix.Prefix)
}
sort.Strings(keys)
got = append(got, keys...)
if !list.IsTruncated {
break
}
page.Marker, page.HasMarker = list.NextMarker, true
}
if !slices.Equal(got, tt.want) {
t.Fatalf("MaxKeys=%d: listing = %v, want %v", maxKeys, got, tt.want)
}
}
})
}
}
+5 -32
View File
@@ -355,6 +355,8 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
return status, err
}
defer release()
// TODO(rost): Support the If-Match, If-None-Match headers? See bradfitz'
// comments in http.checkEtag.
ctx := r.Context()
user := ctx.Value(conf.UserKey).(*model.User)
reqPath, err = user.JoinPath(reqPath)
@@ -392,30 +394,6 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
if !common.CanWrite(user, parentMeta, parentPath) {
return http.StatusForbidden, errs.PermissionDenied
}
fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
exists := err == nil
if err != nil && !errs.IsObjectNotFound(err) {
if errs.IsNotFoundError(err) {
return http.StatusNotFound, err
}
return http.StatusInternalServerError, err
}
var modTime time.Time
if exists {
if fi.IsDir() {
return http.StatusMethodNotAllowed, nil
}
etag, err := findETag(ctx, h.LockSystem, reqPath, fi)
if err != nil {
return http.StatusInternalServerError, err
}
w.Header().Set("Etag", etag)
modTime = fi.ModTime()
}
if done, _ := net.CheckPreconditions(w, r, modTime, exists); done {
return 0, nil
}
w.Header().Del("Etag")
fsStream := &stream.FileStream{
Obj: &obj,
Reader: r.Body,
@@ -433,21 +411,16 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
if err != nil {
return http.StatusMethodNotAllowed, err
}
fi, err = fs.Get(ctx, reqPath, &fs.GetArgs{})
fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
if err != nil {
return http.StatusInternalServerError, err
fi = &obj
}
etag, err := findETag(ctx, h.LockSystem, reqPath, fi)
if err != nil {
return http.StatusInternalServerError, err
}
w.Header().Set("Etag", etag)
if exists {
w.WriteHeader(http.StatusNoContent)
} else {
w.WriteHeader(http.StatusCreated)
}
return 0, nil
return http.StatusCreated, nil
}
func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status int, err error) {