feat(fs): add pipelined multipart upload (#2723)

* feat(multipart): add chunk reassembly window

- Reassemble concurrently uploaded chunks into a sequential stream through a ring file, bounding disk usage to slots*chunkSize per session
- Park writers up to a deadline when their slot is busy instead of rejecting instantly, so flow control does not surface as connection errors in browsers
- Record a per-chunk CRC32 table for re-fill verification and keep it readable after close
- Propagate cancellation to blocked readers via CloseWithError so drivers treat aborts like canceled requests
- Cover ordering, backpressure, idempotent resends and close/abort wake-ups with race-enabled tests

* feat(multipart): add pipelined upload session manager

- Start the driver upload at session init over a sequential stream backed by the window, so client-to-server and server-to-storage transfers run concurrently
- Attach client-provided hashes to the stream so drivers can attempt rapid upload before any chunk arrives, and absorb chunks racing pipeline completion idempotently
- Keep only metadata and chunk CRCs after a failed attempt: re-sending chunk 0 re-fills a fresh window, and content changes between attempts are rejected
- Resume receiving sessions only when client hashes prove the same file; failed_retriable sessions resume unconditionally
- Reclaim sessions with a sliding-TTL GC and sweep orphaned ring files at startup
- Cover the state machine with race-enabled tests over a stubbed storage layer

* feat(setting): add multipart upload settings

- Add multipart_enabled and multipart_chunk_size (MB) as public traffic settings
- Validate the chunk size on save (integer within 1-90) via the setting item hook

* feat(server): add multipart upload API

- Add /api/fs/multipart init/chunk/complete/status/abort endpoints with headers aligned with /fs/put
- Gate init behind the FsUp permission checks and reuse the client upload rate limiter for chunk uploads
- Drain the request body before answering chunk requests on every path, so browsers do not see early responses as network errors
- Start the multipart session GC when the router is initialized to reclaim ring files orphaned by a previous run

* refactor(multipart): remove unused overwrite session field

- The overwrite flag was stored on the session but never read: the handler
  performs the pre-check and op.Put owns the overwrite semantics

* feat(setting): allow any positive multipart chunk size

- Validate the setting as a positive integer only; self-hosted admins decide
  the ceiling themselves instead of an arbitrary 90MB cap
- Keep clamping the client-suggested X-Chunk-Size to the admin value: the
  server buffers a window of 8 chunks per session, so an unbounded client
  suggestion would translate directly into server-side disk usage

* refactor(server): simplify multipart chunk size clamp

- Fold the two-step clamp into one branch: the ceiling is already floored,
  so a client suggestion just lowers the size with a 1MB floor
This commit is contained in:
ThetaPilla.
2026-08-06 14:14:35 +08:00
committed by GitHub
parent a4ae2acb65
commit cca773f07b
9 changed files with 2418 additions and 0 deletions
+2
View File
@@ -249,6 +249,8 @@ func InitialSettings() []model.SettingItem {
{Key: conf.StreamMaxClientUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
{Key: conf.StreamMaxServerDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
{Key: conf.StreamMaxServerUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
{Key: conf.MultipartEnabled, Value: "true", Type: conf.TypeBool, Group: model.TRAFFIC, Flag: model.PUBLIC},
{Key: conf.MultipartChunkSize, Value: "10", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PUBLIC, Help: `chunk size of multipart upload in MB (positive integer), keep it under your CDN's request body limit; each active session buffers up to 8 chunks on the server's disk`},
}
additionalSettingItems := tool.Tools.Items()
// 固定顺序
+2
View File
@@ -164,6 +164,8 @@ const (
StreamMaxClientUploadSpeed = "max_client_upload_speed"
StreamMaxServerDownloadSpeed = "max_server_download_speed"
StreamMaxServerUploadSpeed = "max_server_upload_speed"
MultipartEnabled = "multipart_enabled"
MultipartChunkSize = "multipart_chunk_size"
)
const (
+607
View File
@@ -0,0 +1,607 @@
package multipart
import (
"context"
"errors"
"fmt"
"io"
"math"
"os"
stdpath "path"
"path/filepath"
"sync"
"sync/atomic"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/google/uuid"
)
type State string
const (
StateReceiving State = "receiving"
StateCompleted State = "completed"
StateFailedRetriable State = "failed_retriable"
StateFailedPermanent State = "failed_permanent"
StateAborted State = "aborted"
)
// WindowSlots bounds the per-session disk footprint to WindowSlots*ChunkSize.
var WindowSlots = 8
const (
// defaultSessionTTL is the sliding inactivity timeout; it also serves as the
// grace period during which finished sessions remain queryable.
defaultSessionTTL = 30 * time.Minute
gcInterval = time.Minute
)
var (
ErrSessionNotFound = errors.New("multipart upload session not found")
ErrNotOwner = errors.New("multipart upload session belongs to another user")
// errAborted wraps context.Canceled so a driver blocked on the stream sees
// the abort as a canceled request and runs its cancellation cleanup.
errAborted = fmt.Errorf("multipart upload aborted: %w", context.Canceled)
)
// putFile is the pipeline tail: resolve the storage and run the regular upload
// path. It mirrors the checks of fs.putDirectly (internal/fs/put.go) but calls
// op.Put directly so the driver's progress callback can be observed.
// It is a variable so session tests can stub the storage layer out.
var putFile = func(ctx context.Context, dstDirPath string, fs *stream.FileStream, up driver.UpdateProgress) error {
storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath)
if err != nil {
return err
}
if storage.Config().NoUpload {
return errs.UploadNotSupported
}
return op.Put(ctx, storage, dstDirActualPath, fs, up)
}
// Session is one multipart upload: metadata survives pipeline attempts, the
// window (chunk data) does not.
type Session struct {
ID string
Path string // full destination path (dir + name), already user-joined
DstDir string
Name string
Size int64
ChunkSize int64
Total int
Mimetype string
Modified time.Time
Hashes map[*utils.HashType]string
Creator *model.User
mu sync.Mutex
state State
err error
attempt int
win *Window
done chan struct{}
cancel context.CancelFunc
prevCRCs []uint32
prevSet []bool
storagePct atomic.Uint64 // math.Float64bits of the driver progress (0-100)
lastActive atomic.Int64 // unix nano
}
func (s *Session) touch() { s.lastActive.Store(time.Now().UnixNano()) }
func (s *Session) setStoragePct(p float64) { s.storagePct.Store(math.Float64bits(p)) }
// Snapshot is the wire representation of a session used by all endpoints.
type SessionSnapshot struct {
ID string `json:"upload_id"`
State State `json:"state"`
Attempt int `json:"attempt"`
Path string `json:"path"`
Size int64 `json:"size"`
ChunkSize int64 `json:"chunk_size"`
TotalChunks int `json:"total_chunks"`
Received [][2]int `json:"received"`
ReceivedBytes int64 `json:"received_bytes"`
Frontier int `json:"frontier"`
StorageProgress float64 `json:"storage_progress"`
Error string `json:"error,omitempty"`
}
func (s *Session) snapshotLocked() SessionSnapshot {
snap := SessionSnapshot{
ID: s.ID,
State: s.state,
Attempt: s.attempt,
Path: s.Path,
Size: s.Size,
ChunkSize: s.ChunkSize,
TotalChunks: s.Total,
Received: [][2]int{},
StorageProgress: math.Float64frombits(s.storagePct.Load()),
}
if s.err != nil {
snap.Error = s.err.Error()
}
switch {
case s.state == StateCompleted:
snap.Received = [][2]int{{0, s.Total - 1}}
snap.ReceivedBytes = s.Size
snap.Frontier = s.Total
snap.StorageProgress = 100
case s.win != nil:
ws := s.win.Snapshot()
snap.Received = ws.Received
snap.ReceivedBytes = ws.ReceivedBytes
snap.Frontier = ws.Frontier
}
return snap
}
func (s *Session) Snapshot() SessionSnapshot {
s.mu.Lock()
defer s.mu.Unlock()
return s.snapshotLocked()
}
// InitReq carries everything the handler parsed from the init request.
type InitReq struct {
User *model.User
Path string // full destination path, already user-joined
Size int64
ChunkSize int64 // final chunk size in bytes, already clamped by the handler
Mimetype string
Modified time.Time
Hashes map[*utils.HashType]string
}
// Manager owns all live sessions. Sessions are in-memory only (aligned with
// upload tasks not being persisted); a restart drops them and the ring files
// are swept on the next start.
type Manager struct {
mu sync.Mutex
byID map[string]*Session
byKey map[string]string
gcOnce sync.Once
ttl time.Duration // 0 means defaultSessionTTL; tests shrink it per instance
}
func (m *Manager) sessionTTL() time.Duration {
if m.ttl > 0 {
return m.ttl
}
return defaultSessionTTL
}
var DefaultManager = &Manager{
byID: make(map[string]*Session),
byKey: make(map[string]string),
}
func (m *Manager) dir() string {
return filepath.Join(conf.Conf.TempDir, "multipart")
}
func sessionKey(userID uint, path string, size int64) string {
return fmt.Sprintf("%d|%s|%d", userID, path, size)
}
// hashesQualifyResume reports whether two hash sets prove the client is
// re-uploading the same file: they must share at least one hash type and
// agree on every shared one. Path+size alone is NOT enough to resume into a
// receiving session — buffered chunks of a different same-sized file would be
// silently mixed into the result.
func hashesQualifyResume(old, new map[*utils.HashType]string) bool {
shared := false
for t, ov := range old {
if nv, ok := new[t]; ok {
if ov != nv {
return false
}
shared = true
}
}
return shared
}
// StartGC sweeps ring files orphaned by a previous run and starts the expiry
// loop. It is called at server startup so orphans are reclaimed even if no
// multipart upload ever happens again; Init also calls it, so embedders that
// skip the server wiring still get GC lazily.
func (m *Manager) StartGC() {
m.ensureGC()
}
func (m *Manager) ensureGC() {
m.gcOnce.Do(func() {
// sweep ring files orphaned by a previous run; bootstrap's CleanTempDir
// only runs when no transfer tasks are pending, so do not rely on it
_ = os.RemoveAll(m.dir())
go func() {
ticker := time.NewTicker(gcInterval)
for range ticker.C {
m.gc()
}
}()
})
}
func (m *Manager) gc() {
deadline := time.Now().Add(-m.sessionTTL()).UnixNano()
m.mu.Lock()
var expired []*Session
for _, s := range m.byID {
if s.lastActive.Load() < deadline {
expired = append(expired, s)
}
}
m.mu.Unlock()
for _, s := range expired {
m.terminate(s, errors.New("multipart upload session expired"))
}
}
// terminate aborts a session (if still receiving) and drops it from the maps.
func (m *Manager) terminate(s *Session, cause error) {
s.mu.Lock()
if s.state == StateReceiving {
s.state = StateAborted
s.err = cause
}
s.killAttemptLocked()
s.mu.Unlock()
m.remove(s)
}
// killAttemptLocked stops the running pipeline attempt: the context cancel
// interrupts drivers blocked on network I/O, and closing the window wakes a
// driver blocked in Read (context cancellation cannot interrupt cond.Wait).
// The caller must hold s.mu and must have set the final state first.
func (s *Session) killAttemptLocked() {
if s.cancel != nil {
s.cancel()
}
if s.win != nil {
_ = s.win.CloseWithError(errAborted)
}
}
func (m *Manager) remove(s *Session) {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.byID, s.ID)
key := sessionKey(s.Creator.ID, s.Path, s.Size)
if m.byKey[key] == s.ID {
delete(m.byKey, key)
}
}
func (m *Manager) get(user *model.User, id string) (*Session, error) {
m.mu.Lock()
s, ok := m.byID[id]
m.mu.Unlock()
if !ok {
return nil, ErrSessionNotFound
}
if s.Creator.ID != user.ID {
return nil, ErrNotOwner
}
return s, nil
}
// Init creates a session and starts its pipeline, or returns the live session
// for the same (user, path, size) so an interrupted client resumes implicitly.
func (m *Manager) Init(req InitReq) (SessionSnapshot, bool, error) {
m.ensureGC()
if req.Size <= 0 {
return SessionSnapshot{}, false, fmt.Errorf("multipart upload requires a positive X-File-Size, got %d", req.Size)
}
if req.ChunkSize <= 0 {
return SessionSnapshot{}, false, fmt.Errorf("invalid chunk size %d", req.ChunkSize)
}
key := sessionKey(req.User.ID, req.Path, req.Size)
m.mu.Lock()
if id, ok := m.byKey[key]; ok {
if s, ok := m.byID[id]; ok {
s.mu.Lock()
st := s.state
s.mu.Unlock()
// failed_retriable resumes unconditionally: nothing of the old
// attempt survives except CRCs, and the re-fill CRC check catches
// a changed file. A receiving session still holds data, so it only
// resumes when hashes prove it is the same file.
if st == StateFailedRetriable ||
(st == StateReceiving && hashesQualifyResume(s.Hashes, req.Hashes)) {
m.mu.Unlock()
s.touch()
return s.Snapshot(), true, nil
}
// finished session, or same path+size without proof of identity:
// drop the old session and start fresh
m.mu.Unlock()
m.terminate(s, errors.New("superseded by a new upload of the same path and size"))
m.mu.Lock()
}
}
m.mu.Unlock()
dstDir, name := stdpath.Split(req.Path)
s := &Session{
ID: uuid.NewString(),
Path: req.Path,
DstDir: dstDir,
Name: name,
Size: req.Size,
ChunkSize: req.ChunkSize,
Mimetype: req.Mimetype,
Modified: req.Modified,
Hashes: req.Hashes,
Creator: req.User,
state: StateReceiving,
}
s.touch()
s.mu.Lock()
if err := m.startAttemptLocked(s); err != nil {
s.mu.Unlock()
return SessionSnapshot{}, false, err
}
s.Total = s.win.TotalChunks()
snap := s.snapshotLocked()
s.mu.Unlock()
m.mu.Lock()
m.byID[s.ID] = s
m.byKey[key] = s.ID
m.mu.Unlock()
return snap, false, nil
}
// startAttemptLocked builds a fresh window and spawns the pipeline goroutine.
// The caller must hold s.mu.
func (m *Manager) startAttemptLocked(s *Session) error {
win, err := NewWindow(m.dir(), fmt.Sprintf("%s.%d", s.ID, s.attempt), s.ChunkSize, s.Size, WindowSlots)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(context.WithValue(context.Background(), conf.UserKey, s.Creator))
done := make(chan struct{})
s.win = win
s.cancel = cancel
s.done = done
s.state = StateReceiving
s.err = nil
s.setStoragePct(0)
fileStream := &stream.FileStream{
Obj: &model.Object{
Name: s.Name,
Size: s.Size,
Modified: s.Modified,
HashInfo: utils.NewHashInfoByMap(s.Hashes),
},
Reader: win,
Mimetype: s.Mimetype,
}
fileStream.Add(win)
dstDir := s.DstDir
put := putFile // capture: the seam must not be read after spawn
go func() {
err := put(ctx, dstDir, fileStream, s.setStoragePct)
s.finishAttempt(win, err)
close(done)
}()
return nil
}
// finishAttempt records the pipeline outcome and harvests the CRC table for
// re-fill verification. The window's data is gone at this point (op.Put closed
// it); only metadata survives.
func (s *Session) finishAttempt(win *Window, err error) {
crcs, set := win.CRCs()
_ = win.Close() // op.Put already closed it; make sure the ring file is gone anyway
s.mu.Lock()
defer s.mu.Unlock()
s.touch()
if s.win == win {
s.win = nil
}
s.prevCRCs, s.prevSet = crcs, set
switch {
case err == nil:
s.state = StateCompleted
s.err = nil
s.setStoragePct(100)
case s.state != StateReceiving:
// Abort/expiry already labeled this attempt; keep that state.
if s.err == nil {
s.err = err
}
case isPermanentPutError(err):
s.state = StateFailedPermanent
s.err = err
default:
s.state = StateFailedRetriable
s.err = err
s.attempt++
}
}
func isPermanentPutError(err error) bool {
if errors.Is(err, context.Canceled) {
return false
}
for _, target := range []error{
errs.UploadNotSupported,
errs.PermissionDenied,
errs.StorageNotFound,
errs.ObjectAlreadyExists,
errs.RelativePath,
errs.IgnoredSystemFile,
} {
if errors.Is(err, target) {
return true
}
}
return false
}
// Chunk feeds one chunk into the session. Re-sending chunk 0 to a
// failed_retriable session re-fills it: a fresh window and pipeline attempt.
func (m *Manager) Chunk(user *model.User, id string, idx int, body io.Reader) (SessionSnapshot, error) {
s, err := m.get(user, id)
if err != nil {
return SessionSnapshot{}, err
}
s.touch()
s.mu.Lock()
switch s.state {
case StateReceiving:
case StateFailedRetriable:
if idx != 0 {
snap := s.snapshotLocked()
s.mu.Unlock()
return snap, fmt.Errorf("upload attempt failed, resend from chunk 0 to retry: %w", s.err)
}
if err := m.startAttemptLocked(s); err != nil {
snap := s.snapshotLocked()
s.mu.Unlock()
return snap, err
}
case StateCompleted:
snap := s.snapshotLocked()
s.mu.Unlock()
return snap, nil // idempotent: stragglers after rapid-upload/finish succeed
default:
snap := s.snapshotLocked()
s.mu.Unlock()
return snap, fmt.Errorf("session is %s: %w", s.state, s.err)
}
win := s.win
var prevCRC uint32
hasPrev := false
if idx < len(s.prevSet) && s.prevSet[idx] {
prevCRC, hasPrev = s.prevCRCs[idx], true
}
s.mu.Unlock()
crc, err := win.WriteChunk(idx, body)
if err != nil {
// A closed window means the pipeline ended while this chunk was in
// flight — rapid upload makes this the NORMAL case: the driver
// succeeds off the hash alone with chunks still arriving. The window
// closes (op.Put's defer) moments before the verdict is recorded, so
// wait for the verdict instead of racing it, then absorb the chunk
// idempotently if the upload in fact succeeded.
if errors.Is(err, ErrClosed) || errors.Is(err, context.Canceled) {
s.mu.Lock()
done := s.done
s.mu.Unlock()
select {
case <-done:
case <-time.After(10 * time.Second): // pipeline teardown is µs-scale; never expected
}
}
s.mu.Lock()
completed := s.state == StateCompleted
snap := s.snapshotLocked()
s.mu.Unlock()
if completed {
return snap, nil
}
return snap, err
}
if hasPrev && crc != prevCRC {
err := fmt.Errorf("chunk %d content changed between attempts, aborting", idx)
s.mu.Lock()
s.state = StateFailedPermanent
s.err = err
s.killAttemptLocked()
s.mu.Unlock()
return s.Snapshot(), err
}
return s.Snapshot(), nil
}
// Complete waits for the pipeline outcome. It refuses to block while chunks
// are still missing, so a buggy client cannot park a connection for the TTL.
func (m *Manager) Complete(ctx context.Context, user *model.User, id string) (SessionSnapshot, error) {
s, err := m.get(user, id)
if err != nil {
return SessionSnapshot{}, err
}
s.touch()
for {
s.mu.Lock()
st := s.state
done := s.done
if st == StateReceiving && s.win != nil {
if ws := s.win.Snapshot(); ws.ReceivedBytes < s.Size {
snap := s.snapshotLocked()
s.mu.Unlock()
return snap, fmt.Errorf("cannot complete: %d of %d bytes received", ws.ReceivedBytes, s.Size)
}
}
s.mu.Unlock()
if st != StateReceiving {
break
}
select {
case <-done:
case <-ctx.Done():
return s.Snapshot(), ctx.Err()
}
}
s.touch()
s.mu.Lock()
st, serr := s.state, s.err
snap := s.snapshotLocked()
s.mu.Unlock()
if st == StateCompleted {
m.remove(s) // served its purpose; frees the key for future uploads
return snap, nil
}
return snap, fmt.Errorf("upload failed (%s): %w", st, serr)
}
// Status looks a session up by id.
func (m *Manager) Status(user *model.User, id string) (SessionSnapshot, error) {
s, err := m.get(user, id)
if err != nil {
return SessionSnapshot{}, err
}
s.touch()
return s.Snapshot(), nil
}
// Find looks a live session up by destination path and size, for resume discovery.
func (m *Manager) Find(user *model.User, path string, size int64) (SessionSnapshot, error) {
m.mu.Lock()
id, ok := m.byKey[sessionKey(user.ID, path, size)]
m.mu.Unlock()
if !ok {
return SessionSnapshot{}, ErrSessionNotFound
}
return m.Status(user, id)
}
// Abort cancels the pipeline and forgets the session immediately.
func (m *Manager) Abort(user *model.User, id string) error {
s, err := m.get(user, id)
if err != nil {
return err
}
m.terminate(s, errors.New("multipart upload aborted by client"))
return nil
}
+650
View File
@@ -0,0 +1,650 @@
package multipart
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"sync"
"testing"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// Tests in this file share the putFile seam and must not run in parallel.
func setupSessionTest(t *testing.T) *Manager {
t.Helper()
oldConf := conf.Conf
conf.Conf = conf.DefaultConfig(t.TempDir())
t.Cleanup(func() { conf.Conf = oldConf })
return &Manager{byID: make(map[string]*Session), byKey: make(map[string]string)}
}
func stubPut(t *testing.T, fn func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error) {
t.Helper()
orig := putFile
putFile = fn
t.Cleanup(func() { putFile = orig })
}
func testUser() *model.User { return &model.User{ID: 7, Username: "tester"} }
func initReq(user *model.User, size, chunkSize int64) InitReq {
return InitReq{
User: user,
Path: "/local/test.bin",
Size: size,
ChunkSize: chunkSize,
Mimetype: "application/octet-stream",
Modified: time.Unix(1700000000, 0),
}
}
func sendChunk(t *testing.T, m *Manager, user *model.User, id string, data []byte, idx int, chunkSize int64) SessionSnapshot {
t.Helper()
snap, err := m.Chunk(user, id, idx, bytes.NewReader(chunkOf(data, idx, chunkSize)))
if err != nil {
t.Fatalf("Chunk(%d): %v", idx, err)
}
return snap
}
func waitState(t *testing.T, m *Manager, user *model.User, id string, want State) SessionSnapshot {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
snap, err := m.Status(user, id)
if err != nil {
t.Fatalf("Status: %v", err)
}
if snap.State == want {
return snap
}
if time.Now().After(deadline) {
t.Fatalf("session state = %s, want %s (err: %s)", snap.State, want, snap.Error)
}
time.Sleep(2 * time.Millisecond)
}
}
func TestSessionHappyPath(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(3*chunkSize + 300)
data := genData(totalSize)
user := testUser()
got := make(chan []byte, 1)
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
if dst != "/local/" {
return fmt.Errorf("unexpected dst dir %q", dst)
}
if fs.GetName() != "test.bin" || fs.GetSize() != totalSize {
return fmt.Errorf("unexpected stream meta %s/%d", fs.GetName(), fs.GetSize())
}
b, err := io.ReadAll(fs)
if err != nil {
return err
}
up(100)
got <- b
return nil
})
snap, resumed, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil || resumed {
t.Fatalf("Init = (resumed=%v, err=%v)", resumed, err)
}
if snap.TotalChunks != 4 || snap.State != StateReceiving {
t.Fatalf("init snapshot = %+v", snap)
}
for _, idx := range []int{1, 0, 3, 2} { // out of order on purpose
sendChunk(t, m, user, snap.ID, data, idx, chunkSize)
}
final, err := m.Complete(context.Background(), user, snap.ID)
if err != nil {
t.Fatalf("Complete: %v", err)
}
if final.State != StateCompleted || final.StorageProgress != 100 {
t.Fatalf("final snapshot = %+v", final)
}
if !bytes.Equal(<-got, data) {
t.Fatal("driver received different bytes")
}
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
t.Fatalf("session should be removed after Complete, got %v", err)
}
}
func TestSessionRetriableRefill(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
user := testUser()
attempts := 0
got := make(chan []byte, 1)
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
attempts++
if attempts == 1 {
buf := make([]byte, chunkSize)
if _, err := io.ReadFull(fs, buf); err != nil {
return err
}
return errors.New("transient storage hiccup")
}
b, err := io.ReadAll(fs)
if err != nil {
return err
}
got <- b
return nil
})
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
failed := waitState(t, m, user, snap.ID, StateFailedRetriable)
if failed.Attempt != 1 {
t.Fatalf("attempt = %d, want 1", failed.Attempt)
}
if len(failed.Received) != 0 {
t.Fatalf("failed session must report nothing received, got %v", failed.Received)
}
// chunks other than 0 are rejected until the client restarts the fill
if _, err := m.Chunk(user, snap.ID, 1, bytes.NewReader(chunkOf(data, 1, chunkSize))); err == nil {
t.Fatal("chunk 1 on failed_retriable session: expected error")
}
// re-fill from chunk 0 respawns the pipeline
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
sendChunk(t, m, user, snap.ID, data, 1, chunkSize)
final, err := m.Complete(context.Background(), user, snap.ID)
if err != nil {
t.Fatalf("Complete after refill: %v", err)
}
if final.State != StateCompleted || attempts != 2 {
t.Fatalf("state=%s attempts=%d, want completed/2", final.State, attempts)
}
if !bytes.Equal(<-got, data) {
t.Fatal("driver received different bytes after refill")
}
}
func TestSessionPermanentFailure(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
data := genData(chunkSize)
user := testUser()
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
return fmt.Errorf("denied: %w", errs.PermissionDenied)
})
snap, _, err := m.Init(initReq(user, chunkSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
waitState(t, m, user, snap.ID, StateFailedPermanent)
if _, err := m.Chunk(user, snap.ID, 0, bytes.NewReader(data)); err == nil {
t.Fatal("chunk on failed_permanent session: expected error")
}
if _, err := m.Complete(context.Background(), user, snap.ID); err == nil {
t.Fatal("Complete on failed_permanent session: expected error")
}
}
func TestRefillCRCMismatch(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
user := testUser()
attempts := 0
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
attempts++
if attempts == 1 {
buf := make([]byte, chunkSize)
if _, err := io.ReadFull(fs, buf); err != nil {
return err
}
return errors.New("transient")
}
_, err := io.ReadAll(fs)
return err
})
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
waitState(t, m, user, snap.ID, StateFailedRetriable)
tampered := genData(chunkSize + 5)[:chunkSize] // different content, same length
if _, err := m.Chunk(user, snap.ID, 0, bytes.NewReader(tampered)); err == nil {
t.Fatal("re-fill with changed content: expected error")
}
waitState(t, m, user, snap.ID, StateFailedPermanent)
}
func TestCompleteRefusesIncomplete(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(3 * chunkSize)
data := genData(totalSize)
user := testUser()
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
_, err := io.ReadAll(fs)
return err
})
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
if _, err := m.Complete(context.Background(), user, snap.ID); err == nil {
t.Fatal("Complete with missing chunks: expected error")
}
// unblock the pipeline goroutine before the test tears down
if err := m.Abort(user, snap.ID); err != nil {
t.Fatalf("Abort: %v", err)
}
}
func TestAbortAndOwnership(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(4 * chunkSize)
data := genData(totalSize)
user := testUser()
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
_, err := io.ReadAll(fs)
return err
})
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
stranger := &model.User{ID: 99, Username: "stranger"}
if _, err := m.Status(stranger, snap.ID); !errors.Is(err, ErrNotOwner) {
t.Fatalf("stranger Status err = %v, want ErrNotOwner", err)
}
if err := m.Abort(stranger, snap.ID); !errors.Is(err, ErrNotOwner) {
t.Fatalf("stranger Abort err = %v, want ErrNotOwner", err)
}
if err := m.Abort(user, snap.ID); err != nil {
t.Fatalf("Abort: %v", err)
}
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
t.Fatalf("Status after Abort err = %v, want ErrSessionNotFound", err)
}
}
func TestExpiry(t *testing.T) {
m := setupSessionTest(t)
m.ttl = 30 * time.Millisecond
const chunkSize = 1024
user := testUser()
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
_, err := io.ReadAll(fs)
return err
})
snap, _, err := m.Init(initReq(user, 4*chunkSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
time.Sleep(60 * time.Millisecond)
m.gc()
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
t.Fatalf("Status after expiry err = %v, want ErrSessionNotFound", err)
}
// the terminated pipeline must release the ring file
deadline := time.Now().Add(5 * time.Second)
for {
entries, _ := os.ReadDir(m.dir())
if len(entries) == 0 {
break
}
if time.Now().After(deadline) {
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
t.Fatalf("ring files not cleaned up after expiry: %v", names)
}
time.Sleep(5 * time.Millisecond)
}
}
func TestInitResume(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(4 * chunkSize)
data := genData(totalSize)
user := testUser()
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
_, err := io.ReadAll(fs)
return err
})
hashed := initReq(user, totalSize, chunkSize)
hashed.Hashes = map[*utils.HashType]string{utils.MD5: "0123456789abcdef0123456789abcdef"}
snap, _, err := m.Init(hashed)
if err != nil {
t.Fatalf("Init: %v", err)
}
sendChunk(t, m, user, snap.ID, data, 2, chunkSize)
// same hashes prove the same file: resume with buffered chunks skippable
again, resumed, err := m.Init(hashed)
if err != nil || !resumed {
t.Fatalf("hashed re-Init = (resumed=%v, err=%v), want resumed", resumed, err)
}
if again.ID != snap.ID {
t.Fatalf("resumed session id = %s, want %s", again.ID, snap.ID)
}
if len(again.Received) != 1 || again.Received[0] != [2]int{2, 2} {
t.Fatalf("resumed received = %v, want [[2,2]]", again.Received)
}
// a different size is a different upload
other, resumed, err := m.Init(initReq(user, totalSize+1, chunkSize))
if err != nil || resumed || other.ID == snap.ID {
t.Fatalf("different-size Init = (id=%s, resumed=%v, err=%v)", other.ID, resumed, err)
}
if _, err := m.Find(user, "/local/test.bin", totalSize); err != nil {
t.Fatalf("Find: %v", err)
}
if _, err := m.Find(user, "/local/nope.bin", totalSize); !errors.Is(err, ErrSessionNotFound) {
t.Fatalf("Find miss err = %v, want ErrSessionNotFound", err)
}
// without hashes, path+size cannot prove identity against a receiving
// session holding buffered data — the old session must be dropped, or a
// same-sized different file would be silently mixed into the result
bare := initReq(user, totalSize, chunkSize)
fresh, resumed, err := m.Init(bare)
if err != nil || resumed || fresh.ID == snap.ID {
t.Fatalf("bare re-Init = (id=%s, resumed=%v, err=%v), want a fresh session", fresh.ID, resumed, err)
}
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
t.Fatalf("superseded session err = %v, want ErrSessionNotFound", err)
}
}
func TestResumeFailedRetriableWithoutHash(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
user := testUser()
attempts := 0
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
attempts++
if attempts == 1 {
buf := make([]byte, chunkSize)
if _, err := io.ReadFull(fs, buf); err != nil {
return err
}
return errors.New("transient")
}
_, err := io.ReadAll(fs)
return err
})
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
waitState(t, m, user, snap.ID, StateFailedRetriable)
// a failed_retriable session keeps no chunk data, so identity proof is not
// required to resume: the retry-button flow works without rapid hashing,
// and the re-fill CRC check still catches a changed file
again, resumed, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil || !resumed || again.ID != snap.ID {
t.Fatalf("re-Init on failed_retriable = (id=%s, resumed=%v, err=%v), want resumed same session", again.ID, resumed, err)
}
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
sendChunk(t, m, user, snap.ID, data, 1, chunkSize)
if _, err := m.Complete(context.Background(), user, snap.ID); err != nil {
t.Fatalf("Complete after hashless refill: %v", err)
}
}
func TestRapidUploadShortCircuit(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(6 * chunkSize)
data := genData(totalSize)
user := testUser()
wantMD5 := "0123456789abcdef0123456789abcdef"
wantSHA1 := "da39a3ee5e6b4b0d3255bfef95601890afd80709"
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
// the hash provided at init must reach the driver through the stream,
// that is what lets PutRapid-style drivers skip the transfer entirely
if got := fs.GetHash().GetHash(utils.MD5); got != wantMD5 {
return fmt.Errorf("md5 not propagated: %q", got)
}
if got := fs.GetHash().GetHash(utils.SHA1); got != wantSHA1 {
return fmt.Errorf("sha1 not propagated: %q", got)
}
time.Sleep(50 * time.Millisecond) // simulated rapid-upload API round trip
up(100)
return nil // rapid upload hit: succeed without reading the stream
})
req := initReq(user, totalSize, chunkSize)
req.Hashes = map[*utils.HashType]string{utils.MD5: wantMD5, utils.SHA1: wantSHA1}
snap, _, err := m.Init(req)
if err != nil {
t.Fatalf("Init: %v", err)
}
// spam chunks across the completion moment: some land while receiving,
// some race the window close, some arrive after completion — with the
// completed-session absorption none of them may surface an error
var wg sync.WaitGroup
errCh := make(chan error, 64)
for w := 0; w < 4; w++ {
wg.Add(1)
go func(w int) {
defer wg.Done()
for round := 0; round < 8; round++ {
idx := (w*8 + round) % 6
_, err := m.Chunk(user, snap.ID, idx, bytes.NewReader(chunkOf(data, idx, chunkSize)))
// flow-control signals are part of the protocol, not failures
if err != nil && !errors.Is(err, ErrChunkInFlight) && !errors.Is(err, ErrOutOfWindow) {
errCh <- fmt.Errorf("worker %d chunk %d: %w", w, idx, err)
return
}
time.Sleep(5 * time.Millisecond)
}
}(w)
}
wg.Wait()
close(errCh)
for err := range errCh {
t.Error(err)
}
final, err := m.Complete(context.Background(), user, snap.ID)
if err != nil {
t.Fatalf("Complete: %v", err)
}
if final.State != StateCompleted || final.StorageProgress != 100 {
t.Fatalf("final snapshot = %+v", final)
}
}
// TestChunkRacesRapidCompletion pins the exact race the completed-session
// absorption exists for: a chunk request grabs the live window, stalls while
// receiving its body, the pipeline completes off the hash alone (rapid
// upload) and closes the window — the stalled chunk must then succeed
// idempotently instead of surfacing "window closed" to a client whose upload
// in fact just finished.
func TestChunkRacesRapidCompletion(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
user := testUser()
proceed := make(chan struct{})
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
defer fs.Close()
<-proceed // the rapid-upload verdict arrives when the test says so
return nil
})
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
gate := make(chan struct{})
type result struct {
snap SessionSnapshot
err error
}
resCh := make(chan result, 1)
go func() {
s, e := m.Chunk(user, snap.ID, 0,
&gatedReader{release: gate, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))})
resCh <- result{s, e}
}()
// wait until the chunk writer holds slot 0 in the filling state
m.mu.Lock()
sess := m.byID[snap.ID]
m.mu.Unlock()
sess.mu.Lock()
win := sess.win
sess.mu.Unlock()
deadline := time.Now().Add(5 * time.Second)
for {
win.mu.Lock()
filling := win.slotState[0] == slotFilling
win.mu.Unlock()
if filling {
break
}
if time.Now().After(deadline) {
t.Fatal("chunk writer never reached the filling state")
}
time.Sleep(time.Millisecond)
}
close(proceed) // rapid upload succeeds, window closes
waitState(t, m, user, snap.ID, StateCompleted) // completion recorded
close(gate) // stalled chunk finishes against the closed window
res := <-resCh
if res.err != nil {
t.Fatalf("in-flight chunk across rapid completion must be absorbed, got: %v", res.err)
}
if res.snap.State != StateCompleted {
t.Fatalf("absorbed chunk snapshot state = %s, want completed", res.snap.State)
}
}
// TestChunkDuringCompletionGap covers the moment between op.Put closing the
// window (its defer) and the verdict being recorded: a chunk hitting the
// closed window inside that gap must wait for the verdict and be absorbed,
// not bounce a "window closed" error at a client whose upload just succeeded.
func TestChunkDuringCompletionGap(t *testing.T) {
m := setupSessionTest(t)
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
user := testUser()
windowClosed := make(chan struct{})
allowVerdict := make(chan struct{})
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
_ = fs.Close() // what op.Put's defer does before Put returns
close(windowClosed)
<-allowVerdict // hold the pipeline return open: this IS the gap
return nil
})
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
if err != nil {
t.Fatalf("Init: %v", err)
}
<-windowClosed
type result struct {
snap SessionSnapshot
err error
}
resCh := make(chan result, 1)
go func() {
s, e := m.Chunk(user, snap.ID, 0, bytes.NewReader(chunkOf(data, 0, chunkSize)))
resCh <- result{s, e}
}()
select {
case r := <-resCh:
t.Fatalf("chunk inside the gap returned early with (%s, %v); it must wait for the verdict", r.snap.State, r.err)
case <-time.After(150 * time.Millisecond):
// still waiting on the verdict, as designed
}
close(allowVerdict)
select {
case r := <-resCh:
if r.err != nil {
t.Fatalf("gap chunk must be absorbed after completion, got: %v", r.err)
}
if r.snap.State != StateCompleted {
t.Fatalf("gap chunk snapshot state = %s, want completed", r.snap.State)
}
case <-time.After(5 * time.Second):
t.Fatal("gap chunk never returned after the verdict")
}
}
func TestInitRejectsBadSize(t *testing.T) {
m := setupSessionTest(t)
if _, _, err := m.Init(initReq(testUser(), 0, 1024)); err == nil {
t.Fatal("Init with size 0: expected error")
}
}
+377
View File
@@ -0,0 +1,377 @@
package multipart
import (
"errors"
"fmt"
"hash/crc32"
"io"
"os"
"path/filepath"
"sync"
"time"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
const (
slotFree uint8 = iota
slotFilling
slotReady
)
var (
// ErrClosed is the sticky error after Close; all pending and future reads/writes fail with it.
ErrClosed = errors.New("multipart upload window closed")
// ErrChunkInFlight means another request is uploading the same chunk right now.
ErrChunkInFlight = errors.New("chunk is being uploaded by another request")
// ErrOutOfWindow means the chunk is still too far ahead of the consumption
// frontier after waiting WindowWaitTimeout; the client should back off and
// resend it later (flow control, not a failure).
ErrOutOfWindow = errors.New("chunk is out of the receiving window")
)
// WindowWaitTimeout bounds how long WriteChunk blocks waiting for its slot.
// Browsers cannot reliably read responses sent before the request body is
// consumed (they report a network error), so under backpressure it is far
// better to hold the request until a slot frees — the wait must just stay
// well below CDN request deadlines (Cloudflare: ~100s). Tests shrink this.
var WindowWaitTimeout = 10 * time.Second
// Window reassembles concurrently uploaded chunks into a sequential stream.
// Chunks land in a ring file of slots*chunkSize bytes (chunk i -> slot i%slots),
// and Read serves bytes in order, blocking until the next needed chunk arrives.
// A chunk slot is released as soon as the reader crosses its boundary, so the
// disk footprint is bounded by slots*chunkSize regardless of the file size.
//
// WriteChunk is safe for concurrent use; Read must be called from a single
// goroutine (the same contract as the FileStreamer it backs).
type Window struct {
mu sync.Mutex
cond *sync.Cond
f *os.File
path string
chunkSize int64
totalSize int64
total int
slots int
slotState []uint8
slotChunk []int
readPos int64
crcs []uint32
crcSet []bool
err error
}
// Snapshot describes the receiving state, used for status responses and resume.
type Snapshot struct {
// Frontier is the next chunk index to be consumed (== TotalChunks when the stream is fully consumed).
Frontier int
// ReadPos is the number of bytes already consumed by the pipeline.
ReadPos int64
// ReceivedBytes is the number of payload bytes received from the client (consumed + buffered).
ReceivedBytes int64
// Received holds inclusive ranges of chunk indexes the client does not need to resend.
Received [][2]int
}
func NewWindow(dir, id string, chunkSize, totalSize int64, slots int) (*Window, error) {
if chunkSize <= 0 || totalSize <= 0 || slots <= 0 {
return nil, fmt.Errorf("invalid window params: chunkSize=%d totalSize=%d slots=%d", chunkSize, totalSize, slots)
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, err
}
path := filepath.Join(dir, id+".ring")
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return nil, err
}
total := int((totalSize + chunkSize - 1) / chunkSize)
w := &Window{
f: f,
path: path,
chunkSize: chunkSize,
totalSize: totalSize,
total: total,
slots: slots,
slotState: make([]uint8, slots),
slotChunk: make([]int, slots),
crcs: make([]uint32, total),
crcSet: make([]bool, total),
}
for i := range w.slotChunk {
w.slotChunk[i] = -1
}
w.cond = sync.NewCond(&w.mu)
return w, nil
}
func (w *Window) TotalChunks() int { return w.total }
// ChunkLen returns the payload length of chunk idx (the last chunk may be short).
func (w *Window) ChunkLen(idx int) int64 {
if idx == w.total-1 {
return w.totalSize - int64(idx)*w.chunkSize
}
return w.chunkSize
}
func stopTimer(t *time.Timer) {
if t != nil {
t.Stop()
}
}
// frontier returns the next chunk index to be consumed. Callers must hold mu.
func (w *Window) frontier() int {
if w.readPos >= w.totalSize {
return w.total
}
return int(w.readPos / w.chunkSize)
}
// WriteChunk reads exactly the chunk payload from r into the ring and returns its CRC32 (IEEE).
// Re-sending an already buffered or consumed chunk succeeds immediately without touching data.
func (w *Window) WriteChunk(idx int, r io.Reader) (uint32, error) {
w.mu.Lock()
if w.err != nil {
w.mu.Unlock()
return 0, w.err
}
if idx < 0 || idx >= w.total {
w.mu.Unlock()
return 0, fmt.Errorf("chunk index %d out of range [0,%d)", idx, w.total)
}
length := w.ChunkLen(idx)
slot := idx % w.slots
// Admission control with a bounded wait: instead of bouncing a chunk the
// moment its slot is busy, park the request until the reader frees the
// slot. Rejecting fast would answer before the request body is read, which
// browsers surface as a network error — so under backpressure, waiting IS
// the flow control. sync.Cond has no timed wait; a timer broadcast wakes
// the loop at the deadline.
deadline := time.Now().Add(WindowWaitTimeout)
var timer *time.Timer
for {
if w.err != nil {
w.mu.Unlock()
stopTimer(timer)
return 0, w.err
}
if int64(idx)*w.chunkSize+length <= w.readPos {
// already fully consumed
crc := w.crcs[idx]
w.mu.Unlock()
stopTimer(timer)
return crc, nil
}
if w.slotChunk[slot] == idx {
if w.slotState[slot] == slotReady {
crc := w.crcs[idx]
w.mu.Unlock()
stopTimer(timer)
return crc, nil
}
if w.slotState[slot] == slotFilling {
w.mu.Unlock()
stopTimer(timer)
return 0, ErrChunkInFlight
}
}
if w.slotState[slot] == slotFree && idx < w.frontier()+w.slots {
break // admissible
}
if !time.Now().Before(deadline) {
w.mu.Unlock()
stopTimer(timer)
return 0, ErrOutOfWindow
}
if timer == nil {
timer = time.AfterFunc(time.Until(deadline), func() {
w.mu.Lock()
w.cond.Broadcast()
w.mu.Unlock()
})
}
w.cond.Wait()
}
stopTimer(timer)
w.slotState[slot] = slotFilling
w.slotChunk[slot] = idx
f := w.f
w.mu.Unlock()
h := crc32.NewIEEE()
n, err := utils.CopyWithBufferN(io.NewOffsetWriter(f, int64(slot)*w.chunkSize), io.TeeReader(r, h), length)
if err == nil {
// the body must contain exactly one chunk
var b [1]byte
if m, _ := io.ReadFull(r, b[:]); m > 0 {
err = fmt.Errorf("chunk %d larger than expected %d bytes", idx, length)
}
} else {
err = fmt.Errorf("incomplete chunk %d: got %d of %d bytes: %w", idx, n, length, err)
}
w.mu.Lock()
defer w.mu.Unlock()
if w.slotState[slot] != slotFilling || w.slotChunk[slot] != idx {
// the window was closed and reset the slot while we were writing
if w.err != nil {
return 0, w.err
}
return 0, ErrClosed
}
if err == nil && w.err != nil {
err = w.err
}
if err != nil {
w.slotState[slot] = slotFree
w.slotChunk[slot] = -1
return 0, err
}
w.slotState[slot] = slotReady
w.crcs[idx] = h.Sum32()
w.crcSet[idx] = true
w.cond.Broadcast()
return w.crcs[idx], nil
}
// Read serves the reassembled stream in order, blocking until the next chunk
// is available, the window is closed, or the stream ends (io.EOF).
func (w *Window) Read(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
w.mu.Lock()
for {
if w.err != nil {
w.mu.Unlock()
return 0, w.err
}
if w.readPos >= w.totalSize {
w.mu.Unlock()
return 0, io.EOF
}
cur := int(w.readPos / w.chunkSize)
slot := cur % w.slots
if w.slotState[slot] == slotReady && w.slotChunk[slot] == cur {
chunkStart := int64(cur) * w.chunkSize
chunkEnd := chunkStart + w.ChunkLen(cur)
n := int64(len(p))
if avail := chunkEnd - w.readPos; n > avail {
n = avail
}
off := int64(slot)*w.chunkSize + (w.readPos - chunkStart)
f := w.f
w.mu.Unlock()
read, err := f.ReadAt(p[:n], off)
w.mu.Lock()
if read > 0 {
w.readPos += int64(read)
if w.readPos >= chunkEnd {
w.slotState[slot] = slotFree
w.slotChunk[slot] = -1
w.cond.Broadcast() // writers may be parked waiting for this slot
}
}
sticky := w.err
w.mu.Unlock()
if read > 0 {
return read, nil
}
if sticky != nil {
return 0, sticky
}
if err == nil {
err = io.ErrUnexpectedEOF
}
return 0, err
}
w.cond.Wait()
}
}
// Close makes all pending and future operations fail with ErrClosed and removes
// the ring file. It is invoked by op.Put via FileStream.Closers when the
// pipeline ends, and is safe to call multiple times.
func (w *Window) Close() error {
return w.CloseWithError(ErrClosed)
}
// CloseWithError is Close with a caller-chosen sticky error. The session's
// abort path passes an error wrapping context.Canceled so that a driver woken
// up from a blocked Read treats the abort exactly like a canceled request
// (e.g. the local driver only removes partially written files in that case).
func (w *Window) CloseWithError(sticky error) error {
w.mu.Lock()
if w.err == nil {
w.err = sticky
}
f := w.f
w.f = nil
for i := range w.slotState {
w.slotState[i] = slotFree
w.slotChunk[i] = -1
}
w.cond.Broadcast()
w.mu.Unlock()
if f == nil {
return nil
}
err := f.Close()
if rmErr := os.Remove(w.path); rmErr != nil && err == nil {
err = rmErr
}
return err
}
// Snapshot reports the receiving state for status responses and resume discovery.
func (w *Window) Snapshot() Snapshot {
w.mu.Lock()
defer w.mu.Unlock()
snap := Snapshot{Frontier: w.frontier(), ReadPos: w.readPos, ReceivedBytes: w.readPos}
var ranges [][2]int
if snap.Frontier > 0 {
ranges = append(ranges, [2]int{0, snap.Frontier - 1})
}
for idx := snap.Frontier; idx < snap.Frontier+w.slots && idx < w.total; idx++ {
slot := idx % w.slots
if w.slotState[slot] != slotReady || w.slotChunk[slot] != idx {
continue
}
start := int64(idx) * w.chunkSize
var consumed int64
if w.readPos > start {
consumed = w.readPos - start
}
snap.ReceivedBytes += w.ChunkLen(idx) - consumed
if len(ranges) > 0 && ranges[len(ranges)-1][1] == idx-1 {
ranges[len(ranges)-1][1] = idx
} else {
ranges = append(ranges, [2]int{idx, idx})
}
}
snap.Received = ranges
return snap
}
// CRCs returns a copy of the per-chunk CRC32 table and which entries are set.
// It remains readable after Close, so the session can compare re-filled chunks
// against a previous attempt.
func (w *Window) CRCs() ([]uint32, []bool) {
w.mu.Lock()
defer w.mu.Unlock()
crcs := make([]uint32, len(w.crcs))
set := make([]bool, len(w.crcSet))
copy(crcs, w.crcs)
copy(set, w.crcSet)
return crcs, set
}
+527
View File
@@ -0,0 +1,527 @@
package multipart
import (
"bytes"
"crypto/sha256"
"errors"
"hash/crc32"
"io"
"math/rand"
"sync"
"sync/atomic"
"testing"
"time"
)
func genData(size int64) []byte {
data := make([]byte, size)
rnd := rand.New(rand.NewSource(size*7919 + 13))
rnd.Read(data)
return data
}
func newTestWindow(t *testing.T, chunkSize, totalSize int64, slots int) *Window {
t.Helper()
w, err := NewWindow(t.TempDir(), "test", chunkSize, totalSize, slots)
if err != nil {
t.Fatalf("NewWindow: %v", err)
}
t.Cleanup(func() { _ = w.Close() })
return w
}
func chunkOf(data []byte, idx int, chunkSize int64) []byte {
start := int64(idx) * chunkSize
end := start + chunkSize
if end > int64(len(data)) {
end = int64(len(data))
}
return data[start:end]
}
func writeChunkOK(t *testing.T, w *Window, data []byte, idx int) uint32 {
t.Helper()
crc, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, w.chunkSize)))
if err != nil {
t.Fatalf("WriteChunk(%d): %v", idx, err)
}
if want := crc32.ChecksumIEEE(chunkOf(data, idx, w.chunkSize)); crc != want {
t.Fatalf("WriteChunk(%d) crc = %08x, want %08x", idx, crc, want)
}
return crc
}
// readAllWithin reads the whole stream in a goroutine and fails the test on timeout,
// so a reassembly bug cannot hang the suite.
func readAllWithin(t *testing.T, w *Window, timeout time.Duration) []byte {
t.Helper()
type result struct {
data []byte
err error
}
ch := make(chan result, 1)
go func() {
data, err := io.ReadAll(w)
ch <- result{data, err}
}()
select {
case res := <-ch:
if res.err != nil {
t.Fatalf("ReadAll: %v", res.err)
}
return res.data
case <-time.After(timeout):
t.Fatal("ReadAll timed out")
return nil
}
}
func TestSequentialReadWrite(t *testing.T) {
const chunkSize = 64 * 1024
totalSize := int64(4*chunkSize + 32*1024) // last chunk is short
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 8)
go func() {
for i := 0; i < w.TotalChunks(); i++ {
if _, err := w.WriteChunk(i, bytes.NewReader(chunkOf(data, i, chunkSize))); err != nil {
t.Errorf("WriteChunk(%d): %v", i, err)
return
}
time.Sleep(time.Millisecond)
}
}()
got := readAllWithin(t, w, 10*time.Second)
if !bytes.Equal(got, data) {
t.Fatalf("reassembled stream differs: got %d bytes, want %d", len(got), len(data))
}
}
func TestOutOfOrderWrites(t *testing.T) {
const chunkSize = 16 * 1024
totalSize := int64(5*chunkSize - 100)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 8)
for _, idx := range []int{3, 0, 4, 2, 1} {
writeChunkOK(t, w, data, idx)
}
got := readAllWithin(t, w, 10*time.Second)
if !bytes.Equal(got, data) {
t.Fatal("reassembled stream differs after out-of-order writes")
}
}
func TestConcurrentWriters(t *testing.T) {
const chunkSize = 32 * 1024
totalSize := int64(8 * chunkSize)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 8)
var wg sync.WaitGroup
for i := 0; i < w.TotalChunks(); i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
if _, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, chunkSize))); err != nil {
t.Errorf("WriteChunk(%d): %v", idx, err)
}
}(i)
}
got := readAllWithin(t, w, 10*time.Second)
wg.Wait()
if !bytes.Equal(got, data) {
t.Fatal("reassembled stream differs after concurrent writes")
}
}
func setWaitTimeout(t *testing.T, d time.Duration) {
t.Helper()
old := WindowWaitTimeout
WindowWaitTimeout = d
t.Cleanup(func() { WindowWaitTimeout = old })
}
func TestBackpressure(t *testing.T) {
setWaitTimeout(t, 50*time.Millisecond) // assert the post-deadline rejection
const chunkSize = 1024
totalSize := int64(5 * chunkSize)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 2)
if _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))); !errors.Is(err, ErrOutOfWindow) {
t.Fatalf("chunk 2 with frontier 0: err = %v, want ErrOutOfWindow", err)
}
writeChunkOK(t, w, data, 0)
writeChunkOK(t, w, data, 1)
// both slots occupied, frontier still 0
if _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))); !errors.Is(err, ErrOutOfWindow) {
t.Fatalf("chunk 2 with full window: err = %v, want ErrOutOfWindow", err)
}
// consume chunk 0 -> slot released, frontier advances
buf := make([]byte, chunkSize)
if _, err := io.ReadFull(w, buf); err != nil {
t.Fatalf("ReadFull chunk 0: %v", err)
}
if !bytes.Equal(buf, chunkOf(data, 0, chunkSize)) {
t.Fatal("chunk 0 content differs")
}
writeChunkOK(t, w, data, 2)
// chunk 3 maps to the slot still holding buffered chunk 1
if _, err := w.WriteChunk(3, bytes.NewReader(chunkOf(data, 3, chunkSize))); !errors.Is(err, ErrOutOfWindow) {
t.Fatalf("chunk 3 with occupied slot: err = %v, want ErrOutOfWindow", err)
}
}
// TestWriteChunkWaitsForSlot pins the browser-friendly flow control: a chunk
// whose slot is occupied parks until the reader frees it instead of bouncing
// with an immediate rejection (early responses read as network errors in
// browsers).
func TestWriteChunkWaitsForSlot(t *testing.T) {
setWaitTimeout(t, 5*time.Second)
const chunkSize = 1024
totalSize := int64(5 * chunkSize)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 2)
writeChunkOK(t, w, data, 0)
writeChunkOK(t, w, data, 1)
done := make(chan error, 1)
go func() {
_, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize)))
done <- err
}()
select {
case err := <-done:
t.Fatalf("chunk 2 should be parked while the window is full, returned %v", err)
case <-time.After(100 * time.Millisecond):
// parked, as designed
}
// consuming chunk 0 frees its slot and must wake the parked writer
buf := make([]byte, chunkSize)
if _, err := io.ReadFull(w, buf); err != nil {
t.Fatalf("ReadFull: %v", err)
}
select {
case err := <-done:
if err != nil {
t.Fatalf("parked chunk after slot freed: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("parked chunk never admitted after its slot freed")
}
}
func TestIdempotentResend(t *testing.T) {
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 4)
first := writeChunkOK(t, w, data, 0)
again := writeChunkOK(t, w, data, 0) // buffered, not yet consumed
if first != again {
t.Fatalf("resend crc = %08x, want %08x", again, first)
}
buf := make([]byte, chunkSize)
if _, err := io.ReadFull(w, buf); err != nil {
t.Fatalf("ReadFull: %v", err)
}
consumed := writeChunkOK(t, w, data, 0) // already consumed
if consumed != first {
t.Fatalf("post-consume resend crc = %08x, want %08x", consumed, first)
}
}
// gatedReader blocks the first Read until released, to hold a chunk in the filling state.
type gatedReader struct {
release <-chan struct{}
inner io.Reader
}
func (g *gatedReader) Read(p []byte) (int, error) {
<-g.release
return g.inner.Read(p)
}
func TestInFlightConflict(t *testing.T) {
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 4)
release := make(chan struct{})
done := make(chan error, 1)
go func() {
_, err := w.WriteChunk(0, &gatedReader{release: release, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))})
done <- err
}()
// wait until the writer marked the slot as filling
deadline := time.Now().Add(5 * time.Second)
for {
w.mu.Lock()
filling := w.slotState[0] == slotFilling
w.mu.Unlock()
if filling {
break
}
if time.Now().After(deadline) {
t.Fatal("writer never reached filling state")
}
time.Sleep(time.Millisecond)
}
if _, err := w.WriteChunk(0, bytes.NewReader(chunkOf(data, 0, chunkSize))); !errors.Is(err, ErrChunkInFlight) {
t.Fatalf("concurrent same-chunk write: err = %v, want ErrChunkInFlight", err)
}
close(release)
if err := <-done; err != nil {
t.Fatalf("gated WriteChunk: %v", err)
}
writeChunkOK(t, w, data, 0) // idempotent after settle
}
func TestShortBodyRecovers(t *testing.T) {
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 4)
if _, err := w.WriteChunk(0, bytes.NewReader(chunkOf(data, 0, chunkSize)[:100])); err == nil {
t.Fatal("short body: expected error")
}
writeChunkOK(t, w, data, 0) // slot must have been recycled
}
func TestOversizeBodyRejected(t *testing.T) {
const chunkSize = 1024
totalSize := int64(2 * chunkSize)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 4)
oversize := append(append([]byte{}, chunkOf(data, 0, chunkSize)...), 0xFF)
if _, err := w.WriteChunk(0, bytes.NewReader(oversize)); err == nil {
t.Fatal("oversize body: expected error")
}
writeChunkOK(t, w, data, 0)
// the short last chunk must also reject a full-size body
last := w.TotalChunks() - 1
if last == 0 {
t.Fatal("test needs at least 2 chunks")
}
}
func TestLastChunkShortStrict(t *testing.T) {
const chunkSize = 1024
totalSize := int64(chunkSize + 100)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 4)
if _, err := w.WriteChunk(1, bytes.NewReader(genData(chunkSize))); err == nil {
t.Fatal("full-size body for short last chunk: expected error")
}
writeChunkOK(t, w, data, 1)
writeChunkOK(t, w, data, 0)
got := readAllWithin(t, w, 10*time.Second)
if !bytes.Equal(got, data) {
t.Fatal("reassembled stream differs")
}
}
func TestCloseUnblocksReader(t *testing.T) {
w := newTestWindow(t, 1024, 4096, 4)
errCh := make(chan error, 1)
go func() {
buf := make([]byte, 16)
_, err := w.Read(buf)
errCh <- err
}()
time.Sleep(20 * time.Millisecond)
_ = w.Close()
select {
case err := <-errCh:
if !errors.Is(err, ErrClosed) {
t.Fatalf("blocked Read after Close: err = %v, want ErrClosed", err)
}
case <-time.After(5 * time.Second):
t.Fatal("Read still blocked after Close")
}
}
func TestCloseWithErrorPropagatesSticky(t *testing.T) {
w := newTestWindow(t, 1024, 4096, 4)
cause := errors.New("aborted for a reason")
errCh := make(chan error, 1)
go func() {
buf := make([]byte, 16)
_, err := w.Read(buf)
errCh <- err
}()
time.Sleep(20 * time.Millisecond)
_ = w.CloseWithError(cause)
select {
case err := <-errCh:
if !errors.Is(err, cause) {
t.Fatalf("blocked Read after CloseWithError: err = %v, want %v", err, cause)
}
case <-time.After(5 * time.Second):
t.Fatal("Read still blocked after CloseWithError")
}
if _, err := w.WriteChunk(0, bytes.NewReader(make([]byte, 1024))); !errors.Is(err, cause) {
t.Fatalf("WriteChunk after CloseWithError: err = %v, want %v", err, cause)
}
}
func TestWriteAfterClose(t *testing.T) {
w := newTestWindow(t, 1024, 4096, 4)
_ = w.Close()
if _, err := w.WriteChunk(0, bytes.NewReader(make([]byte, 1024))); !errors.Is(err, ErrClosed) {
t.Fatalf("WriteChunk after Close: err = %v, want ErrClosed", err)
}
}
func TestCloseUnblocksInFlightWriter(t *testing.T) {
const chunkSize = 1024
data := genData(2 * chunkSize)
w := newTestWindow(t, chunkSize, 2*chunkSize, 4)
release := make(chan struct{})
done := make(chan error, 1)
go func() {
_, err := w.WriteChunk(0, &gatedReader{release: release, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))})
done <- err
}()
time.Sleep(20 * time.Millisecond)
_ = w.Close()
close(release)
if err := <-done; !errors.Is(err, ErrClosed) {
t.Fatalf("in-flight WriteChunk across Close: err = %v, want ErrClosed", err)
}
}
func TestEOFExact(t *testing.T) {
const chunkSize = 1024
totalSize := int64(chunkSize + 5)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 4)
writeChunkOK(t, w, data, 0)
writeChunkOK(t, w, data, 1)
got := readAllWithin(t, w, 10*time.Second)
if !bytes.Equal(got, data) {
t.Fatal("reassembled stream differs")
}
buf := make([]byte, 1)
if n, err := w.Read(buf); n != 0 || err != io.EOF {
t.Fatalf("Read at EOF = (%d, %v), want (0, io.EOF)", n, err)
}
}
func TestManyLapsSmallWindow(t *testing.T) {
const chunkSize = 8 * 1024
const chunks = 64
totalSize := int64(chunks*chunkSize - 777)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 3)
var next atomic.Int64
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
idx := int(next.Add(1) - 1)
if idx >= w.TotalChunks() {
return
}
for {
_, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, chunkSize)))
if err == nil {
break
}
if errors.Is(err, ErrOutOfWindow) || errors.Is(err, ErrChunkInFlight) {
time.Sleep(200 * time.Microsecond)
continue
}
t.Errorf("WriteChunk(%d): %v", idx, err)
return
}
}
}()
}
got := readAllWithin(t, w, 30*time.Second)
wg.Wait()
if wantSum, gotSum := sha256.Sum256(data), sha256.Sum256(got); wantSum != gotSum {
t.Fatalf("reassembled stream differs: got %d bytes, want %d", len(got), len(data))
}
}
func TestSnapshot(t *testing.T) {
const chunkSize = 1024
totalSize := int64(5*chunkSize + 512)
data := genData(totalSize)
w := newTestWindow(t, chunkSize, totalSize, 4)
writeChunkOK(t, w, data, 0)
writeChunkOK(t, w, data, 1)
writeChunkOK(t, w, data, 3)
snap := w.Snapshot()
if snap.Frontier != 0 || snap.ReadPos != 0 {
t.Fatalf("snapshot frontier/readPos = %d/%d, want 0/0", snap.Frontier, snap.ReadPos)
}
wantRanges := [][2]int{{0, 1}, {3, 3}}
if len(snap.Received) != len(wantRanges) || snap.Received[0] != wantRanges[0] || snap.Received[1] != wantRanges[1] {
t.Fatalf("snapshot received = %v, want %v", snap.Received, wantRanges)
}
if snap.ReceivedBytes != 3*chunkSize {
t.Fatalf("snapshot receivedBytes = %d, want %d", snap.ReceivedBytes, 3*chunkSize)
}
buf := make([]byte, chunkSize)
if _, err := io.ReadFull(w, buf); err != nil {
t.Fatalf("ReadFull: %v", err)
}
snap = w.Snapshot()
if snap.Frontier != 1 || snap.ReadPos != chunkSize {
t.Fatalf("snapshot frontier/readPos = %d/%d, want 1/%d", snap.Frontier, snap.ReadPos, chunkSize)
}
if len(snap.Received) != 2 || snap.Received[0] != [2]int{0, 1} || snap.Received[1] != [2]int{3, 3} {
t.Fatalf("snapshot received = %v, want [[0,1],[3,3]]", snap.Received)
}
if snap.ReceivedBytes != 3*chunkSize {
t.Fatalf("snapshot receivedBytes = %d, want %d", snap.ReceivedBytes, 3*chunkSize)
}
}
func TestIndexOutOfRange(t *testing.T) {
w := newTestWindow(t, 1024, 4096, 4)
if _, err := w.WriteChunk(-1, bytes.NewReader(nil)); err == nil {
t.Fatal("negative index: expected error")
}
if _, err := w.WriteChunk(4, bytes.NewReader(nil)); err == nil {
t.Fatal("index == total: expected error")
}
}
func TestCRCsSurviveClose(t *testing.T) {
const chunkSize = 1024
data := genData(2 * chunkSize)
w := newTestWindow(t, chunkSize, 2*chunkSize, 4)
want := writeChunkOK(t, w, data, 0)
_ = w.Close()
crcs, set := w.CRCs()
if !set[0] || crcs[0] != want {
t.Fatalf("CRCs after Close = (%08x, %v), want (%08x, true)", crcs[0], set[0], want)
}
if set[1] {
t.Fatal("chunk 1 crc should not be set")
}
}
+13
View File
@@ -2,7 +2,9 @@ package op
import (
"context"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
@@ -83,6 +85,17 @@ var settingItemHooks = map[string]SettingItemHook{
conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",")
return nil
},
conf.MultipartChunkSize: func(item *model.SettingItem) error {
size, err := strconv.Atoi(strings.TrimSpace(item.Value))
if err != nil || size < 1 {
// deliberately a plain error: SaveSettings formats hook errors
// with %+v, which would dump a full stack trace into the UI
// notification for stack-carrying errors
return fmt.Errorf("multipart chunk size must be a positive integer (MB), got %q", item.Value)
}
item.Value = strconv.Itoa(size)
return nil
},
}
func RegisterSettingItemHook(key string, hook SettingItemHook) {
+232
View File
@@ -0,0 +1,232 @@
package handles
import (
"errors"
"io"
"net/url"
stdpath "path"
"strconv"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/multipart"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/server/common"
"github.com/gin-gonic/gin"
)
const multipartMinChunkSize = int64(1) << 20 // 1MB
// multipartChunkSize resolves the effective chunk size. The admin setting is
// the ceiling: a client may suggest a smaller chunk via X-Chunk-Size but never
// a larger one — the server buffers a window of several chunks per session, so
// an unbounded client suggestion would translate directly into server-side
// disk usage.
func multipartChunkSize(requested int64) int64 {
ceiling := int64(setting.GetInt(conf.MultipartChunkSize, 10)) << 20
if ceiling < multipartMinChunkSize {
ceiling = multipartMinChunkSize
}
size := ceiling
if requested > 0 && requested < ceiling {
size = max(requested, multipartMinChunkSize)
}
return size
}
type MultipartInitResp struct {
multipart.SessionSnapshot
Resumed bool `json:"resumed"`
}
// MultipartInit creates (or resumes) a multipart upload session and starts its
// upload pipeline. Headers mirror FsStream (fsup.go).
func MultipartInit(c *gin.Context) {
if !setting.GetBool(conf.MultipartEnabled) {
common.ErrorStrResp(c, "multipart upload is disabled", 403)
return
}
path := c.GetHeader("File-Path")
path, err := url.PathUnescape(path)
if err != nil {
common.ErrorResp(c, err, 400)
return
}
user := c.Request.Context().Value(conf.UserKey).(*model.User)
path, err = user.JoinPath(path)
if err != nil {
common.ErrorResp(c, err, 403)
return
}
size, err := strconv.ParseInt(c.GetHeader("X-File-Size"), 10, 64)
if err != nil {
common.ErrorStrResp(c, "multipart upload requires a valid X-File-Size header", 400)
return
}
if size <= 0 {
common.ErrorStrResp(c, "multipart upload requires a positive X-File-Size; upload empty files via /fs/put", 400)
return
}
var requestedChunkSize int64
if v := c.GetHeader("X-Chunk-Size"); v != "" {
requestedChunkSize, err = strconv.ParseInt(v, 10, 64)
if err != nil {
common.ErrorResp(c, err, 400)
return
}
}
overwrite := c.GetHeader("Overwrite") != "false"
if !overwrite {
if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil {
common.ErrorStrResp(c, "file exists", 403)
return
}
}
dir, name := stdpath.Split(path)
if shouldIgnoreSystemFile(name) {
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
return
}
// fail fast on unusable destinations instead of letting the pipeline discover it
storage, _, err := op.GetStorageAndActualPath(dir)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
if storage.Config().NoUpload {
common.ErrorResp(c, errs.UploadNotSupported, 405)
return
}
h := make(map[*utils.HashType]string)
if md5 := c.GetHeader("X-File-Md5"); md5 != "" {
h[utils.MD5] = md5
}
if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" {
h[utils.SHA1] = sha1
}
if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" {
h[utils.SHA256] = sha256
}
mimetype := c.GetHeader("Content-Type")
if len(mimetype) == 0 {
mimetype = utils.GetMimeType(name)
}
snap, resumed, err := multipart.DefaultManager.Init(multipart.InitReq{
User: user,
Path: path,
Size: size,
ChunkSize: multipartChunkSize(requestedChunkSize),
Mimetype: mimetype,
Modified: getLastModified(c),
Hashes: h,
})
if err != nil {
common.ErrorResp(c, err, 500)
return
}
common.SuccessResp(c, MultipartInitResp{SessionSnapshot: snap, Resumed: resumed})
}
// MultipartChunk ingests one chunk. Chunks are idempotent and may be sent
// concurrently and out of order within the receiving window.
// code 429 = window full (flow control, retry after a short delay),
// code 409 = the same chunk is already in flight on another connection.
func MultipartChunk(c *gin.Context) {
user := c.Request.Context().Value(conf.UserKey).(*model.User)
id := c.GetHeader("X-Upload-Id")
idx, err := strconv.Atoi(c.GetHeader("X-Chunk-Index"))
if err != nil {
common.ErrorStrResp(c, "invalid X-Chunk-Index header", 400)
return
}
snap, err := multipart.DefaultManager.Chunk(user, id, idx, c.Request.Body)
// Answer only after the request body is consumed — on EVERY path. Flow
// control (429), absorbed chunks and validation errors would otherwise
// respond while the browser is still streaming the body, which it reports
// as a network error and which poisons its connection pool. A rejected
// chunk gets resent anyway, so draining costs no extra round trip. The
// drain is bounded so a malformed request cannot pin the handler.
limit := multipartChunkSize(0) + 64*1024
if snap.ChunkSize > 0 {
limit = snap.ChunkSize + 64*1024
}
_, _ = utils.CopyWithBuffer(io.Discard, io.LimitReader(c.Request.Body, limit))
if err != nil {
common.ErrorWithDataResp(c, err, multipartErrCode(err), snap)
return
}
common.SuccessResp(c, snap)
}
// MultipartComplete waits for the pipeline outcome and reports it, mirroring
// how /fs/put only responds once the driver upload finished.
func MultipartComplete(c *gin.Context) {
user := c.Request.Context().Value(conf.UserKey).(*model.User)
id := c.GetHeader("X-Upload-Id")
snap, err := multipart.DefaultManager.Complete(c.Request.Context(), user, id)
if err != nil {
common.ErrorWithDataResp(c, err, multipartErrCode(err), snap)
return
}
common.SuccessResp(c, snap)
}
// MultipartStatus looks a session up by upload_id, or by path+size so an
// interrupted client can discover a resumable session.
func MultipartStatus(c *gin.Context) {
user := c.Request.Context().Value(conf.UserKey).(*model.User)
if id := c.Query("upload_id"); id != "" {
snap, err := multipart.DefaultManager.Status(user, id)
if err != nil {
common.ErrorResp(c, err, multipartErrCode(err))
return
}
common.SuccessResp(c, snap)
return
}
path, err := user.JoinPath(c.Query("path"))
if err != nil {
common.ErrorResp(c, err, 403)
return
}
size, err := strconv.ParseInt(c.Query("size"), 10, 64)
if err != nil {
common.ErrorStrResp(c, "status lookup requires upload_id, or path and size", 400)
return
}
snap, err := multipart.DefaultManager.Find(user, path, size)
if err != nil {
common.ErrorResp(c, err, multipartErrCode(err))
return
}
common.SuccessResp(c, snap)
}
// MultipartAbort cancels the pipeline and discards the session.
func MultipartAbort(c *gin.Context) {
user := c.Request.Context().Value(conf.UserKey).(*model.User)
if err := multipart.DefaultManager.Abort(user, c.GetHeader("X-Upload-Id")); err != nil {
common.ErrorResp(c, err, multipartErrCode(err))
return
}
common.SuccessResp(c)
}
func multipartErrCode(err error) int {
switch {
case errors.Is(err, multipart.ErrOutOfWindow):
return 429
case errors.Is(err, multipart.ErrChunkInFlight):
return 409
case errors.Is(err, multipart.ErrSessionNotFound):
return 404
case errors.Is(err, multipart.ErrNotOwner):
return 403
default:
return 400
}
}
+8
View File
@@ -4,6 +4,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/cmd/flags"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/message"
multipartPkg "github.com/OpenListTeam/OpenList/v4/internal/multipart"
"github.com/OpenListTeam/OpenList/v4/internal/sign"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
@@ -213,6 +214,13 @@ func _fs(g *gin.RouterGroup) {
uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit)
g.PUT("/put", middlewares.FsUp, uploadLimiter, handles.FsStream)
g.PUT("/form", middlewares.FsUp, uploadLimiter, handles.FsForm)
multipartPkg.DefaultManager.StartGC() // reclaim ring files orphaned by a previous run
multipart := g.Group("/multipart")
multipart.POST("/init", middlewares.FsUp, handles.MultipartInit)
multipart.PUT("/chunk", uploadLimiter, handles.MultipartChunk)
multipart.POST("/complete", handles.MultipartComplete)
multipart.GET("/status", handles.MultipartStatus)
multipart.POST("/abort", handles.MultipartAbort)
g.POST("/link", middlewares.AuthAdmin, handles.Link)
// g.POST("/add_aria2", handles.AddOfflineDownload)
// g.POST("/add_qbit", handles.AddQbittorrent)