Skip to content

Commit 381a539

Browse files
committed
fix: resolve OSS chunk cleanup and legacy download issues
Use compatible S3 checksum settings, retry failed cleanup without reusing completed uploads, and include legacy attachment filenames. Correct admin-page URLs in both documentation languages. Refs #556, #531, #502
1 parent 6f88eae commit 381a539

19 files changed

Lines changed: 502 additions & 67 deletions

‎apps/base/local_share.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,11 @@
1313

1414
import time
1515
from pathlib import Path
16-
from urllib.parse import quote
1716

1817
from fastapi import HTTPException
1918

2019
from core.errors import StorageError
21-
from core.storage import StoredDownload
20+
from core.storage import StoredDownload, build_attachment_headers
2221

2322
LOCAL_REF_MARKER = "local-ref"
2423
MAX_RELPATH_LEN = 255
@@ -97,8 +96,7 @@ def build_local_ref_download(file_code) -> StoredDownload:
9796
filename = f"{getattr(file_code, 'prefix', '') or ''}{getattr(file_code, 'suffix', '') or ''}"
9897
if not filename:
9998
filename = path.name
100-
encoded_filename = quote(filename, safe="")
101-
headers = {"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}"}
99+
headers = build_attachment_headers(filename)
102100
try:
103101
headers["Content-Length"] = str(path.stat().st_size)
104102
except OSError:

‎apps/base/services.py‎

Lines changed: 38 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
from fastapi import HTTPException, UploadFile
1111
from fastapi.responses import FileResponse, Response, StreamingResponse
12+
from tortoise.transactions import in_transaction
1213

1314
from core.logger import logger
1415
from core.settings import settings
@@ -210,13 +211,15 @@ async def complete_chunked_upload(
210211
失败路径的配额释放与清理范围与原实现逐一对齐:
211212
完整性校验失败仅抛 400(预留由 TTL 兜底);合并失败清理分片文件后抛 500。
212213
"""
214+
if chunk_info.completed:
215+
raise HTTPException(409, "上传会话已完成,仅待清理")
213216
storage = FileUploadService._storage()
214217
await reserve_storage(
215218
f"chunk:{upload_id}", chunk_info.file_size, ttl_seconds=chunk_reservation_ttl()
216219
)
217220

218221
completed_chunks = await UploadChunk.filter(
219-
upload_id=upload_id, completed=True
222+
upload_id=upload_id, completed=True, chunk_index__gte=0
220223
).all()
221224
if len(completed_chunks) != chunk_info.total_chunks:
222225
raise HTTPException(400, "分片不完整")
@@ -225,16 +228,19 @@ async def complete_chunked_upload(
225228
max_total_size = len(completed_chunks) * chunk_info.chunk_size
226229
if max_total_size > settings.upload_size:
227230
save_path = chunk_info.save_path
231+
cleanup_succeeded = True
228232
if save_path:
229233
try:
230234
await storage.clean_chunks(upload_id, save_path)
231235
except Exception:
236+
cleanup_succeeded = False
232237
logger.warning(
233238
"分片超限中止:清理分片文件失败 upload_id=%s",
234239
upload_id,
235240
exc_info=True,
236241
)
237-
await UploadChunk.filter(upload_id=upload_id).delete()
242+
if cleanup_succeeded:
243+
await UploadChunk.filter(upload_id=upload_id).delete()
238244
await release_storage(f"chunk:{upload_id}")
239245
max_size_mb = settings.upload_size / (1024 * 1024)
240246
raise HTTPException(
@@ -258,22 +264,36 @@ async def complete_chunked_upload(
258264
expired_at, expired_count, used_count, code = await get_expire_info(
259265
expire_value, expire_style
260266
)
261-
await FileCodes.create(
262-
code=code,
263-
file_hash=file_hash, # 使用合并后计算的哈希
264-
is_chunked=True,
265-
upload_id=upload_id,
266-
size=chunk_info.file_size,
267-
expired_at=expired_at,
268-
expired_count=expired_count,
269-
used_count=used_count,
270-
file_path=path,
271-
uuid_file_name=safe_file_name,
272-
prefix=prefix,
273-
suffix=suffix,
274-
)
275-
await storage.clean_chunks(upload_id, save_path)
276-
await UploadChunk.filter(upload_id=upload_id).delete()
267+
# 发布记录与会话终态一起提交;即使分享过期删除,会话也不能被复用。
268+
async with in_transaction() as connection:
269+
await FileCodes.create(
270+
code=code,
271+
file_hash=file_hash, # 使用合并后计算的哈希
272+
is_chunked=True,
273+
upload_id=upload_id,
274+
size=chunk_info.file_size,
275+
expired_at=expired_at,
276+
expired_count=expired_count,
277+
used_count=used_count,
278+
file_path=path,
279+
uuid_file_name=safe_file_name,
280+
prefix=prefix,
281+
suffix=suffix,
282+
using_db=connection,
283+
)
284+
chunk_info.completed = True
285+
await chunk_info.save(using_db=connection, update_fields=["completed"])
286+
try:
287+
await storage.clean_chunks(upload_id, save_path)
288+
except Exception:
289+
# 分享已创建:不把清理故障伪装成合并失败,保留会话供定时任务重试。
290+
logger.warning(
291+
"分片合并完成:清理临时分片文件失败 upload_id=%s",
292+
upload_id,
293+
exc_info=True,
294+
)
295+
else:
296+
await UploadChunk.filter(upload_id=upload_id).delete()
277297
await release_storage(f"chunk:{upload_id}")
278298
return {"code": code, "name": safe_file_name}
279299
except ValueError as e:

‎apps/base/tasks.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ async def clean_incomplete_uploads():
8080
logger.error(
8181
f"清理分片文件失败 upload_id={session.upload_id}: {e}"
8282
)
83+
# 存储清理失败(包括部分删除)时保留记录,下轮继续重试。
84+
continue
8385

8486
try:
8587
await UploadChunk.filter(upload_id=session.upload_id).delete()

‎apps/base/views.py‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import os
44
import uuid
55
from datetime import timedelta
6-
from urllib.parse import quote, unquote
6+
from urllib.parse import unquote
77

88
from typing import Optional, Tuple, Union
99

@@ -24,7 +24,7 @@
2424
stored_file_of,
2525
validate_file_size,
2626
)
27-
from core.storage import StoredFile
27+
from core.storage import StoredFile, build_attachment_headers
2828
from core.logger import logger
2929
from apps.base.schemas import (
3030
SelectFileModel,
@@ -212,11 +212,7 @@ async def get_code_file(code: str, ip: str = Depends(ip_limit["error"])):
212212
return Response(
213213
content=file_code.text,
214214
media_type="text/plain",
215-
headers={
216-
"Content-Disposition": (
217-
f"attachment; filename*=UTF-8''{quote(filename, safe='')}"
218-
)
219-
},
215+
headers=build_attachment_headers(filename),
220216
)
221217
return response_from_download(await get_stored_download(file_code, file_storage))
222218

@@ -313,6 +309,7 @@ async def init_chunk_upload(data: InitChunkUploadModel = Depends(parse_init_chun
313309
existing_session = await UploadChunk.filter(
314310
chunk_hash=data.file_hash,
315311
chunk_index=-1,
312+
completed=False,
316313
file_size=data.file_size,
317314
file_name=safe_file_name,
318315
).first()
@@ -331,7 +328,7 @@ async def init_chunk_upload(data: InitChunkUploadModel = Depends(parse_init_chun
331328
* 3600,
332329
)
333330
uploaded_chunks = await UploadChunk.filter(
334-
upload_id=existing_session.upload_id, completed=True
331+
upload_id=existing_session.upload_id, completed=True, chunk_index__gte=0
335332
).values_list("chunk_index", flat=True)
336333
return APIResponse(
337334
detail={
@@ -391,6 +388,8 @@ async def upload_chunk(
391388
chunk_info = await UploadChunk.filter(upload_id=upload_id, chunk_index=-1).first()
392389
if not chunk_info:
393390
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="上传会话不存在")
391+
if chunk_info.completed:
392+
raise HTTPException(409, "上传会话已完成,仅待清理")
394393
await reserve_storage(
395394
f"chunk:{upload_id}",
396395
chunk_info.file_size,
@@ -425,7 +424,7 @@ async def upload_chunk(
425424

426425
# 计算已上传分片数,校验累计大小不超限(用分片数 * chunk_size 估算)
427426
uploaded_count = await UploadChunk.filter(
428-
upload_id=upload_id, completed=True
427+
upload_id=upload_id, completed=True, chunk_index__gte=0
429428
).count()
430429
# 已上传分片的最大可能大小 + 当前分片
431430
max_uploaded_size = uploaded_count * chunk_info.chunk_size + chunk_size
@@ -483,6 +482,7 @@ async def cancel_upload(upload_id: str):
483482
await storage.clean_chunks(upload_id, save_path)
484483
except Exception:
485484
logger.warning("取消分片上传:清理分片文件失败 upload_id=%s", upload_id, exc_info=True)
485+
raise HTTPException(503, "分片清理失败,请稍后重试")
486486

487487
# 清理数据库记录
488488
await UploadChunk.filter(upload_id=upload_id).delete()
@@ -502,7 +502,7 @@ async def get_upload_status(upload_id: str):
502502

503503
# 获取已上传的分片列表
504504
uploaded_chunks = await UploadChunk.filter(
505-
upload_id=upload_id, completed=True
505+
upload_id=upload_id, completed=True, chunk_index__gte=0
506506
).values_list("chunk_index", flat=True)
507507

508508
return APIResponse(
@@ -530,6 +530,8 @@ async def complete_upload(
530530
chunk_info = await UploadChunk.filter(upload_id=upload_id, chunk_index=-1).first()
531531
if not chunk_info:
532532
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="上传会话不存在")
533+
if chunk_info.completed:
534+
raise HTTPException(409, "上传会话已完成,仅待清理")
533535
validate_expire_style(data.expire_style)
534536
detail = await FileUploadService.complete_chunked_upload(
535537
upload_id, chunk_info, data.expire_value, data.expire_style

‎core/storage/_base.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# @File : storage.py
44
# @Software: PyCharm
55
import hashlib
6+
import re
67
from typing import BinaryIO, Optional
78
from urllib.parse import quote
89

@@ -66,8 +67,15 @@ def build_attachment_headers(filename: str, content_length=None) -> dict:
6667
(/share/download)因此永不内联渲染 HTML/SVG。此函数是唯一构造点,
6768
新增后端必须复用(tests/test_attachment_guard.py 有源码级 tripwire)。
6869
"""
70+
# Older Android download managers ignore filename*. Keep an ASCII fallback
71+
# with the extension, without allowing quoted-string/header injection.
72+
fallback = re.sub(r"[^A-Za-z0-9._ -]", "_", filename).strip(" .") or "download"
6973
encoded_filename = quote(filename, safe="")
70-
headers = {"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}"}
74+
headers = {
75+
"Content-Disposition": (
76+
f'attachment; filename="{fallback}"; filename*=UTF-8\'\'{encoded_filename}'
77+
)
78+
}
7179
if content_length is not None:
7280
headers["Content-Length"] = str(content_length)
7381
return headers

‎core/storage/s3.py‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,12 @@ def __init__(self):
4545
self.endpoint_url = settings.s3_endpoint_url
4646

4747
def _client_config(self) -> Config:
48-
config = {"signature_version": self.signature_version}
48+
# OSS 等兼容服务不支持默认 flexible checksum 的 aws-chunked trailer。
49+
config = {
50+
"signature_version": self.signature_version,
51+
"request_checksum_calculation": "when_required",
52+
"response_checksum_validation": "when_required",
53+
}
4954
s3_config = {}
5055
if self.addressing_style in {"path", "virtual", "auto"}:
5156
s3_config["addressing_style"] = self.addressing_style
@@ -277,21 +282,15 @@ async def clean_chunks(self, upload_id: str, save_path: str):
277282
:param upload_id: 上传会话ID
278283
:param save_path: 文件保存路径
279284
"""
280-
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
285+
# 尾部斜杠防止误删 upload_id 相同前缀的其他会话。
286+
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + "/"
281287
async with self._client() as s3:
282-
try:
283-
# 列出并删除所有分片对象
284-
paginator = s3.get_paginator('list_objects_v2')
285-
async for page in paginator.paginate(Bucket=self.bucket_name, Prefix=chunk_dir):
286-
objects = page.get('Contents', [])
287-
if objects:
288-
delete_objects = [{'Key': obj['Key']} for obj in objects]
289-
await s3.delete_objects(
290-
Bucket=self.bucket_name,
291-
Delete={'Objects': delete_objects}
292-
)
293-
except Exception as e:
294-
logger.warning(f"清理 S3 分片数据时出错: {e}")
288+
paginator = s3.get_paginator("list_objects_v2")
289+
async for page in paginator.paginate(Bucket=self.bucket_name, Prefix=chunk_dir):
290+
for obj in page.get("Contents", []):
291+
# OSS 批量删除要求 Content-MD5;单对象删除不依赖该扩展。
292+
# 失败向调用方传播,保留会话供重试,不能当作清理成功。
293+
await s3.delete_object(Bucket=self.bucket_name, Key=obj["Key"])
295294

296295
async def generate_presigned_upload_url(self, save_path: str, expires_in: int = 900) -> Optional[str]:
297296
"""

‎docs/en/guide/configuration.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ FileCodeBox provides rich configuration options. Prefer the admin panel; directl
66

77
FileCodeBox supports two configuration methods:
88

9-
1. **Admin Panel Configuration** (Recommended): Access `/admin` to enter the admin panel and modify settings on the settings page
9+
1. **Admin Panel Configuration** (Recommended): Access `/#/admin` to enter the admin panel and modify settings on the settings page
1010
2. **Database Configuration**: Configuration is stored in the `data/filecodebox.db` database
1111

1212
::: tip Note

‎docs/en/guide/introduction.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ python main.py
179179
3. Download file or view text
180180

181181
### Admin Panel
182-
1. Visit `/admin`
182+
1. Visit `/#/admin`
183183
2. Enter admin password
184184
3. Manage files and settings
185185

‎docs/en/guide/management.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ FileCodeBox provides a fully-featured admin panel that allows administrators to
1010

1111
### Login Method
1212

13-
The admin panel is located at the `/admin` path. Access method:
13+
The admin panel is located at `/#/admin` (a frontend hash route). Access method:
1414

1515
1. On first run, set the admin password on the setup page
16-
2. Visit `http://your-domain.com/admin` in your browser
16+
2. Visit `http://your-domain.com/#/admin` in your browser
1717
3. Enter the admin password (the value of the `admin_token` configuration)
1818
4. Click the login button
1919

@@ -30,7 +30,7 @@ By default, the admin panel entry is not shown on the homepage. You can control
3030
| `show_admin_addr` | int | `0` | Show admin entry on homepage (1=show, 0=hide) |
3131

3232
::: warning Security Recommendation
33-
For public services, it's recommended to keep `show_admin_addr` at `0` and access the admin panel directly via the `/admin` path to reduce the risk of malicious scanning.
33+
For public services, keep `show_admin_addr` at `0` and access the admin panel via `/#/admin`. Hiding the link is not access control; a strong password is still required.
3434
:::
3535

3636
### Authentication Mechanism

‎docs/en/guide/security.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ FileCodeBox does not ship with a default admin password. Open the site in your b
1212

1313
Change the admin password through the admin panel:
1414

15-
1. Access `/admin` to enter the admin panel
15+
1. Access `/#/admin` to enter the admin panel
1616
2. Log in with the current password
1717
3. Go to the "System Settings" page
1818
4. Find the `admin_token` configuration item
@@ -41,7 +41,7 @@ By default, the admin panel entry is hidden. You can control whether to show the
4141
| `show_admin_addr` | int | `0` | Show admin entry (1=show, 0=hide) |
4242

4343
::: tip Recommendation
44-
For public services, it's recommended to keep `show_admin_addr` at `0` and access the admin panel directly via the `/admin` path.
44+
For public services, it's recommended to keep `show_admin_addr` at `0` and access the admin panel directly via `/#/admin`.
4545
:::
4646

4747
## IP Rate Limiting

0 commit comments

Comments
 (0)