fix(sync): 流式处理已完成对象且不泄漏传输资源

This commit is contained in:
2026-09-09 13:06:28 +08:00
parent 2858babfe3
commit ebb042695a
2 changed files with 26 additions and 6 deletions
+11 -4
View File
@@ -76,11 +76,18 @@ class S3Objects:
return stream.read()
def put_file(self, key: str, path: Path, content_hash: str):
from boto3.s3.transfer import TransferConfig
with path.open("rb") as stream:
self.client.upload_fileobj(stream, self.bucket, key,
ExtraArgs={"Metadata": {"sha256": content_hash}},
Config=TransferConfig(use_threads=False, max_concurrency=1))
# Objects are capped at 100 MiB, well below S3's 5 GiB single-PUT
# limit. A direct streaming request has one explicit connection
# lifetime; constructing a transfer manager per completion can
# retain pooled MinIO connections under repeated multi-worker use.
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=stream,
ContentLength=path.stat().st_size,
Metadata={"sha256": content_hash},
)
@contextmanager
def open(self, key: str):