switch chat media messages to client-side MinIO uploads

Frontend now uploads media directly to MinIO and sends only the
resulting object_key; the backend never touches file bytes. Mattermost
post bodies carry "<type>:<object_key>" instead of a permanent public
URL, and download links are signed fresh on every read (send + list)
so they can't outlive their expiry. Also wires up MINIO_SECURE, which
settings.py previously never read from env.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Ali Asadi 2026-07-12 12:12:31 +03:30
parent 49d6bd6cd2
commit d85be31e77
8 changed files with 136 additions and 63 deletions

View file

@ -27,10 +27,15 @@ MATTERMOST_SERVICE_USERNAME=chat-service
MATTERMOST_SERVICE_EMAIL=chat-service@local.invalid MATTERMOST_SERVICE_EMAIL=chat-service@local.invalid
# MinIO object storage # MinIO object storage
# The frontend uploads media directly to MinIO and sends us only the
# resulting object key — these credentials are used solely to sign
# short-lived download URLs when messages are read back.
MINIO_ENDPOINT=localhost:9000 MINIO_ENDPOINT=localhost:9000
MINIO_ACCESS_KEY=minioadmin MINIO_ACCESS_KEY=minioadmin
MINIO_SECRET_KEY=minioadmin MINIO_SECRET_KEY=minioadmin
MINIO_BUCKET_CHAT=chat MINIO_BUCKET_CHAT=chat
MINIO_SECURE=false
MINIO_PRESIGN_EXPIRY_SECONDS=3600
# Chat long-poll tuning # Chat long-poll tuning
CHAT_LONG_POLL_TIMEOUT_SECONDS=25 CHAT_LONG_POLL_TIMEOUT_SECONDS=25

View file

@ -1,11 +1,28 @@
from rest_framework import serializers from rest_framework import serializers
MEDIA_MESSAGE_TYPES = {"image", "video", "voice"}
class SendMessageSerializer(serializers.Serializer): class SendMessageSerializer(serializers.Serializer):
sender_id = serializers.UUIDField() sender_id = serializers.UUIDField()
message_type = serializers.ChoiceField(choices=["text", "image", "file"], default="text") message_type = serializers.ChoiceField(
choices=["text", "image", "video", "voice"], default="text"
)
text = serializers.CharField(required=False, allow_null=True, allow_blank=True) text = serializers.CharField(required=False, allow_null=True, allow_blank=True)
file = serializers.FileField(required=False, allow_null=True) object_key = serializers.CharField(required=False, allow_null=True, allow_blank=True)
def validate(self, attrs):
message_type = attrs.get("message_type", "text")
if message_type in MEDIA_MESSAGE_TYPES:
if not attrs.get("object_key"):
raise serializers.ValidationError(
{"object_key": "object_key is required for image, video, and voice messages."}
)
elif not attrs.get("text"):
raise serializers.ValidationError(
{"text": "text is required for text messages."}
)
return attrs
class MessageSerializer(serializers.Serializer): class MessageSerializer(serializers.Serializer):

View file

@ -10,6 +10,8 @@ from apps.chat.models import Conversation, MattermostAccountMapping
from apps.chat.services.account import AccountService from apps.chat.services.account import AccountService
from apps.chat.services.storage import StorageService from apps.chat.services.storage import StorageService
MEDIA_MESSAGE_TYPES = {"image", "video", "voice"}
class MessageService: class MessageService:
def __init__( def __init__(
@ -40,20 +42,20 @@ class MessageService:
sender_id: UUID, sender_id: UUID,
message_type: str, message_type: str,
text: str | None = None, text: str | None = None,
file=None, object_key: str | None = None,
) -> dict: ) -> dict:
if not self._account.validate_user(sender_id): if not self._account.validate_user(sender_id):
raise ValueError(f"Sender {sender_id} is not valid") raise ValueError(f"Sender {sender_id} is not valid")
file_url: str | None = None # Media was already uploaded straight to MinIO by the client; the
if file is not None: # Mattermost post body only ever carries "<type>:<object_key>", never
filename = getattr(file, "name", f"{sender_id}") # a URL, so a stored link can't outlive its presigned expiry.
file_url = self._storage.upload_file(file, filename) download_url: str | None = None
if message_type in MEDIA_MESSAGE_TYPES:
if message_type == "text": mm_message = f"{message_type}:{object_key}"
mm_message = text or "" download_url = self._storage.get_download_url(object_key)
else: else:
mm_message = file_url or text or "" mm_message = text or ""
conversation = Conversation.objects.get(id=conversation_id) conversation = Conversation.objects.get(id=conversation_id)
post_id = self._mm.post_message(conversation.mattermost_channel_id, mm_message) post_id = self._mm.post_message(conversation.mattermost_channel_id, mm_message)
@ -63,7 +65,7 @@ class MessageService:
post_id=post_id, post_id=post_id,
sender_id=sender_id, sender_id=sender_id,
message_type=message_type, message_type=message_type,
payload={"text": text, "file_url": file_url}, payload={"text": text, "object_key": object_key, "url": download_url},
) )
for publisher in self._publishers: for publisher in self._publishers:
publisher.publish(event) publisher.publish(event)
@ -73,7 +75,7 @@ class MessageService:
"sender_id": sender_id, "sender_id": sender_id,
"message_type": message_type, "message_type": message_type,
"text": text, "text": text,
"url": file_url, "url": download_url,
"created_at": None, "created_at": None,
} }
@ -102,23 +104,33 @@ class MessageService:
return [self._normalize_post(p, mappings) for p in raw_posts] return [self._normalize_post(p, mappings) for p in raw_posts]
@staticmethod def _normalize_post(self, post: dict, mappings: dict) -> dict:
def _normalize_post(post: dict, mappings: dict) -> dict:
mm_uid = post.get("user_id") mm_uid = post.get("user_id")
sender_id = mappings.get(mm_uid) sender_id = mappings.get(mm_uid)
msg = post.get("message", "") msg = post.get("message", "")
is_url = msg.startswith("http")
created_ms = post.get("create_at") created_ms = post.get("create_at")
created_at = ( created_at = (
datetime.fromtimestamp(created_ms / 1000, tz=timezone.utc) datetime.fromtimestamp(created_ms / 1000, tz=timezone.utc)
if created_ms if created_ms
else None else None
) )
message_type, sep, object_key = msg.partition(":")
if sep and message_type in MEDIA_MESSAGE_TYPES and object_key:
return {
"post_id": post.get("id"),
"sender_id": sender_id,
"message_type": message_type,
"text": None,
"url": self._storage.get_download_url(object_key),
"created_at": created_at,
}
return { return {
"post_id": post.get("id"), "post_id": post.get("id"),
"sender_id": sender_id, "sender_id": sender_id,
"message_type": "file" if is_url else "text", "message_type": "text",
"text": None if is_url else msg, "text": msg,
"url": msg if is_url else None, "url": None,
"created_at": created_at, "created_at": created_at,
} }

View file

@ -1,12 +1,17 @@
import io from datetime import timedelta
import mimetypes
from uuid import uuid4
from django.conf import settings from django.conf import settings
from minio import Minio from minio import Minio
class StorageService: class StorageService:
"""
Thin adapter around MinIO for the object-key-in-message-body flow: the
frontend uploads media directly to MinIO and only ever sends us the
resulting object key, so this service's job is limited to turning that
key back into a short-lived download URL on read.
"""
def __init__(self): def __init__(self):
endpoint = getattr(settings, "MINIO_ENDPOINT", None) endpoint = getattr(settings, "MINIO_ENDPOINT", None)
if not endpoint: if not endpoint:
@ -20,23 +25,21 @@ class StorageService:
secure=secure, secure=secure,
) )
self._bucket = getattr(settings, "MINIO_BUCKET_CHAT", "chat") self._bucket = getattr(settings, "MINIO_BUCKET_CHAT", "chat")
scheme = "https" if secure else "http" self._expiry = timedelta(
self._base_url = f"{scheme}://{endpoint}" seconds=getattr(settings, "MINIO_PRESIGN_EXPIRY_SECONDS", 3600)
def upload_file(self, file, filename: str) -> str:
data = file.read() if hasattr(file, "read") else file
size = len(data)
content_type, _ = mimetypes.guess_type(filename)
if not content_type:
content_type = "application/octet-stream"
object_name = f"{uuid4().hex}/{filename}"
self._client.put_object(
self._bucket,
object_name,
io.BytesIO(data),
size,
content_type=content_type,
) )
return f"{self._base_url}/{self._bucket}/{object_name}"
def get_download_url(self, object_key: str) -> str | None:
"""
Sign a time-limited download URL for an object the client already
uploaded. Generated fresh on every call rather than cached/stored,
since a stored link would eventually expire while still displayed.
Returns None on failure so one bad key can't fail an entire message
list — callers should treat that as "link unavailable", not an error.
"""
try:
return self._client.presigned_get_object(
self._bucket, object_key, expires=self._expiry
)
except Exception:
return None

View file

@ -7,14 +7,13 @@ from apps.chat.models import Conversation, MattermostAccountMapping
from apps.chat.services.message import MessageService from apps.chat.services.message import MessageService
def _make_service(*, mm_post_id="post-1", storage_url=None, publishers=None): def _make_service(*, mm_post_id="post-1", download_url=None, publishers=None):
"""Return a MessageService with all external deps mocked.""" """Return a MessageService with all external deps mocked."""
mm_client = Mock() mm_client = Mock()
mm_client.post_message.return_value = mm_post_id mm_client.post_message.return_value = mm_post_id
storage = Mock() storage = Mock()
if storage_url: storage.get_download_url.return_value = download_url
storage.upload_file.return_value = storage_url
return ( return (
MessageService( MessageService(
@ -33,11 +32,12 @@ def test_send_text_message():
sender_id = uuid.uuid4() sender_id = uuid.uuid4()
ws_publisher = Mock() ws_publisher = Mock()
svc, mm_client, _ = _make_service(mm_post_id="post-abc", publishers=[ws_publisher]) svc, mm_client, storage = _make_service(mm_post_id="post-abc", publishers=[ws_publisher])
result = svc.send(conv.id, sender_id, "text", text="Hello world") result = svc.send(conv.id, sender_id, "text", text="Hello world")
mm_client.post_message.assert_called_once_with("ch-send-text", "Hello world") mm_client.post_message.assert_called_once_with("ch-send-text", "Hello world")
storage.get_download_url.assert_not_called()
ws_publisher.publish.assert_called_once() ws_publisher.publish.assert_called_once()
event = ws_publisher.publish.call_args[0][0] event = ws_publisher.publish.call_args[0][0]
@ -47,6 +47,7 @@ def test_send_text_message():
assert result["post_id"] == "post-abc" assert result["post_id"] == "post-abc"
assert result["text"] == "Hello world" assert result["text"] == "Hello world"
assert result["url"] is None
@pytest.mark.django_db @pytest.mark.django_db
@ -54,17 +55,16 @@ def test_send_image_message():
conv = Conversation.objects.create(mattermost_channel_id="ch-send-image") conv = Conversation.objects.create(mattermost_channel_id="ch-send-image")
sender_id = uuid.uuid4() sender_id = uuid.uuid4()
file_url = "http://minio.local/chat/abc123/photo.jpg" object_key = "images/abc123/photo.jpg"
svc, mm_client, storage = _make_service(storage_url=file_url, publishers=[]) download_url = "http://minio.local/chat/images/abc123/photo.jpg?X-Amz-Signature=..."
svc, mm_client, storage = _make_service(download_url=download_url, publishers=[])
fake_file = Mock() result = svc.send(conv.id, sender_id, "image", object_key=object_key)
fake_file.name = "photo.jpg"
result = svc.send(conv.id, sender_id, "image", file=fake_file) storage.get_download_url.assert_called_once_with(object_key)
mm_client.post_message.assert_called_once_with("ch-send-image", f"image:{object_key}")
storage.upload_file.assert_called_once_with(fake_file, "photo.jpg") assert result["url"] == download_url
mm_client.post_message.assert_called_once_with("ch-send-image", file_url) assert result["message_type"] == "image"
assert result["url"] == file_url
@pytest.mark.django_db @pytest.mark.django_db
@ -77,6 +77,7 @@ def test_list_messages_normalized():
user_id=sender_uuid, mattermost_user_id=mm_user_id user_id=sender_uuid, mattermost_user_id=mm_user_id
) )
object_key = "voice/def456/clip.ogg"
raw_posts = [ raw_posts = [
{ {
"id": "p1", "id": "p1",
@ -87,14 +88,15 @@ def test_list_messages_normalized():
{ {
"id": "p2", "id": "p2",
"user_id": mm_user_id, "user_id": mm_user_id,
"message": "http://minio.local/chat/file.pdf", "message": f"voice:{object_key}",
"create_at": 1700000001000, "create_at": 1700000001000,
}, },
] ]
download_url = "http://minio.local/chat/voice/def456/clip.ogg?X-Amz-Signature=..."
mm_client = Mock() mm_client = Mock()
mm_client.get_posts.return_value = raw_posts mm_client.get_posts.return_value = raw_posts
svc, _, _ = _make_service(publishers=[]) svc, _, storage = _make_service(download_url=download_url, publishers=[])
svc._mm = mm_client # inject after construction to keep _make_service simple svc._mm = mm_client # inject after construction to keep _make_service simple
messages = svc.list_messages(conv.id, page=0, per_page=20) messages = svc.list_messages(conv.id, page=0, per_page=20)
@ -108,9 +110,39 @@ def test_list_messages_normalized():
assert text_msg["text"] == "First message" assert text_msg["text"] == "First message"
assert text_msg["url"] is None assert text_msg["url"] is None
file_msg = messages[1] voice_msg = messages[1]
assert file_msg["post_id"] == "p2" assert voice_msg["post_id"] == "p2"
assert file_msg["sender_id"] == sender_uuid assert voice_msg["sender_id"] == sender_uuid
assert file_msg["message_type"] == "file" assert voice_msg["message_type"] == "voice"
assert file_msg["url"] == "http://minio.local/chat/file.pdf" assert voice_msg["url"] == download_url
assert file_msg["text"] is None assert voice_msg["text"] is None
storage.get_download_url.assert_called_once_with(object_key)
@pytest.mark.django_db
def test_list_messages_treats_colon_in_plain_text_as_text():
"""A plain-text message that happens to contain a colon but doesn't use
a recognized media-type prefix must not be mistaken for a media message.
"""
conv = Conversation.objects.create(mattermost_channel_id="ch-list-colon")
raw_posts = [
{
"id": "p1",
"user_id": None,
"message": "Meeting at 10:30",
"create_at": 1700000000000,
},
]
mm_client = Mock()
mm_client.get_posts.return_value = raw_posts
svc, _, storage = _make_service(publishers=[])
svc._mm = mm_client
messages = svc.list_messages(conv.id, page=0, per_page=20)
assert messages[0]["message_type"] == "text"
assert messages[0]["text"] == "Meeting at 10:30"
storage.get_download_url.assert_not_called()

View file

@ -27,7 +27,7 @@ class MessageView(APIView):
sender_id=data["sender_id"], sender_id=data["sender_id"],
message_type=data["message_type"], message_type=data["message_type"],
text=data.get("text"), text=data.get("text"),
file=data.get("file"), object_key=data.get("object_key"),
) )
return Response(MessageSerializer(result).data, status=status.HTTP_201_CREATED) return Response(MessageSerializer(result).data, status=status.HTTP_201_CREATED)

View file

@ -24,3 +24,5 @@ MINIO_ENDPOINT=localhost:9000
MINIO_ACCESS_KEY=minioadmin MINIO_ACCESS_KEY=minioadmin
MINIO_SECRET_KEY=minioadmin MINIO_SECRET_KEY=minioadmin
MINIO_BUCKET_CHAT=chat MINIO_BUCKET_CHAT=chat
MINIO_SECURE=false
MINIO_PRESIGN_EXPIRY_SECONDS=3600

View file

@ -288,5 +288,7 @@ MINIO_ENDPOINT = config('MINIO_ENDPOINT', default=None)
MINIO_ACCESS_KEY = config('MINIO_ACCESS_KEY', default=None) MINIO_ACCESS_KEY = config('MINIO_ACCESS_KEY', default=None)
MINIO_SECRET_KEY = config('MINIO_SECRET_KEY', default=None) MINIO_SECRET_KEY = config('MINIO_SECRET_KEY', default=None)
MINIO_BUCKET_CHAT = config('MINIO_BUCKET_CHAT', default='chat') MINIO_BUCKET_CHAT = config('MINIO_BUCKET_CHAT', default='chat')
MINIO_SECURE = config('MINIO_SECURE', default=False, cast=bool)
MINIO_PRESIGN_EXPIRY_SECONDS = config('MINIO_PRESIGN_EXPIRY_SECONDS', default=3600, cast=int)
GDAL_LIBRARY_PATH = config('GDAL_LIBRARY_PATH', default=None) GDAL_LIBRARY_PATH = config('GDAL_LIBRARY_PATH', default=None)
GEOS_LIBRARY_PATH = config('GEOS_LIBRARY_PATH', default=None) GEOS_LIBRARY_PATH = config('GEOS_LIBRARY_PATH', default=None)