From 889da74ce7af390cfa06d0d94291339abb7e6573 Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Thu, 4 Dec 2025 17:03:04 +0530 Subject: [PATCH 1/5] feat: enhance APITokenLogMiddleware to support logging to MongoDB - Added functionality to log external API requests to MongoDB, with a fallback to PostgreSQL if MongoDB is unavailable. - Implemented error handling for MongoDB connection and logging operations. - Introduced additional fields for MongoDB logs, including timestamps and user identifiers. - Refactored request logging logic to streamline the process and improve maintainability. --- apps/api/plane/middleware/logger.py | 112 +++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 19 deletions(-) diff --git a/apps/api/plane/middleware/logger.py b/apps/api/plane/middleware/logger.py index d513ee3e36d..e24e568eab0 100644 --- a/apps/api/plane/middleware/logger.py +++ b/apps/api/plane/middleware/logger.py @@ -4,6 +4,7 @@ # Django imports from django.http import HttpRequest +from django.utils import timezone # Third party imports from rest_framework.request import Request @@ -11,6 +12,8 @@ # Module imports from plane.utils.ip_address import get_client_ip from plane.db.models import APIActivityLog +from plane.settings.mongo import MongoConnection +from plane.utils.exception_logger import log_exception api_logger = logging.getLogger("plane.api.request") @@ -70,8 +73,32 @@ def __call__(self, request): class APITokenLogMiddleware: + """ + Middleware to log External API requests to MongoDB or PostgreSQL. + """ + def __init__(self, get_response): self.get_response = get_response + self.mongo_available = False + + # Initialize MongoDB collection + try: + self.mongo_collection = self.get_mongo_collection() + self.mongo_available = True + except Exception as e: + api_logger.error(f"Error getting MongoDB collection: {str(e)}") + log_exception(e) + + def get_mongo_collection(self): + """ + Returns the MongoDB collection for API activity logs. + """ + try: + return MongoConnection.get_collection("api_activity_logs") + except Exception as e: + api_logger.error(f"Error getting MongoDB collection: {str(e)}") + log_exception(e) + return None def __call__(self, request): request_body = request.body @@ -101,27 +128,74 @@ def _safe_decode_body(self, content): except UnicodeDecodeError: return "[Could not decode content]" + def log_to_mongo(self, log_document): + """ + Logs the request to MongoDB if available. + """ + + if not self.mongo_available: + return False + + try: + self.mongo_collection.insert_one(log_document) + return True + except Exception as e: + log_exception(e) + self.mongo_available = self.get_mongo_collection() is not None + return False + + def log_to_postgres(self, log_data): + """ + Fallback to logging to PostgreSQL if MongoDB is unavailable. + """ + try: + APIActivityLog.objects.create(**log_data) + return True + except Exception as e: + log_exception(e) + return False + def process_request(self, request, response, request_body): api_key_header = "X-Api-Key" api_key = request.headers.get(api_key_header) - # If the API key is present, log the request - if api_key: - try: - APIActivityLog.objects.create( - token_identifier=api_key, - path=request.path, - method=request.method, - query_params=request.META.get("QUERY_STRING", ""), - headers=str(request.headers), - body=(self._safe_decode_body(request_body) if request_body else None), - response_body=(self._safe_decode_body(response.content) if response.content else None), - response_code=response.status_code, - ip_address=get_client_ip(request=request), - user_agent=request.META.get("HTTP_USER_AGENT", None), - ) - - except Exception as e: - api_logger.exception(e) - # If the token does not exist, you can decide whether to log this as an invalid attempt + + # If the API key is not present, return + if not api_key: + return + + try: + log_data = { + "token_identifier": api_key, + "path": request.path, + "method": request.method, + "query_params": request.META.get("QUERY_STRING", ""), + "headers": str(request.headers), + "body": self._safe_decode_body(request_body) if request_body else None, + "response_body": self._safe_decode_body(response.content) if response.content else None, + "response_code": response.status_code, + "ip_address": get_client_ip(request=request), + "user_agent": request.META.get("HTTP_USER_AGENT", None), + } + user_id = ( + str(request.user.id) + if getattr(request, "user") and getattr(request.user, "is_authenticated", False) + else None + ) + # Additional fields for MongoDB + mongo_log = { + **log_data, + "created_at": timezone.now(), + "updated_at": timezone.now(), + "created_by": user_id, + "updated_by": user_id, + } + + # Log to MongoDB if available + if not self.log_to_mongo(mongo_log): + # Fallback to logging to PostgreSQL + self.log_to_postgres(log_data) + + except Exception as e: + log_exception(e) return None From 415f6e33f8d35314535988d902ff65929c473c22 Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Thu, 4 Dec 2025 21:13:49 +0530 Subject: [PATCH 2/5] fix: improve MongoDB availability checks in APITokenLogMiddleware - Enhanced the logic for determining MongoDB availability by checking if the collection is not None. - Added a check for MongoDB configuration before attempting to retrieve the collection. - Updated error handling to ensure the middleware correctly reflects the state of MongoDB connectivity. --- apps/api/plane/middleware/logger.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/api/plane/middleware/logger.py b/apps/api/plane/middleware/logger.py index e24e568eab0..ee39e50988d 100644 --- a/apps/api/plane/middleware/logger.py +++ b/apps/api/plane/middleware/logger.py @@ -84,7 +84,7 @@ def __init__(self, get_response): # Initialize MongoDB collection try: self.mongo_collection = self.get_mongo_collection() - self.mongo_available = True + self.mongo_available = True if self.mongo_collection is not None else False except Exception as e: api_logger.error(f"Error getting MongoDB collection: {str(e)}") log_exception(e) @@ -93,6 +93,10 @@ def get_mongo_collection(self): """ Returns the MongoDB collection for API activity logs. """ + if not MongoConnection.is_configured(): + api_logger.info("MongoDB not configured") + return None + try: return MongoConnection.get_collection("api_activity_logs") except Exception as e: @@ -133,7 +137,7 @@ def log_to_mongo(self, log_document): Logs the request to MongoDB if available. """ - if not self.mongo_available: + if not self.mongo_available or self.mongo_collection is None: return False try: @@ -141,7 +145,8 @@ def log_to_mongo(self, log_document): return True except Exception as e: log_exception(e) - self.mongo_available = self.get_mongo_collection() is not None + self.mongo_collection = self.get_mongo_collection() + self.mongo_available = True if self.mongo_collection is not None else False return False def log_to_postgres(self, log_data): From 6a9534192a1192fa285640fe822490bbfff4cea8 Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Fri, 5 Dec 2025 17:46:48 +0530 Subject: [PATCH 3/5] feat: implement logging functionality in logger_task for API activity - Added a new logger_task module to handle logging of API activity to MongoDB and PostgreSQL. - Introduced functions for safely decoding request/response bodies and processing logs based on MongoDB availability. - Refactored APITokenLogMiddleware to utilize the new logging functions, improving code organization and maintainability. --- apps/api/plane/bgtasks/logger_task.py | 93 +++++++++++++++++++++++++++ apps/api/plane/middleware/logger.py | 58 +---------------- 2 files changed, 95 insertions(+), 56 deletions(-) create mode 100644 apps/api/plane/bgtasks/logger_task.py diff --git a/apps/api/plane/bgtasks/logger_task.py b/apps/api/plane/bgtasks/logger_task.py new file mode 100644 index 00000000000..96c2e62c517 --- /dev/null +++ b/apps/api/plane/bgtasks/logger_task.py @@ -0,0 +1,93 @@ +# Python imports +import logging +from typing import Optional, Dict, Any + +# Third party imports +from pymongo.collection import Collection + +# Django imports +from plane.settings.mongo import MongoConnection +from plane.utils.exception_logger import log_exception +from plane.db.models import APIActivityLog + + +logger = logging.getLogger("plane.worker") + + +def get_mongo_collection() -> Optional[Collection]: + """ + Returns the MongoDB collection for external API activity logs. + """ + if not MongoConnection.is_configured(): + logger.info("MongoDB not configured") + return None + + try: + return MongoConnection.get_collection("api_activity_logs") + except Exception as e: + logger.error(f"Error getting MongoDB collection: {str(e)}") + log_exception(e) + return None + + +def safe_decode_body(content: bytes) -> Optional[str]: + """ + Safely decodes request/response body content, handling binary data. + Returns "[Binary Content]" if the content is binary, or a string representation of the content. + Returns None if the content is None or empty. + """ + # If the content is None, return None + if content is None: + return None + + # If the content is an empty bytes object, return None + if content == b"": + return None + + # Check if content is binary by looking for common binary file signatures + if content.startswith(b"\x89PNG") or content.startswith(b"\xff\xd8\xff") or content.startswith(b"%PDF"): + return "[Binary Content]" + + try: + return content.decode("utf-8") + except UnicodeDecodeError: + return "[Could not decode content]" + + +def log_to_mongo(mongo_collection: Optional[Collection], log_document: Dict[str, Any]) -> bool: + """ + Logs the request to MongoDB if available. + """ + + if mongo_collection is None: + return False + + try: + mongo_collection.insert_one(log_document) + return True + except Exception as e: + log_exception(e) + return False + + +def log_to_postgres(log_data: Dict[str, Any]) -> bool: + """ + Fallback to logging to PostgreSQL if MongoDB is unavailable. + """ + try: + APIActivityLog.objects.create(**log_data) + return True + except Exception as e: + log_exception(e) + return False + + +def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]): + """ + Process logs to save to MongoDB or Postgres based on the configuration + """ + mongo_collection = get_mongo_collection() + if mongo_collection is not None: + log_to_mongo(mongo_collection, mongo_log) + else: + log_to_postgres(log_data) diff --git a/apps/api/plane/middleware/logger.py b/apps/api/plane/middleware/logger.py index ee39e50988d..d28784a13ca 100644 --- a/apps/api/plane/middleware/logger.py +++ b/apps/api/plane/middleware/logger.py @@ -14,6 +14,7 @@ from plane.db.models import APIActivityLog from plane.settings.mongo import MongoConnection from plane.utils.exception_logger import log_exception +from plane.bgtasks.logger_task import process_logs api_logger = logging.getLogger("plane.api.request") @@ -79,30 +80,6 @@ class APITokenLogMiddleware: def __init__(self, get_response): self.get_response = get_response - self.mongo_available = False - - # Initialize MongoDB collection - try: - self.mongo_collection = self.get_mongo_collection() - self.mongo_available = True if self.mongo_collection is not None else False - except Exception as e: - api_logger.error(f"Error getting MongoDB collection: {str(e)}") - log_exception(e) - - def get_mongo_collection(self): - """ - Returns the MongoDB collection for API activity logs. - """ - if not MongoConnection.is_configured(): - api_logger.info("MongoDB not configured") - return None - - try: - return MongoConnection.get_collection("api_activity_logs") - except Exception as e: - api_logger.error(f"Error getting MongoDB collection: {str(e)}") - log_exception(e) - return None def __call__(self, request): request_body = request.body @@ -132,34 +109,6 @@ def _safe_decode_body(self, content): except UnicodeDecodeError: return "[Could not decode content]" - def log_to_mongo(self, log_document): - """ - Logs the request to MongoDB if available. - """ - - if not self.mongo_available or self.mongo_collection is None: - return False - - try: - self.mongo_collection.insert_one(log_document) - return True - except Exception as e: - log_exception(e) - self.mongo_collection = self.get_mongo_collection() - self.mongo_available = True if self.mongo_collection is not None else False - return False - - def log_to_postgres(self, log_data): - """ - Fallback to logging to PostgreSQL if MongoDB is unavailable. - """ - try: - APIActivityLog.objects.create(**log_data) - return True - except Exception as e: - log_exception(e) - return False - def process_request(self, request, response, request_body): api_key_header = "X-Api-Key" api_key = request.headers.get(api_key_header) @@ -195,10 +144,7 @@ def process_request(self, request, response, request_body): "updated_by": user_id, } - # Log to MongoDB if available - if not self.log_to_mongo(mongo_log): - # Fallback to logging to PostgreSQL - self.log_to_postgres(log_data) + process_logs.delay(log_data=log_data, mongo_log=mongo_log) except Exception as e: log_exception(e) From acc39709f5e52259679093fb8748a608d1f3f56d Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Fri, 5 Dec 2025 18:01:46 +0530 Subject: [PATCH 4/5] refactor: simplify MongoDB logging in logger_task and middleware - Removed direct dependency on MongoDB collection in log_to_mongo function, now retrieving it internally. - Updated process_logs to check MongoDB configuration before logging, enhancing error handling. - Cleaned up logger.py by removing unused imports related to MongoDB. --- apps/api/plane/bgtasks/logger_task.py | 11 ++++++----- apps/api/plane/middleware/logger.py | 2 -- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/apps/api/plane/bgtasks/logger_task.py b/apps/api/plane/bgtasks/logger_task.py index 96c2e62c517..cfb118ea0de 100644 --- a/apps/api/plane/bgtasks/logger_task.py +++ b/apps/api/plane/bgtasks/logger_task.py @@ -54,12 +54,13 @@ def safe_decode_body(content: bytes) -> Optional[str]: return "[Could not decode content]" -def log_to_mongo(mongo_collection: Optional[Collection], log_document: Dict[str, Any]) -> bool: +def log_to_mongo(log_document: Dict[str, Any]) -> bool: """ Logs the request to MongoDB if available. """ - + mongo_collection = get_mongo_collection() if mongo_collection is None: + logger.error("MongoDB not configured") return False try: @@ -86,8 +87,8 @@ def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]): """ Process logs to save to MongoDB or Postgres based on the configuration """ - mongo_collection = get_mongo_collection() - if mongo_collection is not None: - log_to_mongo(mongo_collection, mongo_log) + + if MongoConnection.is_configured(): + log_to_mongo(mongo_log) else: log_to_postgres(log_data) diff --git a/apps/api/plane/middleware/logger.py b/apps/api/plane/middleware/logger.py index d28784a13ca..07facdab0e0 100644 --- a/apps/api/plane/middleware/logger.py +++ b/apps/api/plane/middleware/logger.py @@ -11,8 +11,6 @@ # Module imports from plane.utils.ip_address import get_client_ip -from plane.db.models import APIActivityLog -from plane.settings.mongo import MongoConnection from plane.utils.exception_logger import log_exception from plane.bgtasks.logger_task import process_logs From e83ddd2439a045c068cd8d196770a7ab585320b7 Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Fri, 5 Dec 2025 18:05:29 +0530 Subject: [PATCH 5/5] feat: add Celery task decorator to process_logs function in logger_task - Introduced the @shared_task decorator to the process_logs function, enabling asynchronous processing of log data. - Updated function signature to include a return type of None for clarity. --- apps/api/plane/bgtasks/logger_task.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/plane/bgtasks/logger_task.py b/apps/api/plane/bgtasks/logger_task.py index cfb118ea0de..01723ef77a3 100644 --- a/apps/api/plane/bgtasks/logger_task.py +++ b/apps/api/plane/bgtasks/logger_task.py @@ -4,6 +4,7 @@ # Third party imports from pymongo.collection import Collection +from celery import shared_task # Django imports from plane.settings.mongo import MongoConnection @@ -83,7 +84,8 @@ def log_to_postgres(log_data: Dict[str, Any]) -> bool: return False -def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]): +@shared_task +def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]) -> None: """ Process logs to save to MongoDB or Postgres based on the configuration """