diff --git a/apps/discord_bot/src/five08/discord_bot/cogs/crm.py b/apps/discord_bot/src/five08/discord_bot/cogs/crm.py index c2f95f13..961bc7e3 100644 --- a/apps/discord_bot/src/five08/discord_bot/cogs/crm.py +++ b/apps/discord_bot/src/five08/discord_bot/cogs/crm.py @@ -10,12 +10,10 @@ import io import json import logging -from datetime import date, datetime, timezone +from datetime import date, datetime import re from typing import Any, Literal -from uuid import uuid4 -import aiohttp import discord from discord import app_commands from discord.ext import commands @@ -30,6 +28,10 @@ is_reserved_resume_name_token, normalize_resume_name_token, ) +from five08.resume_profile_processor import ( + ResumeProcessorConfig, + ResumeProfileProcessor, +) from five08.skills import normalize_skill, normalize_skill_list from five08.discord_bot.utils.audit import DiscordAuditCogMixin from five08.discord_bot.utils.role_decorators import ( @@ -50,15 +52,8 @@ ID_VERIFIED_AT_FIELD = "cIdVerifiedAt" ID_VERIFIED_BY_FIELD = "cIdVerifiedBy" ID_VERIFIED_TYPE_FIELD = "cVerifiedIdType" -ONBOARDING_STATUS_FIELD_CANDIDATES = ( - "cOnboardingState", - "cOnboardingStatus", - "cOnboarding", -) -ONBOARDER_FIELD_CANDIDATES = ( - "cOnboarder", - "cOnboardingCoordinator", -) +ONBOARDING_STATUS_FIELD = "cOnboardingState" +ONBOARDER_FIELD = "cOnboarder" _DISCORD_ROLES_PROTECTED_FROM_APPLY: frozenset[str] = frozenset( {"Member", "Admin", "Steering Committee"} ) @@ -68,20 +63,14 @@ EXCLUDED_ONBOARDING_STATES = frozenset({"onboarded", "waitlist", "rejected"}) ONBOARDING_QUEUE_MAX_SIZE = 200 ONBOARDING_QUEUE_PAGE_SIZE = 1 +ESPO_DATE_FORMAT = "%Y-%m-%d" +ESPO_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" +LINKEDIN_FIELD = "cLinkedIn" EspoClient = espo.EspoClient EspoAPI = EspoClient EspoAPIError = espo.EspoAPIError -def _configured_linkedin_field_from_settings() -> str: - value = getattr(settings, "crm_linkedin_field", None) - if isinstance(value, str): - value = value.strip() - if value: - return value - return "cLinkedIn" - - def _format_seniority_label(value: str | None) -> str: if value is None: return "Unknown" @@ -1703,8 +1692,7 @@ def _format_location_summary(cls, values: dict[str, Any]) -> str: @classmethod def _field_label(cls, field: str) -> str: - linkedin_field = _configured_linkedin_field_from_settings() - if field == linkedin_field: + if field == LINKEDIN_FIELD: return "LinkedIn" return cls._FIELD_LABELS.get(field, field) @@ -2212,7 +2200,7 @@ async def confirm_updates( interaction: discord.Interaction, button: discord.ui.Button["ResumeUpdateConfirmationView"], ) -> None: - """Apply confirmed updates through the worker.""" + """Apply confirmed updates directly through shared CRM logic.""" await interaction.response.defer(thinking=True, ephemeral=True) if not self.proposed_updates and not self.link_discord: await interaction.followup.send( @@ -2238,84 +2226,39 @@ def _audit_apply_event(result: str, metadata: dict[str, Any]) -> None: exc, ) - try: - apply_job_id = await self.crm_cog._enqueue_resume_apply_job( - contact_id=self.contact_id, - updates=self.proposed_updates, - link_discord=self.link_discord, - ) - except Exception as exc: - logger.error("Failed to enqueue resume apply job: %s", exc) - _audit_apply_event( - "error", - { - "contact_id": self.contact_id, - "stage": "apply_enqueue", - "error": str(exc), - "updated_fields": [], - "proposed_updates_count": len(self.proposed_updates), - "link_member_requested": bool(self.link_discord), - "link_discord_applied": None, - }, - ) - await interaction.followup.send( - "❌ Failed to enqueue CRM apply job. Please try again.", - ephemeral=True, - ) - return - await interaction.followup.send( "🛠️ Applying confirmed updates to CRM...", ephemeral=True, ) try: - apply_result = await self.crm_cog._wait_for_backend_job_result(apply_job_id) + result = await self.crm_cog._apply_resume_profile_direct( + contact_id=self.contact_id, + updates=self.proposed_updates, + link_discord=self.link_discord, + ) except Exception as exc: logger.error( - "Worker polling failed for apply_job_id=%s contact_id=%s error=%s", - apply_job_id, - self.contact_id, - exc, + "Resume apply failed for contact_id=%s error=%s", self.contact_id, exc ) _audit_apply_event( "error", { "contact_id": self.contact_id, - "stage": "apply_polling_failed", - "job_id": apply_job_id, + "stage": "apply_execute", "error": str(exc), "updated_fields": [], "link_discord_applied": None, }, ) await interaction.followup.send( - "⚠️ Resume apply polling failed. Please retry or check CRM manually.", + "❌ Failed to apply CRM updates. Please try again.", ephemeral=True, ) return - - if not apply_result: - _audit_apply_event( - "error", - { - "contact_id": self.contact_id, - "stage": "apply_timeout", - "job_id": apply_job_id, - "updated_fields": [], - "link_discord_applied": None, - }, - ) - await interaction.followup.send( - "⚠️ Timed out waiting for apply job. Please check again shortly.", - ephemeral=True, - ) - return - - status = str(apply_result.get("status", "unknown")) - result = apply_result.get("result") updated_fields: list[str] = [] updated_values: dict[str, Any] = {} link_discord_applied: bool | None = None + warning_message = "" if isinstance(result, dict): raw_fields = result.get("updated_fields") if isinstance(raw_fields, list): @@ -2328,75 +2271,25 @@ def _audit_apply_event(result: str, metadata: dict[str, Any]) -> None: raw_link_applied = result.get("link_discord_applied") if isinstance(raw_link_applied, bool): link_discord_applied = raw_link_applied - - if status != "succeeded": - result_error = str(apply_result.get("last_error", "")) - result_success = None - if isinstance(result, dict): - result_success = result.get("success") - - if result_success is False: - error_message = str( - result.get("error") if isinstance(result, dict) else "" - ) - if not error_message: - error_message = result_error or "Unknown error" - _audit_apply_event( - "error", - { - "contact_id": self.contact_id, - "stage": "apply_failed", - "job_id": apply_job_id, - "job_status": status, - "last_error": result_error, - "apply_error": error_message, - "updated_fields": updated_fields, - "link_discord_applied": link_discord_applied, - }, - ) - await interaction.followup.send( - f"❌ Apply job failed (status: {status}). Error: {error_message}", - ephemeral=True, - ) - return - - _audit_apply_event( - "error", - { - "contact_id": self.contact_id, - "stage": "apply_failed", - "job_id": apply_job_id, - "job_status": status, - "last_error": str(apply_result.get("last_error", "")), - "updated_fields": updated_fields, - "link_discord_applied": link_discord_applied, - }, - ) - await interaction.followup.send( - f"❌ Apply job failed (status: {status}). " - f"Error: {apply_result.get('last_error') or 'Unknown error'}", - ephemeral=True, - ) - return + raw_warning = result.get("warning") + if raw_warning is not None: + warning_message = str(raw_warning).strip() if isinstance(result, dict) and result.get("success") is False: error_message = str(result.get("error") or "") if not error_message: - error_message = str(apply_result.get("last_error", "Unknown error")) + error_message = "Unknown error" _audit_apply_event( "error", { "contact_id": self.contact_id, "stage": "apply_failed", - "job_id": apply_job_id, - "job_status": status, "updated_fields": updated_fields, "link_discord_applied": link_discord_applied, }, ) await interaction.followup.send( - "❌ Apply completed but returned a failed result. " - f"Error: {error_message}", + f"❌ Failed to apply CRM updates. Error: {error_message}", ephemeral=True, ) return @@ -2411,8 +2304,6 @@ def _audit_apply_event(result: str, metadata: dict[str, Any]) -> None: { "contact_id": self.contact_id, "stage": "apply_no_updates", - "job_id": apply_job_id, - "job_status": status, "updated_fields": updated_fields, "link_discord_applied": link_discord_applied, }, @@ -2448,6 +2339,12 @@ def _audit_apply_event(result: str, metadata: dict[str, Any]) -> None: value=applied_updates_value, inline=False, ) + if warning_message: + embed.add_field( + name="Warning", + value=self.crm_cog._sanitize_error_message_for_discord(warning_message), + inline=False, + ) profile_url = f"{self.crm_cog.base_url}/#Contact/view/{self.contact_id}" embed.add_field(name="🔗 CRM Profile", value=f"[View in CRM]({profile_url})") _audit_apply_event( @@ -2455,11 +2352,11 @@ def _audit_apply_event(result: str, metadata: dict[str, Any]) -> None: { "contact_id": self.contact_id, "stage": "apply_succeeded", - "job_id": apply_job_id, "updated_fields": updated_fields, "proposed_updates_count": len(self.proposed_updates), "link_member_requested": bool(self.link_discord), "link_discord_applied": link_discord_applied, + "warning": warning_message or None, }, ) await interaction.followup.send(embed=embed, ephemeral=True) @@ -2858,11 +2755,6 @@ def __init__(self, bot: commands.Bot) -> None: ) = None self._init_audit_logger() - @staticmethod - def _configured_linkedin_field() -> str: - """Return the configured field for LinkedIn profile values.""" - return _configured_linkedin_field_from_settings() - @staticmethod def _sanitize_error_message_for_discord( raw_error: Any, @@ -2889,129 +2781,44 @@ def _sanitize_error_message_for_discord( return text[: max_length - 1].rstrip() + "…" - def _backend_headers(self) -> dict[str, str]: - """Build auth headers for internal backend API calls.""" - if not settings.api_shared_secret: - raise ValueError("API_SHARED_SECRET is required for backend API requests.") - return { - "X-API-Secret": settings.api_shared_secret, - "Content-Type": "application/json", - } - - async def _backend_request_json( - self, - method: Literal["GET", "POST"], - path: str, - *, - expected_status: int, - payload: dict[str, Any] | None = None, - ) -> dict[str, Any]: - timeout = aiohttp.ClientTimeout(total=30) - request_kwargs: dict[str, Any] = { - "headers": self._backend_headers(), - "timeout": timeout, - } - if payload is not None: - request_kwargs["json"] = payload - - async with aiohttp.ClientSession() as session: - async with session.request( - method, - self._backend_url(path), - **request_kwargs, - ) as response: - data = await response.json() - if response.status != expected_status: - raise ValueError(f"Backend {method} {path} failed: {data}") - if not isinstance(data, dict): - raise ValueError( - f"Backend {method} {path} returned a non-object response." - ) - return data + def _create_resume_profile_processor(self) -> ResumeProfileProcessor: + return ResumeProfileProcessor(self._resume_processor_config()) - def _backend_url(self, path: str) -> str: - return f"{settings.backend_api_base_url.rstrip('/')}{path}" + @staticmethod + def _resume_processor_config() -> ResumeProcessorConfig: + return ResumeProcessorConfig.from_settings(settings) - async def _enqueue_resume_extract_job( + async def _extract_resume_profile_direct( self, *, contact_id: str, attachment_id: str, filename: str, - refresh_token: str | None = None, - ) -> str: - payload = { - "contact_id": contact_id, - "attachment_id": attachment_id, - "filename": filename, - } - if refresh_token: - payload["refresh_token"] = refresh_token - data = await self._backend_request_json( - "POST", - "/jobs/resume-extract", - payload=payload, - expected_status=202, + ) -> dict[str, Any]: + processor = self._create_resume_profile_processor() + result = await asyncio.to_thread( + processor.extract_profile_proposal, + contact_id=contact_id, + attachment_id=attachment_id, + filename=filename, ) - job_id = data.get("job_id") - if not isinstance(job_id, str) or not job_id: - raise ValueError("Missing backend extract job_id in response.") - return job_id + return result.model_dump() - async def _enqueue_resume_apply_job( + async def _apply_resume_profile_direct( self, *, contact_id: str, updates: dict[str, Any], link_discord: dict[str, str] | None = None, - ) -> str: - payload = { - "contact_id": contact_id, - "updates": updates, - "link_discord": link_discord, - } - data = await self._backend_request_json( - "POST", - "/jobs/resume-apply", - payload=payload, - expected_status=202, - ) - job_id = data.get("job_id") - if not isinstance(job_id, str) or not job_id: - raise ValueError("Missing backend apply job_id in response.") - return job_id - - async def _get_backend_job_status(self, job_id: str) -> dict[str, Any]: - return await self._backend_request_json( - "GET", - f"/jobs/{job_id}", - expected_status=200, + ) -> dict[str, Any]: + processor = self._create_resume_profile_processor() + result = await asyncio.to_thread( + processor.apply_profile_updates, + contact_id=contact_id, + updates=updates, + link_discord=link_discord, ) - - async def _wait_for_backend_job_result( - self, job_id: str, *, timeout_seconds: int = 180, poll_seconds: float = 2.0 - ) -> dict[str, Any] | None: - """Poll backend job status until terminal or timeout.""" - terminal = {"succeeded", "dead", "canceled"} - max_attempts = max(1, int(timeout_seconds / poll_seconds)) - - for _ in range(max_attempts): - job = await self._get_backend_job_status(job_id) - status = str(job.get("status", "")) - if status in terminal: - return job - await asyncio.sleep(poll_seconds) - - return None - - def _resolve_field_name( - self, contact: dict[str, Any], *, candidates: tuple[str, ...] - ) -> str | None: - """Return the first matching field name that exists on a contact.""" - for field_name in candidates: - if field_name in contact: - return field_name - return None + return result.model_dump() def _normalize_onboarding_state(self, value: Any) -> str: """Normalize onboarding state for comparisons.""" @@ -3024,32 +2831,23 @@ def _format_onboarding_updated_at(self, raw_value: Any) -> str: if raw_value is None: return "Unknown" - if isinstance(raw_value, (int, float)): - try: - return datetime.fromtimestamp(raw_value, tz=timezone.utc).strftime( - "%Y-%m-%d %H:%M UTC" - ) - except (OSError, OverflowError, ValueError): - return str(raw_value) - raw_value_text = str(raw_value).strip() if not raw_value_text: return "Unknown" try: - parsed = datetime.fromisoformat(raw_value_text.replace("Z", "+00:00")) + return datetime.strptime(raw_value_text, ESPO_DATETIME_FORMAT).strftime( + "%Y-%m-%d %H:%M UTC" + ) except ValueError: - return raw_value_text - - if parsed.tzinfo is None: - if parsed.time() and parsed.time() != datetime.min.time(): - return parsed.strftime("%Y-%m-%d %H:%M") - return parsed.strftime("%Y-%m-%d") + pass - parsed_utc = parsed.astimezone(timezone.utc) - if parsed_utc.time() and parsed_utc.time() != datetime.min.time(): - return parsed_utc.strftime("%Y-%m-%d %H:%M UTC") - return parsed_utc.strftime("%Y-%m-%d") + try: + return datetime.strptime(raw_value_text, ESPO_DATE_FORMAT).strftime( + ESPO_DATE_FORMAT + ) + except ValueError: + return raw_value_text async def _resolve_onboarder_username( self, interaction: discord.Interaction, raw_onboarder: str @@ -3146,14 +2944,7 @@ def _build_onboarding_queue_row( else discord_username.strip() ) - onboarder_field = self._resolve_field_name( - contact_record, candidates=ONBOARDER_FIELD_CANDIDATES - ) - onboarder_value = ( - str(contact_record.get(onboarder_field, "")).strip() - if onboarder_field - else "" - ) + onboarder_value = str(contact_record.get(ONBOARDER_FIELD, "")).strip() return { "name": name, @@ -3755,39 +3546,11 @@ async def _run_resume_extract_and_preview( action: str = "crm.upload_resume", status_message: str | None = None, ) -> None: - """Kick off worker extraction and show confirmation preview.""" + """Run extraction directly and show confirmation preview.""" action_name = action status_text = ( status_message or "📥 Resume uploaded. Extracting profile fields now..." ) - refresh_token = uuid4().hex if action_name == "crm.reprocess_resume" else None - try: - job_id = await self._enqueue_resume_extract_job( - contact_id=contact_id, - attachment_id=attachment_id, - filename=filename, - refresh_token=refresh_token, - ) - except Exception as exc: - logger.error("Failed to enqueue resume extract job: %s", exc) - self._audit_command( - interaction=interaction, - action=action_name, - result="error", - metadata={ - "filename": filename, - "attachment_id": attachment_id, - "stage": "extract_enqueue", - "error": str(exc), - }, - resource_type="crm_contact", - resource_id=str(contact_id), - ) - await interaction.followup.send( - "⚠️ Resume uploaded, but extraction job could not be enqueued.", - ephemeral=True, - ) - return await interaction.followup.send( status_text, @@ -3795,50 +3558,18 @@ async def _run_resume_extract_and_preview( ) try: - job = await self._wait_for_backend_job_result(job_id) - except Exception as exc: - logger.error("Worker polling failed for job_id=%s error=%s", job_id, exc) - self._audit_command( - interaction=interaction, - action=action_name, - result="error", - metadata={ - "filename": filename, - "attachment_id": attachment_id, - "job_id": job_id, - "stage": "extract_polling", - "error": str(exc), - }, - resource_type="crm_contact", - resource_id=str(contact_id), - ) - await interaction.followup.send( - "⚠️ Resume uploaded, but extraction polling failed.", - ephemeral=True, - ) - return - if not job: - self._audit_command( - interaction=interaction, - action=action_name, - result="error", - metadata={ - "filename": filename, - "attachment_id": attachment_id, - "job_id": job_id, - "stage": "extract_timeout", - }, - resource_type="crm_contact", - resource_id=str(contact_id), + result = await self._extract_resume_profile_direct( + contact_id=contact_id, + attachment_id=attachment_id, + filename=filename, ) - await interaction.followup.send( - "⚠️ Timed out waiting for extraction result. Try again in a moment.", - ephemeral=True, + except Exception as exc: + logger.error( + "Resume extraction failed contact_id=%s attachment_id=%s error=%s", + contact_id, + attachment_id, + exc, ) - return - - status = str(job.get("status", "unknown")) - if status != "succeeded": self._audit_command( interaction=interaction, action=action_name, @@ -3846,22 +3577,17 @@ async def _run_resume_extract_and_preview( metadata={ "filename": filename, "attachment_id": attachment_id, - "job_id": job_id, - "stage": "extract_failed", - "job_status": status, - "last_error": str(job.get("last_error", "")), + "stage": "extract_execute", + "error": str(exc), }, resource_type="crm_contact", resource_id=str(contact_id), ) await interaction.followup.send( - f"❌ Extraction job failed (status: {status}). " - f"Error: {job.get('last_error') or 'Unknown error'}", + "⚠️ Resume uploaded, but extraction failed.", ephemeral=True, ) return - - result = job.get("result") if not isinstance(result, dict): self._audit_command( interaction=interaction, @@ -3870,7 +3596,6 @@ async def _run_resume_extract_and_preview( metadata={ "filename": filename, "attachment_id": attachment_id, - "job_id": job_id, "stage": "extract_malformed_result", }, resource_type="crm_contact", @@ -3898,7 +3623,6 @@ async def _run_resume_extract_and_preview( metadata={ "filename": filename, "attachment_id": attachment_id, - "job_id": job_id, "stage": "extract_unsuccessful", "error": str(result.get("error", "")), }, @@ -3985,7 +3709,6 @@ async def _run_resume_extract_and_preview( metadata={ "filename": filename, "attachment_id": attachment_id, - "job_id": job_id, "stage": "preview_no_changes", }, resource_type="crm_contact", @@ -4017,7 +3740,6 @@ async def _run_resume_extract_and_preview( metadata={ "filename": filename, "attachment_id": attachment_id, - "job_id": job_id, "stage": "preview_ready", "proposed_updates_count": len(proposed_updates), "role_suggestions_count": len(suggested_discord_roles), @@ -4063,7 +3785,6 @@ async def _run_resume_extract_and_preview( metadata={ "filename": filename, "attachment_id": attachment_id, - "job_id": job_id, "stage": "preview_ready", "proposed_updates_count": len(proposed_updates), "link_member_requested": bool(link_member), @@ -4572,10 +4293,7 @@ async def assign_onboarder( return full_contact = self.espo_api.request("GET", f"Contact/{contact_id}") - onboarder_field = self._resolve_field_name( - full_contact, candidates=ONBOARDER_FIELD_CANDIDATES - ) - if not onboarder_field: + if ONBOARDER_FIELD not in full_contact: self._audit_command( interaction=interaction, action="crm.assign_onboarder", @@ -4589,21 +4307,18 @@ async def assign_onboarder( resource_id=str(contact_id), ) await interaction.followup.send( - "❌ Could not locate a known onboarder field for this CRM contact." + "❌ Could not locate the `cOnboarder` field for this CRM contact." ) return - state_field = self._resolve_field_name( - full_contact, candidates=ONBOARDING_STATUS_FIELD_CANDIDATES - ) current_state = self._normalize_onboarding_state( - full_contact.get(state_field) if state_field else None + full_contact.get(ONBOARDING_STATUS_FIELD) ) - update_payload: dict[str, str] = {onboarder_field: onboarder_username} + update_payload: dict[str, str] = {ONBOARDER_FIELD: onboarder_username} state_updated = False - if state_field and current_state == "pending": - update_payload[state_field] = "selected" + if current_state == "pending": + update_payload[ONBOARDING_STATUS_FIELD] = "selected" state_updated = True self.espo_api.request("PUT", f"Contact/{contact_id}", update_payload) @@ -4626,7 +4341,6 @@ async def assign_onboarder( "contact_id": str(contact_id), "contact_name": contact_name, "onboarder": onboarder_username, - "state_field": state_field, "state_updated": state_updated, "previous_state": current_state or None, }, @@ -4672,8 +4386,7 @@ async def view_onboarding_queue(self, interaction: discord.Interaction) -> None: "maxSize": ONBOARDING_QUEUE_MAX_SIZE, "select": ( "id,name,emailAddress,cDiscordUsername,cDiscordUserID," - "cOnboardingState,cOnboardingStatus,cOnboarding," - "cOnboarder,cOnboardingCoordinator,cOnboardingUpdatedAt" + "cOnboardingState,cOnboarder,cOnboardingUpdatedAt" ), }, ) @@ -4681,13 +4394,8 @@ async def view_onboarding_queue(self, interaction: discord.Interaction) -> None: queue_entries: list[tuple[dict[str, Any], str]] = [] for contact_record in contacts: - state_field = self._resolve_field_name( - contact_record, candidates=ONBOARDING_STATUS_FIELD_CANDIDATES - ) - status = ( - self._normalize_onboarding_state(contact_record.get(state_field)) - if state_field - else "" + status = self._normalize_onboarding_state( + contact_record.get(ONBOARDING_STATUS_FIELD) ) if status in EXCLUDED_ONBOARDING_STATES: continue @@ -5027,6 +4735,52 @@ async def _search_contact_for_linking( def _resume_file_extension(filename: str | None) -> str: return document_file_extension(filename) + async def _validate_resume_attachment( + self, + *, + interaction: discord.Interaction, + action: str, + attachment: discord.Attachment, + failure_result: Literal["denied", "error"], + ) -> bool: + resume_config = self._resume_processor_config() + file_extension = self._resume_file_extension(attachment.filename) + if file_extension not in resume_config.allowed_attachment_suffixes: + self._audit_command_safe( + interaction=interaction, + action=action, + result=failure_result, + metadata={ + "filename": attachment.filename, + "reason": "invalid_file_type", + }, + ) + await interaction.followup.send( + "❌ Invalid file type. " + f"Upload a {resume_config.allowed_file_extensions_label} file.\n" + f"You uploaded: `{attachment.filename}`" + ) + return False + + if attachment.size > resume_config.max_file_size_bytes: + self._audit_command_safe( + interaction=interaction, + action=action, + result=failure_result, + metadata={ + "filename": attachment.filename, + "size_bytes": attachment.size, + "reason": "file_too_large", + }, + ) + await interaction.followup.send( + f"❌ File too large. Maximum size is {resume_config.max_file_size_mb}MB.\n" + f"Your file: {attachment.size / (1024 * 1024):.1f}MB" + ) + return False + + return True + @staticmethod def _is_valid_resume_name_candidate(value: str) -> bool: normalized = value.strip() @@ -5216,6 +4970,7 @@ def _build_inference_lookup_summary( *, file_content: bytes, attempts: list[dict[str, Any]] | None, + hints: dict[str, Any] | None = None, filename: str | None = None, ) -> str: """Build a user-facing description of resume-derived lookup values.""" @@ -5223,11 +4978,8 @@ def _build_inference_lookup_summary( if attempts_text: return f"\nTried contact lookups: {attempts_text}" - hints_raw = self._extract_resume_contact_hints(file_content, filename=filename) - if isinstance(hints_raw, dict): - hints: dict[str, Any] = hints_raw - else: - hints = {} + if hints is None: + hints = self._extract_resume_contact_hints(file_content, filename=filename) def _to_values(raw_values: Any) -> list[str]: values: list[str] = [] @@ -5270,6 +5022,7 @@ async def _build_inference_lookup_summary_async( *, file_content: bytes, attempts: list[dict[str, Any]] | None, + hints: dict[str, Any] | None = None, filename: str | None = None, ) -> str: """Build lookup summary without blocking the event loop.""" @@ -5277,14 +5030,20 @@ async def _build_inference_lookup_summary_async( self._build_inference_lookup_summary, file_content=file_content, attempts=attempts, + hints=hints, filename=filename, ) def _build_resume_parsed_identity_summary( - self, file_content: bytes, *, filename: str | None = None + self, + file_content: bytes, + *, + hints: dict[str, Any] | None = None, + filename: str | None = None, ) -> str: """Build a short display summary of parsed contact identity fields.""" - hints = self._extract_resume_contact_hints(file_content, filename=filename) + if hints is None: + hints = self._extract_resume_contact_hints(file_content, filename=filename) parsed_name = str(hints.get("name") or "").strip() if not self._is_valid_resume_name_candidate(parsed_name): parsed_name = self._extract_resume_name_fallback( @@ -5305,12 +5064,17 @@ def _build_resume_parsed_identity_summary( ) async def _build_resume_parsed_identity_summary_async( - self, file_content: bytes, *, filename: str | None = None + self, + file_content: bytes, + *, + hints: dict[str, Any] | None = None, + filename: str | None = None, ) -> str: """Build parsed identity summary without blocking the event loop.""" return await asyncio.to_thread( self._build_resume_parsed_identity_summary, file_content, + hints=hints, filename=filename, ) @@ -5349,64 +5113,67 @@ def _build_resume_create_contact_payload( emails = hints.get("emails", []) github_usernames = hints.get("github_usernames", []) linkedin_urls = hints.get("linkedin_urls", []) - skills = hints.get("skills", []) - description = str(hints.get("description", "")).strip() if not isinstance(emails, list): emails = [] if not isinstance(github_usernames, list): github_usernames = [] if not isinstance(linkedin_urls, list): linkedin_urls = [] - if not isinstance(skills, list): - skills = [] payload: dict[str, Any] = { "type": "Prospect", "name": contact_name, } self._populate_name_fields(payload, source_name=contact_name) - if emails: - primary_email = emails[0] - if primary_email.endswith("@508.dev"): - payload["c508Email"] = primary_email + + normalized_emails = [ + str(email).strip() + for email in emails + if isinstance(email, str) and str(email).strip() + ] + preferred_email = next( + ( + email + for email in normalized_emails + if not email.casefold().endswith("@508.dev") + ), + None, + ) + fallback_508_email = next( + ( + email + for email in normalized_emails + if email.casefold().endswith("@508.dev") + ), + None, + ) + selected_email = preferred_email or fallback_508_email + if selected_email: + if selected_email.casefold().endswith("@508.dev"): + payload["c508Email"] = selected_email else: - payload["emailAddress"] = primary_email - if github_usernames: - payload["cGitHubUsername"] = github_usernames[0] - if linkedin_urls: - payload[self._configured_linkedin_field()] = linkedin_urls[0] + payload["emailAddress"] = selected_email + return payload + + for github_username in github_usernames: + normalized = ( + str(github_username).strip() if isinstance(github_username, str) else "" + ) + if normalized: + payload["cGitHubUsername"] = normalized + return payload + + for linkedin_url in linkedin_urls: + normalized = ( + str(linkedin_url).strip() if isinstance(linkedin_url, str) else "" + ) + if normalized: + payload[LINKEDIN_FIELD] = normalized + return payload + phone = hints.get("phone") if isinstance(phone, str) and phone.strip(): payload["phoneNumber"] = phone.strip() - primary_roles = hints.get("primary_roles") - if isinstance(primary_roles, list): - normalized_roles = [ - str(role).strip() - for role in primary_roles - if isinstance(role, str) and role.strip() - ] - if normalized_roles: - payload["cRoles"] = normalized_roles - address_country = str(hints.get("address_country", "")).strip() - if address_country: - payload["addressCountry"] = address_country - timezone = self._normalize_timezone(hints.get("timezone")) - if timezone: - payload["cTimezone"] = timezone - address_city = str(hints.get("address_city", "")).strip() - if address_city: - payload["addressCity"] = address_city - seniority = str(hints.get("seniority_level", "")).strip() - if seniority: - payload["cSeniority"] = seniority - if description: - payload["description"] = description - if skills: - normalized_skills = [ - str(item).strip() for item in skills if str(item).strip() - ] - if normalized_skills: - payload["skills"] = normalized_skills return payload @@ -5543,6 +5310,7 @@ async def _infer_contact_from_resume( "method": "email", "value": email, "attempts": attempts, + "hints": hints, } if len(contacts) > 1: return None, { @@ -5550,6 +5318,7 @@ async def _infer_contact_from_resume( "value": email, "reason": "multiple_matches", "attempts": attempts, + "hints": hints, } github_usernames = hints.get("github_usernames", []) @@ -5565,6 +5334,7 @@ async def _infer_contact_from_resume( "method": "github", "value": github_username, "attempts": attempts, + "hints": hints, } if len(contacts) > 1: return None, { @@ -5572,6 +5342,7 @@ async def _infer_contact_from_resume( "value": github_username, "reason": "multiple_matches", "attempts": attempts, + "hints": hints, } linkedin_urls = hints.get("linkedin_urls", []) @@ -5580,13 +5351,14 @@ async def _infer_contact_from_resume( for linkedin_url in linkedin_urls: attempts.append({"method": "linkedin", "value": linkedin_url}) contacts = await self._search_contacts_by_field( - field=self._configured_linkedin_field(), value=linkedin_url + field=LINKEDIN_FIELD, value=linkedin_url ) if len(contacts) == 1: return contacts[0], { "method": "linkedin", "value": linkedin_url, "attempts": attempts, + "hints": hints, } if len(contacts) > 1: return None, { @@ -5594,9 +5366,14 @@ async def _infer_contact_from_resume( "value": linkedin_url, "reason": "multiple_matches", "attempts": attempts, + "hints": hints, } - return None, {"reason": "no_matching_contact", "attempts": attempts} + return None, { + "reason": "no_matching_contact", + "attempts": attempts, + "hints": hints, + } async def _perform_discord_linking( self, @@ -7300,58 +7077,12 @@ async def update_contact( return if resume is not None: - if not settings.api_shared_secret: - self._audit_command( - interaction=interaction, - action="crm.update_contact", - result="error", - metadata={ - "filename": resume.filename, - "reason": "api_shared_secret_missing", - }, - ) - await interaction.followup.send( - "❌ API_SHARED_SECRET is not configured for backend API access." - ) - return - - valid_extensions = {".pdf", ".docx", ".txt"} - file_extension = ( - "." + resume.filename.split(".")[-1].lower() - if "." in resume.filename - else "" - ) - if file_extension not in valid_extensions: - self._audit_command( - interaction=interaction, - action="crm.update_contact", - result="denied", - metadata={ - "filename": resume.filename, - "reason": "invalid_file_type", - }, - ) - await interaction.followup.send( - f"❌ Invalid file type. Upload a PDF, DOC, DOCX, or TXT file.\n" - f"You uploaded: `{resume.filename}`" - ) - return - - max_size = 10 * 1024 * 1024 - if resume.size > max_size: - self._audit_command( - interaction=interaction, - action="crm.update_contact", - result="denied", - metadata={ - "filename": resume.filename, - "size_bytes": resume.size, - "reason": "file_too_large", - }, - ) - await interaction.followup.send( - f"❌ File too large. Maximum size is 10MB.\nYour file: {resume.size / (1024 * 1024):.1f}MB" - ) + if not await self._validate_resume_attachment( + interaction=interaction, + action="crm.update_contact", + attachment=resume, + failure_result="denied", + ): return is_steering = hasattr( @@ -7462,7 +7193,7 @@ async def update_contact( if linkedin is not None: clean_linkedin = linkedin.strip() if clean_linkedin: - update_data[self._configured_linkedin_field()] = clean_linkedin + update_data[LINKEDIN_FIELD] = clean_linkedin requested_updates.append("linkedin") if rate_range is not None: @@ -7627,10 +7358,9 @@ async def update_contact( inline=True, ) if "linkedin" in requested_updates: - linkedin_field = self._configured_linkedin_field() embed.add_field( name="🔗 LinkedIn", - value=update_data[linkedin_field], + value=update_data[LINKEDIN_FIELD], inline=True, ) if "skills" in requested_updates: @@ -8272,7 +8002,7 @@ async def _upload_resume_attachment_to_contact( description="Upload resume, extract profile fields, and preview CRM updates", ) @app_commands.describe( - file="Resume file to upload (PDF, DOC, DOCX, TXT)", + file="Resume file to upload (PDF or DOCX)", search_term="Email, name, or contact ID (Steering Committee+ only). Omit to infer from resume.", overwrite="Replace existing resumes instead of appending", link_user="Discord user to link to this CRM contact (optional, Steering Committee+ for others)", @@ -8289,60 +8019,12 @@ async def upload_resume( try: await interaction.response.defer(ephemeral=True) - if not settings.api_shared_secret: - self._audit_command( - interaction=interaction, - action="crm.upload_resume", - result="error", - metadata={ - "filename": file.filename, - "reason": "api_shared_secret_missing", - }, - ) - await interaction.followup.send( - "❌ API_SHARED_SECRET is not configured for backend API access." - ) - return - - # Validate file type - valid_extensions = {".pdf", ".docx", ".txt"} - file_extension = ( - "." + file.filename.split(".")[-1].lower() - if "." in file.filename - else "" - ) - - if file_extension not in valid_extensions: - self._audit_command( - interaction=interaction, - action="crm.upload_resume", - result="error", - metadata={ - "filename": file.filename, - "reason": "invalid_file_type", - }, - ) - await interaction.followup.send( - f"❌ Invalid file type. Please upload a PDF, DOC, DOCX, or TXT file.\nYou uploaded: `{file.filename}`" - ) - return - - # Validate file size (10MB limit) - max_size = 10 * 1024 * 1024 # 10MB in bytes - if file.size > max_size: - self._audit_command( - interaction=interaction, - action="crm.upload_resume", - result="error", - metadata={ - "filename": file.filename, - "size_bytes": file.size, - "reason": "file_too_large", - }, - ) - await interaction.followup.send( - f"❌ File too large. Maximum size is 10MB.\nYour file: {file.size / (1024 * 1024):.1f}MB" - ) + if not await self._validate_resume_attachment( + interaction=interaction, + action="crm.upload_resume", + attachment=file, + failure_result="error", + ): return is_steering = hasattr( @@ -8500,6 +8182,9 @@ async def upload_resume( "reason", "resume_contact_not_found" ) inferred_attempts = (inferred_contact_meta or {}).get("attempts") + inferred_hints = (inferred_contact_meta or {}).get("hints") + if not isinstance(inferred_hints, dict): + inferred_hints = None inference_metadata = { "filename": file.filename, "target_scope": "resume_inferred", @@ -8518,6 +8203,7 @@ async def upload_resume( attempts=inferred_attempts if isinstance(inferred_attempts, list) else None, + hints=inferred_hints, filename=file.filename, ) ) @@ -8560,6 +8246,7 @@ async def upload_resume( + inferred_attempts_text + await self._build_resume_parsed_identity_summary_async( file_content, + hints=inferred_hints, filename=file.filename, ), view=view, @@ -8642,21 +8329,6 @@ async def reprocess_resume( try: await interaction.response.defer(ephemeral=True) - if not settings.api_shared_secret: - self._audit_command( - interaction=interaction, - action="crm.reprocess_resume", - result="error", - metadata={ - "search_term": search_term, - "reason": "api_shared_secret_missing", - }, - ) - await interaction.followup.send( - "❌ API_SHARED_SECRET is not configured for backend API access." - ) - return - is_steering = hasattr( interaction.user, "roles" ) and check_user_roles_with_hierarchy( @@ -8751,20 +8423,6 @@ async def bulk_reprocess_resumes( try: await interaction.response.defer(ephemeral=True) - if not settings.api_shared_secret: - self._audit_command( - interaction=interaction, - action="crm.bulk_reprocess_resumes", - result="error", - metadata={ - "reason": "api_shared_secret_missing", - }, - ) - await interaction.followup.send( - "API_SHARED_SECRET is not configured for backend API access." - ) - return - is_steering = hasattr( interaction.user, "roles" ) and check_user_roles_with_hierarchy( diff --git a/apps/worker/src/five08/worker/config.py b/apps/worker/src/five08/worker/config.py index 7f5bcd75..52709d08 100644 --- a/apps/worker/src/five08/worker/config.py +++ b/apps/worker/src/five08/worker/config.py @@ -10,7 +10,6 @@ class WorkerSettings(SharedSettings): """Worker-specific settings layered on top of shared stack settings.""" - _crm_linkedin_field: str = PrivateAttr(default="cLinkedIn") _crm_intake_completed_field: str = PrivateAttr(default="") worker_name: str = "worker" @@ -30,7 +29,7 @@ class WorkerSettings(SharedSettings): docuseal_member_agreement_template_id: int | None = None max_file_size_mb: int = 10 - allowed_file_types: str = "pdf,docx,txt" + allowed_file_types: str = "pdf,docx" max_attachments_per_contact: int = 3 crm_sync_enabled: bool = True crm_sync_interval_seconds: int = 900 @@ -131,16 +130,6 @@ def allowed_file_extensions(self) -> set[str]: """Allowed resume file extensions.""" return {ext.strip().lower() for ext in self.allowed_file_types.split(",")} - @property - def crm_linkedin_field(self) -> str: - """Resume/profile sync always writes LinkedIn URLs to the canonical CRM field.""" - return self._crm_linkedin_field - - @crm_linkedin_field.setter - def crm_linkedin_field(self, value: str) -> None: - """Allow controlled runtime overrides without reintroducing env loading.""" - self._crm_linkedin_field = value - @property def crm_intake_completed_field(self) -> str: """Intake completion field remains intentionally unset until explicitly adopted.""" diff --git a/apps/worker/src/five08/worker/crm/intake_form_processor.py b/apps/worker/src/five08/worker/crm/intake_form_processor.py index 6afedabc..a05fccd7 100644 --- a/apps/worker/src/five08/worker/crm/intake_form_processor.py +++ b/apps/worker/src/five08/worker/crm/intake_form_processor.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address +LINKEDIN_FIELD = "cLinkedIn" DESCRIPTION_SECTIONS = { "primary_skills_interests": "Primary skills and interests", @@ -42,7 +43,7 @@ FIELD_MAP = { "phone": "phoneNumber", "discord_username": "cDiscordUsername", - "linkedin_url": settings.crm_linkedin_field, + "linkedin_url": LINKEDIN_FIELD, "github_username": "cGitHubUsername", "address_country": "addressCountry", "address_city": "addressCity", @@ -448,7 +449,7 @@ def _build_resume_updates(self, payload: Mapping[str, Any]) -> dict[str, Any]: if profile_github: updates["cGitHubUsername"] = profile_github if profile_linkedin: - updates[settings.crm_linkedin_field] = profile_linkedin + updates[LINKEDIN_FIELD] = profile_linkedin if profile_timezone: updates.setdefault("cTimezone", profile_timezone) if profile_city: diff --git a/apps/worker/src/five08/worker/crm/people_sync.py b/apps/worker/src/five08/worker/crm/people_sync.py index 87b2da79..6de2d1dd 100644 --- a/apps/worker/src/five08/worker/crm/people_sync.py +++ b/apps/worker/src/five08/worker/crm/people_sync.py @@ -14,6 +14,7 @@ logger = logging.getLogger(__name__) _DISCORD_ID_RE = re.compile(r"\(ID:\s*(\d+)\)") +LINKEDIN_FIELD = "cLinkedIn" class EspoPeopleSyncClient: @@ -31,7 +32,7 @@ def list_contact_page( "cDiscordUsername,cDiscordUserId,cDiscordRoles,cDiscordUserID," "cGithubUsername,githubUsername,type,contactType," "addressCountry,addressCity,cTimezone,cSeniority,cMemberAgreementSignedAt," - f"{settings.crm_linkedin_field},skills,cSkillAttrs,resumeIds,resumeNames" + f"{LINKEDIN_FIELD},skills,cSkillAttrs,resumeIds,resumeNames" ) raw = self.api.request( "GET", @@ -323,8 +324,7 @@ def _github_username(self, raw_contact: dict[str, Any]) -> str | None: return None def _coerce_linkedin(self, raw_contact: dict[str, Any]) -> str | None: - configured_field = settings.crm_linkedin_field - for key in (configured_field, "cLinkedIn", "linkedin"): + for key in (LINKEDIN_FIELD, "linkedin"): value = _text_or_none(raw_contact.get(key)) if value: return value diff --git a/apps/worker/src/five08/worker/crm/resume_profile_processor.py b/apps/worker/src/five08/worker/crm/resume_profile_processor.py index 1f7e0087..bed7129f 100644 --- a/apps/worker/src/five08/worker/crm/resume_profile_processor.py +++ b/apps/worker/src/five08/worker/crm/resume_profile_processor.py @@ -1,1304 +1,14 @@ -"""Resume extraction + CRM profile update workflow.""" +"""Worker compatibility shim for shared resume processing.""" -from __future__ import annotations - -import ast -import json -import logging -from collections.abc import Callable -from datetime import datetime, timezone -from typing import Any - -from five08.clients.espo import EspoAPIError, EspoClient -from five08.crm_normalization import ( - ROLE_NORMALIZATION_MAP, - normalized_website_identity_key, - normalize_city, - normalize_country, - normalize_role, - normalize_roles, - normalize_state, - normalize_seniority, - normalize_timezone, - normalize_website_url, +from five08.resume_profile_processor import ResumeProcessorConfig +from five08.resume_profile_processor import ( + ResumeProfileProcessor as SharedResumeProfileProcessor, ) -from five08.skills import ( - DISALLOWED_RESUME_SKILLS, - normalize_skill, - normalize_skill_list, - normalize_skill_payload, -) -from five08.resume_extractor import ResumeProfileExtractor -from five08.queue import get_postgres_connection from five08.worker.config import settings -from five08.worker.crm.document_processor import DocumentProcessor -from five08.worker.crm.skills_extractor import SkillsExtractor -from five08.worker.models import ( - ResumeApplyResult, - ResumeExtractedProfile, - ExtractedSkills, - ResumeExtractionResult, - ResumeFieldChange, - ResumeSkipReason, - SkillAttributes, -) -logger = logging.getLogger(__name__) -DEFAULT_SKILL_STRENGTH = 3 - -class ResumeProfileProcessor: - """End-to-end extraction and apply operations for uploaded resumes.""" +class ResumeProfileProcessor(SharedResumeProfileProcessor): + """Worker-specific wrapper bound to worker settings.""" def __init__(self) -> None: - self.crm = EspoClient(settings.espo_base_url, settings.espo_api_key) - self.extractor = ResumeProfileExtractor( - api_key=settings.openai_api_key, - base_url=settings.openai_base_url, - model=settings.resolved_resume_ai_model, - max_tokens=settings.resume_extractor_max_tokens, - ) - self.skills_extractor = SkillsExtractor() - self.document_processor = DocumentProcessor() - - def extract_profile_proposal( - self, - *, - contact_id: str, - attachment_id: str, - filename: str, - ) -> ResumeExtractionResult: - """Build preview proposal from an uploaded resume attachment.""" - content_hash: str | None = None - model_name = self._configured_model_name() - try: - contact = self.crm.get_contact(contact_id) - content = self.crm.download_attachment(attachment_id) - content_hash = self.document_processor.get_content_hash(content, filename) - text = self.document_processor.extract_text(content, filename) - extracted = self.extractor.extract(text) - model_name = extracted.source - extracted_skills_result = self._coerce_profile_skill_result(extracted, text) - extracted_skills = extracted_skills_result.skills - normalized_extracted_skills = self._dedupe_normalized_skills( - extracted_skills - ) - existing_skills = self._parse_existing_skills(contact.get("skills")) - existing_skill_attrs = self._parse_skill_attrs(contact.get("cSkillAttrs")) - existing_websites = self._coerce_website_links(contact.get("cWebsiteLink")) - existing_social_links = self._coerce_website_links( - contact.get("cSocialLinks") - ) - existing_lower = {item.casefold() for item in existing_skills} - new_skills = [ - skill - for skill in normalized_extracted_skills - if skill.casefold() not in existing_lower - ] - merged_skills = self._dedupe_normalized_skills(existing_skills + new_skills) - merged_websites = self._merge_website_links( - existing=existing_websites, - extracted=extracted.website_links, - ) - merged_social_links = self._merge_website_links( - existing=existing_social_links, - extracted=extracted.social_links, - ) - merged_skill_attrs = self._merge_skill_attrs( - existing_attrs=existing_skill_attrs, - extracted_attrs=extracted_skills_result.skill_attrs, - merged_skills=merged_skills, - ) - - proposed_updates: dict[str, Any] = {} - proposed_changes: list[ResumeFieldChange] = [] - skipped: list[ResumeSkipReason] = [] - - self._collect_change( - crm_field="emailAddress", - label="Email", - current=contact.get("emailAddress"), - proposed=extracted.email, - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - blocked_reason="Skipped because @508.dev emails are managed separately", - is_blocked=lambda value: value.lower().endswith("@508.dev"), - ) - if extracted.additional_emails: - proposed_updates["additional_emails"] = extracted.additional_emails - proposed_changes.append( - ResumeFieldChange( - field="additional_emails", - label="Additional Emails", - current=None, - proposed=", ".join(extracted.additional_emails), - reason="Extracted additional emails from uploaded resume", - ) - ) - self._collect_change( - crm_field="cGitHubUsername", - label="GitHub", - current=contact.get("cGitHubUsername"), - proposed=extracted.github_username, - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - ) - self._collect_change( - crm_field=settings.crm_linkedin_field, - label="LinkedIn", - current=contact.get(settings.crm_linkedin_field), - proposed=extracted.linkedin_url, - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - ) - self._collect_change( - crm_field="phoneNumber", - label="Phone", - current=contact.get("phoneNumber"), - proposed=extracted.phone, - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - ) - self._collect_change( - crm_field="addressCountry", - label="Country", - current=contact.get("addressCountry"), - proposed=self._normalize_country(extracted.address_country), - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - ) - self._collect_change( - crm_field="cTimezone", - label="Timezone", - current=contact.get("cTimezone"), - proposed=self._normalize_timezone(extracted.timezone), - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - ) - self._collect_change( - crm_field="addressCity", - label="City", - current=contact.get("addressCity"), - proposed=self._normalize_city(extracted.address_city), - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - ) - self._collect_change( - crm_field="addressState", - label="State", - current=contact.get("addressState"), - proposed=self._normalize_state(extracted.address_state), - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - ) - self._collect_change( - crm_field="description", - label="Description", - current=contact.get("description"), - proposed=extracted.description.strip() - if extracted.description - else None, - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - ) - extracted_roles = self._normalize_roles(extracted.primary_roles) - existing_roles = self._normalize_roles(contact.get("cRoles")) - if extracted_roles and sorted(extracted_roles) != sorted(existing_roles): - proposed_updates["cRoles"] = extracted_roles - proposed_changes.append( - ResumeFieldChange( - field="cRoles", - label="Roles", - current=", ".join(existing_roles), - proposed=", ".join(extracted_roles), - reason="Extracted from uploaded resume", - ) - ) - current_seniority = self._normalize_seniority(contact.get("cSeniority")) - proposed_seniority = self._normalize_seniority(extracted.seniority_level) - self._collect_change( - crm_field="cSeniority", - label="Seniority", - current=current_seniority, - proposed=proposed_seniority, - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - is_blocked=lambda value: bool( - current_seniority - and current_seniority != "unknown" - and value != current_seniority - ), - blocked_reason="Existing seniority preserved", - ) - if new_skills: - proposed_updates["skills"] = merged_skills - if merged_skill_attrs: - skill_attrs_payload = self._serialize_skill_attrs( - merged_skill_attrs - ) - if skill_attrs_payload: - proposed_updates["cSkillAttrs"] = skill_attrs_payload - proposed_changes.append( - ResumeFieldChange( - field="skills", - label="Skills", - current=( - self._format_skills_with_strength( - existing_skills, existing_skill_attrs - ) - if existing_skills - else None - ), - proposed=self._format_skills_with_strength( - merged_skills, merged_skill_attrs - ), - reason="Added skills from resume extraction", - ) - ) - - if merged_websites != existing_websites: - proposed_updates["cWebsiteLink"] = merged_websites - proposed_changes.append( - ResumeFieldChange( - field="cWebsiteLink", - label="Website", - current=", ".join(existing_websites), - proposed=", ".join(merged_websites), - reason="Extracted from uploaded resume", - ) - ) - - if merged_social_links != existing_social_links: - proposed_updates["cSocialLinks"] = merged_social_links - proposed_changes.append( - ResumeFieldChange( - field="cSocialLinks", - label="Social Links", - current=", ".join(existing_social_links), - proposed=", ".join(merged_social_links), - reason="Extracted from uploaded resume", - ) - ) - - self._record_processing_run( - contact_id=contact_id, - attachment_id=attachment_id, - content_hash=content_hash, - model_name=model_name, - status="succeeded", - ) - - return ResumeExtractionResult( - contact_id=contact_id, - attachment_id=attachment_id, - proposed_updates=proposed_updates, - proposed_changes=proposed_changes, - skipped=skipped, - extracted_profile=extracted, - extracted_skills=extracted_skills, - new_skills=new_skills, - success=True, - ) - except Exception as exc: - logger.error( - "Resume extraction proposal failed contact_id=%s attachment_id=%s error=%s", - contact_id, - attachment_id, - exc, - ) - self._record_processing_run( - contact_id=contact_id, - attachment_id=attachment_id, - content_hash=content_hash, - model_name=model_name, - status="failed", - last_error=str(exc), - ) - return ResumeExtractionResult( - contact_id=contact_id, - attachment_id=attachment_id, - proposed_updates={}, - proposed_changes=[], - skipped=[], - extracted_profile=ResumeExtractedProfile( - email=None, - github_username=None, - linkedin_url=None, - phone=None, - confidence=0.0, - source="error", - ), - extracted_skills=[], - new_skills=[], - success=False, - error=str(exc), - ) - - def apply_profile_updates( - self, - *, - contact_id: str, - updates: dict[str, Any], - link_discord: dict[str, str] | None = None, - ) -> ResumeApplyResult: - """Apply confirmed updates to contact in CRM.""" - try: - candidate_email = None - normalized_updates = dict(updates) - - pre_update_contact: dict[str, Any] | None = None - try: - pre_update_contact = self.crm.get_contact(contact_id) - except Exception as exc: - logger.debug( - "Failed to read pre-update contact for verification contact_id=%s: %s", - contact_id, - exc, - ) - - if "emailAddress" in normalized_updates: - candidate_email = self._normalize_email_address( - normalized_updates.get("emailAddress") - ) - if "emailAddress" in normalized_updates: - normalized_updates.pop("emailAddress", None) - - additional_emails: list[str] = [] - if "additional_emails" in normalized_updates: - additional_emails = self._coerce_additional_emails( - normalized_updates.get("additional_emails") - ) - normalized_updates.pop("additional_emails", None) - - if "skills" in normalized_updates: - normalized_skills = self._coerce_skills_updates( - normalized_updates["skills"] - ) - if normalized_skills: - normalized_updates["skills"] = normalized_skills - else: - normalized_updates.pop("skills", None) - - if "cSkillAttrs" in normalized_updates: - serialized_attrs = self._coerce_skill_attrs_updates( - normalized_updates["cSkillAttrs"] - ) - if serialized_attrs: - normalized_updates["cSkillAttrs"] = serialized_attrs - else: - normalized_updates.pop("cSkillAttrs", None) - - if "cWebsiteLink" in normalized_updates: - normalized_websites = self._coerce_website_links( - normalized_updates["cWebsiteLink"] - ) - if normalized_websites: - normalized_updates["cWebsiteLink"] = normalized_websites - else: - normalized_updates.pop("cWebsiteLink", None) - - if candidate_email is not None: - if candidate_email.endswith("@508.dev"): - candidate_email = None - if candidate_email is not None: - existing_email_data = ( - pre_update_contact.get("emailAddressData") - if pre_update_contact - else None - ) - email_address_data = self._build_email_address_data( - email_candidate=candidate_email, - additional_emails=additional_emails, - existing_email_data=existing_email_data, - ) - if email_address_data: - normalized_updates["emailAddressData"] = email_address_data - elif additional_emails: - existing_email_data = ( - pre_update_contact.get("emailAddressData") - if pre_update_contact - else None - ) - email_address_data = self._build_email_address_data( - email_candidate=None, - additional_emails=additional_emails, - existing_email_data=existing_email_data, - ) - if email_address_data: - normalized_updates["emailAddressData"] = email_address_data - - if "emailAddressData" in normalized_updates: - normalized_email_data = self._coerce_email_address_data( - normalized_updates["emailAddressData"] - ) - if normalized_email_data is not None: - normalized_updates["emailAddressData"] = normalized_email_data - else: - normalized_updates.pop("emailAddressData", None) - - if "cSeniority" in normalized_updates: - normalized_updates["cSeniority"] = self._normalize_seniority( - normalized_updates.get("cSeniority") - ) - if not normalized_updates["cSeniority"]: - normalized_updates.pop("cSeniority", None) - if "cRoles" in normalized_updates: - normalized_roles = self._normalize_roles(normalized_updates["cRoles"]) - if normalized_roles: - normalized_updates["cRoles"] = normalized_roles - else: - normalized_updates.pop("cRoles", None) - if "cTimezone" in normalized_updates: - normalized_tz = self._normalize_timezone( - normalized_updates.get("cTimezone") - ) - if normalized_tz: - normalized_updates["cTimezone"] = normalized_tz - else: - normalized_updates.pop("cTimezone", None) - if "addressCity" in normalized_updates: - normalized_city = self._normalize_city( - normalized_updates.get("addressCity") - ) - if normalized_city: - normalized_updates["addressCity"] = normalized_city - else: - normalized_updates.pop("addressCity", None) - if "addressState" in normalized_updates: - normalized_state = self._normalize_state( - normalized_updates.get("addressState") - ) - if normalized_state: - normalized_updates["addressState"] = normalized_state - else: - normalized_updates.pop("addressState", None) - - allowed_fields = { - "emailAddressData", - "cGitHubUsername", - settings.crm_linkedin_field, - "cSeniority", - "addressCountry", - "cTimezone", - "addressCity", - "addressState", - "description", - "phoneNumber", - "cRoles", - "skills", - "cSkillAttrs", - "cWebsiteLink", - "cSocialLinks", - } - approved_updates: dict[str, Any] = { - field: value - for field, value in normalized_updates.items() - if field in allowed_fields and value - } - parsed_skills_for_apply = self._normalize_skills_for_apply( - approved_updates.get("skills") - ) - if parsed_skills_for_apply is not None: - approved_updates["skills"] = parsed_skills_for_apply - - if not approved_updates: - return ResumeApplyResult( - contact_id=contact_id, - updated_fields=[], - updated_values={}, - success=False, - error="No valid profile fields provided", - ) - - link_applied = False - if link_discord: - discord_user_id = str(link_discord.get("user_id", "")).strip() - discord_username = str(link_discord.get("username", "")).strip() - if discord_user_id and discord_username: - approved_updates["cDiscordUserID"] = discord_user_id - approved_updates["cDiscordUsername"] = ( - f"{discord_username} (ID: {discord_user_id})" - ) - link_applied = True - - if not approved_updates: - return ResumeApplyResult( - contact_id=contact_id, - updated_fields=[], - updated_values={}, - success=False, - error="No valid profile fields provided", - ) - - # NOTE: cResumeLastProcessed is stored as UTC for CRM compatibility. - crm_update_payload = dict(approved_updates) - crm_update_payload["cResumeLastProcessed"] = datetime.now( - tz=timezone.utc - ).strftime("%Y-%m-%d %H:%M:%S") - - try: - self.crm.update_contact(contact_id, crm_update_payload) - verified_fields = self._verify_updated_fields( - contact_id=contact_id, - baseline_contact=pre_update_contact, - candidate_fields=list(approved_updates.keys()), - ) - if verified_fields is None: - verified_fields = sorted(approved_updates.keys()) - return ResumeApplyResult( - contact_id=contact_id, - updated_fields=verified_fields, - updated_values={ - field: approved_updates[field] - for field in verified_fields - if field in approved_updates - }, - link_discord_applied=link_applied, - success=bool(verified_fields), - error=None if verified_fields else "No fields were updated", - ) - except EspoAPIError as batch_error: - logger.warning( - "Resume profile batch update failed for contact_id=%s; applying fields individually. error=%s", - contact_id, - batch_error, - ) - - updated_fields: list[str] = [] - batch_errors: list[str] = [] - for field, value in approved_updates.items(): - try: - self.crm.update_contact(contact_id, {field: value}) - updated_fields.append(field) - except EspoAPIError as field_error: - batch_errors.append(f"{field}: {field_error}") - except Exception as field_error: - batch_errors.append(f"{field}: {field_error}") - - if updated_fields: - verified_fields = self._verify_updated_fields( - contact_id=contact_id, - baseline_contact=pre_update_contact, - candidate_fields=updated_fields, - ) - if verified_fields is not None: - updated_fields = verified_fields - - return ResumeApplyResult( - contact_id=contact_id, - updated_fields=sorted(updated_fields), - updated_values={ - field: approved_updates[field] - for field in sorted(updated_fields) - if field in approved_updates - }, - link_discord_applied=link_applied, - success=True, - ) - - if updated_fields: - return ResumeApplyResult( - contact_id=contact_id, - updated_fields=sorted(updated_fields), - updated_values={ - field: approved_updates[field] - for field in sorted(updated_fields) - if field in approved_updates - }, - link_discord_applied=link_applied, - success=False, - error="; ".join(batch_errors) - if batch_errors - else "Some fields did not persist after update", - ) - - return ResumeApplyResult( - contact_id=contact_id, - updated_fields=sorted(approved_updates.keys()), - updated_values=dict(approved_updates), - link_discord_applied=link_applied, - success=False, - error="; ".join(batch_errors) - if batch_errors - else "No fields were updated", - ) - except EspoAPIError as exc: - logger.error("EspoCRM apply failed contact_id=%s error=%s", contact_id, exc) - return ResumeApplyResult( - contact_id=contact_id, - updated_fields=[], - updated_values={}, - success=False, - error=str(exc), - ) - except Exception as exc: - logger.error( - "Unexpected apply error contact_id=%s error=%s", contact_id, exc - ) - return ResumeApplyResult( - contact_id=contact_id, - updated_fields=[], - updated_values={}, - success=False, - error=str(exc), - ) - - def _normalize_skills_for_apply(self, value: Any) -> list[str] | None: - """Normalize optional skills updates into an array-shaped payload.""" - if value is None: - return None - - if isinstance(value, str): - raw_skills = [item.strip() for item in value.split(",") if item.strip()] - elif isinstance(value, (list, tuple, set)): - raw_skills = [str(item).strip() for item in value if str(item).strip()] - else: - return None - - normalized = normalize_skill_list(raw_skills) - return normalized if normalized else None - - @staticmethod - def _normalize_compare_value(value: Any) -> str: - if value is None: - return "" - if isinstance(value, (dict, list)): - try: - return json.dumps(value, sort_keys=True, separators=(",", ":")) - except Exception: - return str(value) - if isinstance(value, bool): - return str(value).lower() - return str(value).strip() - - def _coerce_skills_updates(self, value: Any) -> list[str]: - if isinstance(value, (list, tuple, set)): - raw_skills = [str(item).strip() for item in value if str(item).strip()] - elif isinstance(value, str): - raw_skills = [item.strip() for item in value.split(",") if item.strip()] - else: - return [] - - return normalize_skill_list(raw_skills) - - def _verify_updated_fields( - self, - *, - contact_id: str, - baseline_contact: dict[str, Any] | None, - candidate_fields: list[str], - ) -> list[str] | None: - if baseline_contact is None: - return None - - baseline: dict[str, str] = {} - for field in candidate_fields: - baseline[field] = self._normalize_compare_value( - baseline_contact.get(field, "") - ) - - try: - after_contact = self.crm.get_contact(contact_id) - except Exception as exc: - logger.debug( - "Failed to read post-update contact for verification contact_id=%s: %s", - contact_id, - exc, - ) - return None - if after_contact is baseline_contact: - return None - if not isinstance(after_contact, dict): - return None - - changed_fields: list[str] = [] - for field in candidate_fields: - after_value = self._normalize_compare_value(after_contact.get(field, "")) - if after_value != baseline.get(field, ""): - changed_fields.append(field) - if not changed_fields: - return candidate_fields - return changed_fields - - def _collect_change( - self, - *, - crm_field: str, - label: str, - current: Any, - proposed: str | None, - proposed_updates: dict[str, Any], - proposed_changes: list[ResumeFieldChange], - skipped: list[ResumeSkipReason], - blocked_reason: str | None = None, - is_blocked: Callable[[str], bool] | None = None, - ) -> None: - if not proposed: - return - - if callable(is_blocked) and is_blocked(proposed): - skipped.append( - ResumeSkipReason( - field=crm_field, - value=proposed, - reason=blocked_reason or "Update blocked by policy", - ) - ) - return - - current_value = str(current).strip() if current is not None else None - if current_value and current_value == proposed: - return - - proposed_updates[crm_field] = proposed - proposed_changes.append( - ResumeFieldChange( - field=crm_field, - label=label, - current=current_value, - proposed=proposed, - reason="Extracted from uploaded resume", - ) - ) - - def _parse_existing_skills(self, value: Any) -> list[str]: - if value is None: - return [] - - if isinstance(value, list): - raw_skills = [str(item).strip() for item in value if str(item).strip()] - elif isinstance(value, (tuple, set)): - raw_skills = [str(item).strip() for item in value if str(item).strip()] - else: - raw_skills = [ - item.strip() for item in str(value).split(",") if item.strip() - ] - return self._dedupe_normalized_skills(raw_skills) - - @staticmethod - def _normalize_seniority(value: Any) -> str | None: - return normalize_seniority(value, empty_as_unknown=True) - - @staticmethod - def _normalize_country(value: Any) -> str | None: - return normalize_country(value) - - @staticmethod - def _normalize_city(value: Any) -> str | None: - return normalize_city(value, strip_parenthetical=False) - - @staticmethod - def _normalize_state(value: Any) -> str | None: - return normalize_state(value) - - @staticmethod - def _normalize_timezone(value: Any) -> str | None: - return normalize_timezone(value) - - @staticmethod - def _normalize_role(value: Any) -> str | None: - return normalize_role(value, ROLE_NORMALIZATION_MAP) - - @staticmethod - def _normalize_roles(value: Any) -> list[str]: - return normalize_roles(value, ROLE_NORMALIZATION_MAP) - - def _format_skills_with_strength( - self, - skills: list[str], - attrs: dict[str, int], - ) -> str: - deduped_skills = self._dedupe_normalized_skills(skills) - formatted: list[str] = [] - for raw_skill in deduped_skills: - skill = raw_skill.strip() - if not skill: - continue - strength = attrs.get(skill.casefold()) - if strength: - formatted.append(f"{skill} ({strength})") - else: - formatted.append(skill) - return ", ".join(formatted) - - def _dedupe_normalized_skills(self, value: Any) -> list[str]: - if value is None: - return [] - - if isinstance(value, str): - raw_skills = [item.strip() for item in value.split(",") if item.strip()] - elif isinstance(value, (list, tuple, set)): - raw_skills = [str(item).strip() for item in value if str(item).strip()] - else: - return [] - - return normalize_skill_list(raw_skills) - - def _normalize_skill(self, value: Any) -> str | None: - normalized = normalize_skill(str(value)) - return normalized or None - - def _coerce_profile_skill_result( - self, - extracted: ResumeExtractedProfile, - resume_text: str, - ) -> ExtractedSkills: - raw_skills = extracted.skills or [] - normalized_skills, normalized_attrs_raw = normalize_skill_payload( - skills_value=raw_skills, - skill_attrs_value=getattr(extracted, "skill_attrs", {}), - disallowed=DISALLOWED_RESUME_SKILLS, - ) - normalized_attrs = { - skill.casefold(): SkillAttributes(strength=strength) - for skill, strength in normalized_attrs_raw.items() - } - - if not normalized_attrs and normalized_skills: - for skill in normalized_skills: - normalized_attrs[skill.casefold()] = SkillAttributes(strength=3) - - if normalized_attrs or normalized_skills: - return ExtractedSkills( - skills=normalized_skills, - skill_attrs=normalized_attrs, - confidence=extracted.confidence, - source=extracted.source, - ) - - fallback = self.skills_extractor.extract_skills(resume_text) - fallback_skills = self._dedupe_normalized_skills(fallback.skills) - if fallback_skills or fallback.skill_attrs: - return ExtractedSkills( - skills=fallback_skills, - skill_attrs=fallback.skill_attrs, - confidence=fallback.confidence, - source=fallback.source, - ) - return fallback - - def _parse_skill_attrs(self, value: Any) -> dict[str, int]: - if value is None: - return {} - - candidate = value - if isinstance(value, str): - raw = value.strip() - if not raw: - return {} - candidate = self._decode_json_like(raw) - if candidate is None: - return {} - - if not isinstance(candidate, dict): - return {} - - parsed: dict[str, int] = {} - for raw_skill, raw_payload in candidate.items(): - normalized = self._normalize_skill(raw_skill) - if not normalized: - continue - skill = normalized.casefold() - - strength_value = raw_payload - if isinstance(raw_payload, dict): - strength_value = raw_payload.get("strength") - - try: - strength = int(float(strength_value)) - except Exception: - strength = 0 - parsed[skill] = max(1, min(5, strength)) if strength else 0 - - return {skill: strength for skill, strength in parsed.items() if strength > 0} - - def _merge_skill_attrs( - self, - *, - existing_attrs: dict[str, int], - extracted_attrs: dict[str, SkillAttributes], - merged_skills: list[str], - ) -> dict[str, int]: - merged: dict[str, int] = dict(existing_attrs) - - for skill, attrs in extracted_attrs.items(): - key = self._normalize_skill(skill) - if not key: - continue - key = key.casefold() - if key: - merged[key] = max(1, min(5, int(attrs.strength))) - - # Ensure every merged skill has a structured strength so attrs never shrink - # to a partial subset when extraction omitted some per-skill scores. - for raw_skill in merged_skills: - canonical = self._normalize_skill(raw_skill) - if not canonical: - continue - key = canonical.casefold() - if key not in merged: - merged[key] = DEFAULT_SKILL_STRENGTH - - return merged - - def _coerce_skill_attrs_updates(self, value: Any) -> str | None: - if value is None: - return None - - candidate = value - if isinstance(value, str): - raw = value.strip() - if not raw: - return None - candidate = self._decode_json_like(raw) - if candidate is None: - return None - - if not isinstance(candidate, dict): - return None - - parsed: dict[str, int] = {} - for raw_skill, raw_payload in candidate.items(): - skill = self._normalize_skill(raw_skill) - if not skill: - continue - strength_source = raw_payload - if isinstance(raw_payload, dict): - strength_source = raw_payload.get("strength") - try: - strength = int(float(strength_source)) - except Exception: - continue - if not 1 <= strength <= 5: - continue - parsed[skill] = strength - - return self._serialize_skill_attrs(parsed) - - @staticmethod - def _decode_json_like(raw: str) -> Any: - """Decode JSON-like strings, including double-encoded and repr payloads.""" - parsed: Any - try: - parsed = json.loads(raw) - except Exception: - try: - parsed = ast.literal_eval(raw) - except Exception: - return None - - if isinstance(parsed, str): - nested = parsed.strip() - if not nested: - return None - try: - reparsed = json.loads(nested) - except Exception: - try: - reparsed = ast.literal_eval(nested) - except Exception: - return parsed - return reparsed - - return parsed - - def _serialize_skill_attrs(self, attrs: dict[str, int]) -> str | None: - normalized: dict[str, dict[str, int]] = {} - for raw_skill, raw_strength in attrs.items(): - skill = self._normalize_skill(raw_skill) - if not skill: - continue - try: - strength = int(float(raw_strength)) - except Exception: - continue - clamped = max(1, min(5, strength)) - normalized[skill] = {"strength": clamped} - if not normalized: - return None - return json.dumps(normalized, sort_keys=True, separators=(",", ":")) - - def _coerce_website_links(self, value: Any) -> list[str]: - if value is None: - return [] - - if isinstance(value, str): - raw_values = [item for item in value.split(",") if item.strip()] - elif isinstance(value, (list, tuple, set)): - raw_values = list(value) - else: - return [] - - normalized: list[str] = [] - seen: set[str] = set() - for raw_value in raw_values: - if not isinstance(raw_value, str): - continue - normalized_link = self._normalize_website_url(raw_value.strip()) - if normalized_link is None: - continue - dedupe_key = normalized_website_identity_key(normalized_link) - if dedupe_key is None or dedupe_key in seen: - continue - seen.add(dedupe_key) - normalized.append(normalized_link) - - return normalized - - def _merge_website_links( - self, *, existing: list[str], extracted: list[str] - ) -> list[str]: - merged: list[str] = [] - seen: set[str] = set() - - for value in existing: - if not isinstance(value, str): - continue - normalized = self._normalize_website_url(value) - if not normalized: - continue - dedupe_key = normalized_website_identity_key(normalized) - if dedupe_key is None or dedupe_key in seen: - continue - seen.add(dedupe_key) - merged.append(normalized) - - for value in extracted: - if not isinstance(value, str): - continue - normalized = self._normalize_website_url(value) - if not normalized: - continue - dedupe_key = normalized_website_identity_key(normalized) - if dedupe_key is None or dedupe_key in seen: - continue - seen.add(dedupe_key) - merged.append(normalized) - - return merged - - @staticmethod - def _normalize_website_url(value: str) -> str | None: - return normalize_website_url(value, allow_scheme_less=True) - - def _normalize_email_address(self, value: Any) -> str | None: - if not isinstance(value, str): - return None - normalized = value.strip().lower() - if not normalized or "@" not in normalized: - return None - return normalized - - @staticmethod - def _coerce_bool(value: Any) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() in {"1", "true", "yes", "on"} - return bool(value) - - def _coerce_additional_emails(self, value: Any) -> list[str]: - if value is None: - return [] - - if isinstance(value, str): - raw_value = value.strip() - if not raw_value: - return [] - try: - parsed = json.loads(raw_value) - except Exception: - parsed = [raw_value] - else: - parsed = parsed if isinstance(parsed, list) else [parsed] - elif isinstance(value, (list, tuple, set)): - parsed = list(value) - else: - parsed = [value] - - deduped: list[str] = [] - seen: set[str] = set() - for item in parsed: - normalized = self._normalize_email_address(item) - if normalized is None: - continue - if normalized in seen: - continue - seen.add(normalized) - deduped.append(normalized) - return deduped - - def _coerce_email_address_data(self, value: Any) -> list[dict[str, Any]] | None: - parsed = self._parse_email_address_data(value) - return parsed if parsed else None - - def _parse_email_address_data(self, value: Any) -> list[dict[str, Any]]: - if value is None: - return [] - - candidate = value - if isinstance(value, str): - raw = value.strip() - if not raw: - return [] - try: - candidate = json.loads(raw) - except Exception: - return [] - - if not isinstance(candidate, list): - if isinstance(candidate, dict): - candidate = [candidate] - else: - return [] - - parsed: list[dict[str, Any]] = [] - for entry in candidate: - if not isinstance(entry, dict): - continue - raw_email = str( - entry.get("lower") - or entry.get("emailAddress") - or entry.get("email") - or "" - ).strip() - normalized_email = self._normalize_email_address(raw_email) - if normalized_email is None: - continue - - parsed.append( - { - "emailAddress": str( - entry.get("emailAddress", normalized_email) - ).strip(), - "lower": normalized_email, - "primary": self._coerce_bool(entry.get("primary")), - "optOut": self._coerce_bool(entry.get("optOut")), - "invalid": self._coerce_bool(entry.get("invalid")), - } - ) - - return parsed - - def _build_email_address_data( - self, - *, - email_candidate: str | None, - additional_emails: list[str] | None = None, - existing_email_data: Any, - ) -> list[dict[str, Any]]: - merged: dict[str, dict[str, Any]] = {} - - for entry in self._parse_email_address_data(existing_email_data): - merged[entry["lower"]] = entry - merged[entry["lower"]]["emailAddress"] = ( - str(entry.get("emailAddress", entry["lower"])).strip().lower() - ) - - candidate_lower = self._normalize_email_address(email_candidate) - if candidate_lower is None: - if not additional_emails: - return list(merged.values()) - candidate_lower = None - - if candidate_lower is not None: - for entry in merged.values(): - entry["primary"] = False - merged[candidate_lower] = { - "emailAddress": candidate_lower, - "lower": candidate_lower, - "primary": True, - "optOut": False, - "invalid": False, - } - - for extra in additional_emails or []: - normalized_extra = self._normalize_email_address(extra) - if normalized_extra is None or normalized_extra == candidate_lower: - continue - if normalized_extra in merged: - if candidate_lower is not None: - merged[normalized_extra]["primary"] = False - continue - merged[normalized_extra] = { - "emailAddress": normalized_extra, - "lower": normalized_extra, - "primary": False, - "optOut": False, - "invalid": False, - } - - return list(merged.values()) - - def _configured_model_name(self) -> str: - """Model identity used for idempotency/ledger keys.""" - if settings.openai_api_key: - return settings.resolved_resume_ai_model - return "heuristic" - - def _record_processing_run( - self, - *, - contact_id: str, - attachment_id: str, - content_hash: str | None, - model_name: str, - status: str, - last_error: str | None = None, - ) -> None: - """Persist one processing result keyed by contact+attachment+version+model.""" - query = """ - INSERT INTO resume_processing_runs ( - contact_id, - attachment_id, - content_hash, - extractor_version, - model_name, - status, - last_error, - processed_at - ) - VALUES (%s, %s, %s, %s, %s, %s, %s, NOW()) - ON CONFLICT (contact_id, attachment_id, extractor_version, model_name) - DO UPDATE SET - content_hash = EXCLUDED.content_hash, - status = EXCLUDED.status, - last_error = EXCLUDED.last_error, - processed_at = NOW(); - """ - try: - with get_postgres_connection(settings) as conn: - with conn.cursor() as cursor: - cursor.execute( - query, - ( - contact_id, - attachment_id, - content_hash, - settings.resume_extractor_version, - model_name, - status, - last_error, - ), - ) - except Exception as exc: - logger.warning( - "Failed to persist resume processing run contact_id=%s attachment_id=%s " - "version=%s model=%s status=%s error=%s", - contact_id, - attachment_id, - settings.resume_extractor_version, - model_name, - status, - exc, - ) + super().__init__(ResumeProcessorConfig.from_settings(settings)) diff --git a/apps/worker/src/five08/worker/models.py b/apps/worker/src/five08/worker/models.py index 1bcbcf06..93a96a96 100644 --- a/apps/worker/src/five08/worker/models.py +++ b/apps/worker/src/five08/worker/models.py @@ -7,8 +7,24 @@ from five08.resume_extractor import ( ResumeExtractedProfile as SharedResumeExtractedProfile, ) +from five08.resume_processing_models import ( + ExtractedSkills as SharedExtractedSkills, + ResumeApplyResult as SharedResumeApplyResult, + ResumeExtractionResult as SharedResumeExtractionResult, + ResumeFieldChange as SharedResumeFieldChange, + ResumeSkipReason as SharedResumeSkipReason, + SkillAttributes as SharedSkillAttributes, +) from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator +ExtractedSkills = SharedExtractedSkills +ResumeApplyResult = SharedResumeApplyResult +ResumeExtractionResult = SharedResumeExtractionResult +ResumeFieldChange = SharedResumeFieldChange +ResumeSkipReason = SharedResumeSkipReason +SkillAttributes = SharedSkillAttributes +ResumeExtractedProfile = SharedResumeExtractedProfile + class WebhookEvent(BaseModel): """Single webhook event from EspoCRM.""" @@ -40,21 +56,6 @@ class ContactData(BaseModel): skills: str | None = None -class ExtractedSkills(BaseModel): - """Skills extraction response.""" - - skills: list[str] - skill_attrs: dict[str, "SkillAttributes"] = Field(default_factory=dict) - confidence: float = Field(..., ge=0.0, le=1.0) - source: str - - -class SkillAttributes(BaseModel): - """Structured per-skill metadata for CRM persistence.""" - - strength: int = Field(..., ge=1, le=5) - - class SkillsExtractionResult(BaseModel): """End-to-end processing result.""" @@ -67,53 +68,6 @@ class SkillsExtractionResult(BaseModel): error: str | None = None -ResumeExtractedProfile = SharedResumeExtractedProfile - - -class ResumeFieldChange(BaseModel): - """Single proposed CRM field update.""" - - field: str - label: str - current: str | None = None - proposed: str - reason: str - - -class ResumeSkipReason(BaseModel): - """Field extraction skip explanation for preview UX.""" - - field: str - value: str - reason: str - - -class ResumeExtractionResult(BaseModel): - """Worker output used by bot preview/confirmation flow.""" - - contact_id: str - attachment_id: str - proposed_updates: dict[str, Any] - proposed_changes: list[ResumeFieldChange] - skipped: list[ResumeSkipReason] - extracted_profile: ResumeExtractedProfile - extracted_skills: list[str] = Field(default_factory=list) - new_skills: list[str] = Field(default_factory=list) - success: bool - error: str | None = None - - -class ResumeApplyResult(BaseModel): - """CRM apply-phase result.""" - - contact_id: str - updated_fields: list[str] - updated_values: dict[str, Any] = Field(default_factory=dict) - link_discord_applied: bool = False - success: bool - error: str | None = None - - class DocusealSubmitter(BaseModel): """Single submitter entry from a Docuseal webhook payload.""" diff --git a/packages/shared/src/five08/resume_document_processor.py b/packages/shared/src/five08/resume_document_processor.py new file mode 100644 index 00000000..35285c80 --- /dev/null +++ b/packages/shared/src/five08/resume_document_processor.py @@ -0,0 +1,67 @@ +"""Resume document text extraction.""" + +import hashlib +import logging +from pathlib import Path + +from five08.document_text import extract_document_text + +logger = logging.getLogger(__name__) + + +class DocumentProcessor: + """Extract text from supported resume file formats.""" + + def __init__( + self, + *, + allowed_extensions: set[str], + max_file_size_mb: int, + ) -> None: + self.allowed_extensions = {ext.strip().lower() for ext in allowed_extensions} + self.max_file_size = max(1, int(max_file_size_mb)) * 1024 * 1024 + self._content_cache: dict[str, str] = {} + + def get_content_hash(self, content: bytes, filename: str) -> str: + """Hash bytes for extraction caching.""" + extension = Path(filename).suffix.lower().encode("utf-8") + return hashlib.sha256(content + b"\0" + extension).hexdigest() + + def is_valid_file(self, filename: str, file_size: int) -> tuple[bool, str | None]: + """Validate extension and size.""" + if file_size > self.max_file_size: + return False, f"File size {file_size} exceeds maximum {self.max_file_size}" + + ext = Path(filename).suffix.lower().lstrip(".") + if ext not in self.allowed_extensions: + return ( + False, + f"File extension '{ext}' not allowed. Allowed: {self.allowed_extensions}", + ) + return True, None + + def extract_text(self, content: bytes, filename: str) -> str: + """Extract text from supported format and cache results.""" + content_hash = self.get_content_hash(content, filename) + if content_hash in self._content_cache: + return self._content_cache[content_hash] + + is_valid, error = self.is_valid_file(filename, len(content)) + if not is_valid: + raise ValueError(error or "Invalid file") + + try: + text = extract_document_text(content, filename=filename) + except Exception as exc: + logger.error( + "Error extracting document text filename=%s: %s", filename, exc + ) + raise ValueError( + f"Failed to extract text from {Path(filename).suffix}: {exc}" + ) from exc + + if not text.strip(): + raise ValueError("No text could be extracted from document") + + self._content_cache[content_hash] = text + return text diff --git a/packages/shared/src/five08/resume_processing_models.py b/packages/shared/src/five08/resume_processing_models.py new file mode 100644 index 00000000..de0c13a8 --- /dev/null +++ b/packages/shared/src/five08/resume_processing_models.py @@ -0,0 +1,91 @@ +"""Shared typed models for resume processing flows.""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field, field_validator + +from five08.resume_extractor import ResumeExtractedProfile + + +class SkillAttributes(BaseModel): + """Structured per-skill metadata for CRM persistence.""" + + strength: int = Field(..., ge=1, le=5) + + +class ExtractedSkills(BaseModel): + """Skills extraction response.""" + + skills: list[str] + skill_attrs: dict[str, SkillAttributes] = Field(default_factory=dict) + confidence: float = Field(..., ge=0.0, le=1.0) + source: str + + @field_validator("skill_attrs", mode="before") + @classmethod + def _coerce_skill_attrs(cls, value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + return {} + + normalized: dict[str, dict[str, int] | SkillAttributes] = {} + for skill, payload in value.items(): + if isinstance(payload, SkillAttributes): + normalized[str(skill)] = payload + continue + + if isinstance(payload, dict): + normalized[str(skill)] = payload + continue + + strength = getattr(payload, "strength", None) + if strength is None: + continue + normalized[str(skill)] = {"strength": int(strength)} + return normalized + + +class ResumeFieldChange(BaseModel): + """Single proposed CRM field update.""" + + field: str + label: str + current: str | None = None + proposed: str + reason: str + + +class ResumeSkipReason(BaseModel): + """Field extraction skip explanation for preview UX.""" + + field: str + value: str + reason: str + + +class ResumeExtractionResult(BaseModel): + """Resume extraction output used by preview/confirmation flows.""" + + contact_id: str + attachment_id: str + proposed_updates: dict[str, Any] + proposed_changes: list[ResumeFieldChange] + skipped: list[ResumeSkipReason] + extracted_profile: ResumeExtractedProfile + extracted_skills: list[str] = Field(default_factory=list) + new_skills: list[str] = Field(default_factory=list) + success: bool + error: str | None = None + + +class ResumeApplyResult(BaseModel): + """CRM apply-phase result.""" + + contact_id: str + updated_fields: list[str] + updated_values: dict[str, Any] = Field(default_factory=dict) + link_discord_applied: bool = False + success: bool + error: str | None = None + warning: str | None = None diff --git a/packages/shared/src/five08/resume_profile_processor.py b/packages/shared/src/five08/resume_profile_processor.py new file mode 100644 index 00000000..291ef336 --- /dev/null +++ b/packages/shared/src/five08/resume_profile_processor.py @@ -0,0 +1,1407 @@ +"""Resume extraction + CRM profile update workflow.""" + +from __future__ import annotations + +import ast +import json +import logging +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +from psycopg import connect + +from five08.clients.espo import EspoAPIError, EspoClient +from five08.crm_normalization import ( + ROLE_NORMALIZATION_MAP, + normalized_website_identity_key, + normalize_city, + normalize_country, + normalize_role, + normalize_roles, + normalize_state, + normalize_seniority, + normalize_timezone, + normalize_website_url, +) +from five08.skills import ( + DISALLOWED_RESUME_SKILLS, + normalize_skill, + normalize_skill_list, + normalize_skill_payload, +) +from five08.resume_document_processor import DocumentProcessor +from five08.resume_extractor import ResumeExtractedProfile, ResumeProfileExtractor +from five08.resume_processing_models import ( + ResumeApplyResult, + ExtractedSkills, + ResumeExtractionResult, + ResumeFieldChange, + ResumeSkipReason, + SkillAttributes, +) +from five08.resume_skills_extractor import SkillsExtractor + +logger = logging.getLogger(__name__) +DEFAULT_SKILL_STRENGTH = 3 +SUPPORTED_RESUME_FILE_EXTENSIONS = ("pdf", "docx") +DEFAULT_RESUME_MAX_FILE_SIZE_MB = 10 +LINKEDIN_FIELD = "cLinkedIn" + + +def _normalize_allowed_resume_extensions(value: Any) -> set[str]: + if isinstance(value, set): + raw_extensions = value + elif isinstance(value, (list, tuple)): + raw_extensions = set(value) + else: + raw_extensions = set() + + normalized = { + str(ext).strip().lower().lstrip(".") + for ext in raw_extensions + if str(ext).strip() + } + supported = { + ext for ext in normalized if ext in set(SUPPORTED_RESUME_FILE_EXTENSIONS) + } + return supported or set(SUPPORTED_RESUME_FILE_EXTENSIONS) + + +@dataclass(frozen=True) +class ResumeProcessorConfig: + """Runtime configuration required by the shared resume processor.""" + + espo_base_url: str + espo_api_key: str + openai_api_key: str | None = None + openai_base_url: str | None = None + resume_model: str = "gpt-5-mini" + resume_extractor_max_tokens: int = 2000 + allowed_file_extensions: set[str] = field( + default_factory=lambda: set(SUPPORTED_RESUME_FILE_EXTENSIONS) + ) + max_file_size_mb: int = DEFAULT_RESUME_MAX_FILE_SIZE_MB + resume_extractor_version: str = "v1" + postgres_url: str = "" + + @property + def allowed_attachment_suffixes(self) -> frozenset[str]: + return frozenset(f".{ext}" for ext in self.allowed_file_extensions) + + @property + def allowed_file_extensions_label(self) -> str: + labels = [ + ext.upper() + for ext in SUPPORTED_RESUME_FILE_EXTENSIONS + if ext in self.allowed_file_extensions + ] + if not labels: + labels = [ext.upper() for ext in SUPPORTED_RESUME_FILE_EXTENSIONS] + if len(labels) == 1: + return labels[0] + return f"{labels[0]} or {labels[1]}" + + @property + def max_file_size_bytes(self) -> int: + return max(1, int(self.max_file_size_mb)) * 1024 * 1024 + + @classmethod + def from_settings(cls, settings: Any) -> "ResumeProcessorConfig": + allowed_extensions = getattr(settings, "allowed_file_extensions", None) + if not isinstance(allowed_extensions, set): + raw_allowed_types = str(getattr(settings, "allowed_file_types", "")).strip() + allowed_extensions = { + ext.strip().lower() + for ext in raw_allowed_types.split(",") + if ext.strip() + } + if not allowed_extensions: + allowed_extensions = set(SUPPORTED_RESUME_FILE_EXTENSIONS) + + resume_model = ( + str(getattr(settings, "resolved_resume_ai_model", "")).strip() + or str(getattr(settings, "resume_ai_model", "")).strip() + or str(getattr(settings, "openai_model", "")).strip() + or "gpt-5-mini" + ) + + return cls( + espo_base_url=str(getattr(settings, "espo_base_url")), + espo_api_key=str(getattr(settings, "espo_api_key")), + openai_api_key=getattr(settings, "openai_api_key", None), + openai_base_url=getattr(settings, "openai_base_url", None), + resume_model=resume_model, + resume_extractor_max_tokens=int( + getattr(settings, "resume_extractor_max_tokens", 2000) + ), + allowed_file_extensions=_normalize_allowed_resume_extensions( + allowed_extensions + ), + max_file_size_mb=int( + getattr(settings, "max_file_size_mb", DEFAULT_RESUME_MAX_FILE_SIZE_MB) + ), + resume_extractor_version=str( + getattr(settings, "resume_extractor_version", "v1") + ).strip() + or "v1", + postgres_url=str(getattr(settings, "postgres_url", "")).strip(), + ) + + +class ResumeProfileProcessor: + """End-to-end extraction and apply operations for uploaded resumes.""" + + def __init__(self, config: ResumeProcessorConfig) -> None: + self.config = config + self.crm = EspoClient(config.espo_base_url, config.espo_api_key) + self.extractor = ResumeProfileExtractor( + api_key=config.openai_api_key, + base_url=config.openai_base_url, + model=config.resume_model, + max_tokens=config.resume_extractor_max_tokens, + ) + self.skills_extractor = SkillsExtractor( + model=config.resume_model, + openai_api_key=config.openai_api_key, + openai_base_url=config.openai_base_url, + ) + self.document_processor = DocumentProcessor( + allowed_extensions=config.allowed_file_extensions, + max_file_size_mb=config.max_file_size_mb, + ) + + def extract_profile_proposal( + self, + *, + contact_id: str, + attachment_id: str, + filename: str, + ) -> ResumeExtractionResult: + """Build preview proposal from an uploaded resume attachment.""" + content_hash: str | None = None + model_name = self._configured_model_name() + try: + contact = self.crm.get_contact(contact_id) + content = self.crm.download_attachment(attachment_id) + content_hash = self.document_processor.get_content_hash(content, filename) + text = self.document_processor.extract_text(content, filename) + extracted = self.extractor.extract(text) + model_name = extracted.source + extracted_skills_result = self._coerce_profile_skill_result(extracted, text) + extracted_skills = extracted_skills_result.skills + normalized_extracted_skills = self._dedupe_normalized_skills( + extracted_skills + ) + existing_skills = self._parse_existing_skills(contact.get("skills")) + existing_skill_attrs = self._parse_skill_attrs(contact.get("cSkillAttrs")) + existing_websites = self._coerce_website_links(contact.get("cWebsiteLink")) + existing_social_links = self._coerce_website_links( + contact.get("cSocialLinks") + ) + existing_lower = {item.casefold() for item in existing_skills} + new_skills = [ + skill + for skill in normalized_extracted_skills + if skill.casefold() not in existing_lower + ] + merged_skills = self._dedupe_normalized_skills(existing_skills + new_skills) + merged_websites = self._merge_website_links( + existing=existing_websites, + extracted=extracted.website_links, + ) + merged_social_links = self._merge_website_links( + existing=existing_social_links, + extracted=extracted.social_links, + ) + merged_skill_attrs = self._merge_skill_attrs( + existing_attrs=existing_skill_attrs, + extracted_attrs=extracted_skills_result.skill_attrs, + merged_skills=merged_skills, + ) + + proposed_updates: dict[str, Any] = {} + proposed_changes: list[ResumeFieldChange] = [] + skipped: list[ResumeSkipReason] = [] + + self._collect_change( + crm_field="emailAddress", + label="Email", + current=contact.get("emailAddress"), + proposed=extracted.email, + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + blocked_reason="Skipped because @508.dev emails are managed separately", + is_blocked=lambda value: value.lower().endswith("@508.dev"), + ) + if extracted.additional_emails: + proposed_updates["additional_emails"] = extracted.additional_emails + proposed_changes.append( + ResumeFieldChange( + field="additional_emails", + label="Additional Emails", + current=None, + proposed=", ".join(extracted.additional_emails), + reason="Extracted additional emails from uploaded resume", + ) + ) + self._collect_change( + crm_field="cGitHubUsername", + label="GitHub", + current=contact.get("cGitHubUsername"), + proposed=extracted.github_username, + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + ) + self._collect_change( + crm_field=LINKEDIN_FIELD, + label="LinkedIn", + current=contact.get(LINKEDIN_FIELD), + proposed=extracted.linkedin_url, + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + ) + self._collect_change( + crm_field="phoneNumber", + label="Phone", + current=contact.get("phoneNumber"), + proposed=extracted.phone, + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + ) + self._collect_change( + crm_field="addressCountry", + label="Country", + current=contact.get("addressCountry"), + proposed=self._normalize_country(extracted.address_country), + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + ) + self._collect_change( + crm_field="cTimezone", + label="Timezone", + current=contact.get("cTimezone"), + proposed=self._normalize_timezone(extracted.timezone), + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + ) + self._collect_change( + crm_field="addressCity", + label="City", + current=contact.get("addressCity"), + proposed=self._normalize_city(extracted.address_city), + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + ) + self._collect_change( + crm_field="addressState", + label="State", + current=contact.get("addressState"), + proposed=self._normalize_state(extracted.address_state), + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + ) + self._collect_change( + crm_field="description", + label="Description", + current=contact.get("description"), + proposed=extracted.description.strip() + if extracted.description + else None, + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + ) + extracted_roles = self._normalize_roles(extracted.primary_roles) + existing_roles = self._normalize_roles(contact.get("cRoles")) + if extracted_roles and sorted(extracted_roles) != sorted(existing_roles): + proposed_updates["cRoles"] = extracted_roles + proposed_changes.append( + ResumeFieldChange( + field="cRoles", + label="Roles", + current=", ".join(existing_roles), + proposed=", ".join(extracted_roles), + reason="Extracted from uploaded resume", + ) + ) + current_seniority = self._normalize_seniority(contact.get("cSeniority")) + proposed_seniority = self._normalize_seniority(extracted.seniority_level) + self._collect_change( + crm_field="cSeniority", + label="Seniority", + current=current_seniority, + proposed=proposed_seniority, + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + is_blocked=lambda value: bool( + current_seniority + and current_seniority != "unknown" + and value != current_seniority + ), + blocked_reason="Existing seniority preserved", + ) + if new_skills: + proposed_updates["skills"] = merged_skills + if merged_skill_attrs: + skill_attrs_payload = self._serialize_skill_attrs( + merged_skill_attrs + ) + if skill_attrs_payload: + proposed_updates["cSkillAttrs"] = skill_attrs_payload + proposed_changes.append( + ResumeFieldChange( + field="skills", + label="Skills", + current=( + self._format_skills_with_strength( + existing_skills, existing_skill_attrs + ) + if existing_skills + else None + ), + proposed=self._format_skills_with_strength( + merged_skills, merged_skill_attrs + ), + reason="Added skills from resume extraction", + ) + ) + + if merged_websites != existing_websites: + proposed_updates["cWebsiteLink"] = merged_websites + proposed_changes.append( + ResumeFieldChange( + field="cWebsiteLink", + label="Website", + current=", ".join(existing_websites), + proposed=", ".join(merged_websites), + reason="Extracted from uploaded resume", + ) + ) + + if merged_social_links != existing_social_links: + proposed_updates["cSocialLinks"] = merged_social_links + proposed_changes.append( + ResumeFieldChange( + field="cSocialLinks", + label="Social Links", + current=", ".join(existing_social_links), + proposed=", ".join(merged_social_links), + reason="Extracted from uploaded resume", + ) + ) + + self._record_processing_run( + contact_id=contact_id, + attachment_id=attachment_id, + content_hash=content_hash, + model_name=model_name, + status="succeeded", + ) + + return ResumeExtractionResult( + contact_id=contact_id, + attachment_id=attachment_id, + proposed_updates=proposed_updates, + proposed_changes=proposed_changes, + skipped=skipped, + extracted_profile=extracted, + extracted_skills=extracted_skills, + new_skills=new_skills, + success=True, + ) + except Exception as exc: + logger.error( + "Resume extraction proposal failed contact_id=%s attachment_id=%s error=%s", + contact_id, + attachment_id, + exc, + ) + self._record_processing_run( + contact_id=contact_id, + attachment_id=attachment_id, + content_hash=content_hash, + model_name=model_name, + status="failed", + last_error=str(exc), + ) + return ResumeExtractionResult( + contact_id=contact_id, + attachment_id=attachment_id, + proposed_updates={}, + proposed_changes=[], + skipped=[], + extracted_profile=ResumeExtractedProfile( + email=None, + github_username=None, + linkedin_url=None, + phone=None, + confidence=0.0, + source="error", + ), + extracted_skills=[], + new_skills=[], + success=False, + error=str(exc), + ) + + def apply_profile_updates( + self, + *, + contact_id: str, + updates: dict[str, Any], + link_discord: dict[str, str] | None = None, + ) -> ResumeApplyResult: + """Apply confirmed updates to contact in CRM.""" + try: + candidate_email = None + normalized_updates = dict(updates) + + pre_update_contact: dict[str, Any] | None = None + try: + pre_update_contact = self.crm.get_contact(contact_id) + except Exception as exc: + logger.debug( + "Failed to read pre-update contact for verification contact_id=%s: %s", + contact_id, + exc, + ) + + if "emailAddress" in normalized_updates: + candidate_email = self._normalize_email_address( + normalized_updates.get("emailAddress") + ) + if "emailAddress" in normalized_updates: + normalized_updates.pop("emailAddress", None) + + additional_emails: list[str] = [] + if "additional_emails" in normalized_updates: + additional_emails = self._coerce_additional_emails( + normalized_updates.get("additional_emails") + ) + normalized_updates.pop("additional_emails", None) + + if "skills" in normalized_updates: + normalized_skills = self._coerce_skills_updates( + normalized_updates["skills"] + ) + if normalized_skills: + normalized_updates["skills"] = normalized_skills + else: + normalized_updates.pop("skills", None) + + if "cSkillAttrs" in normalized_updates: + serialized_attrs = self._coerce_skill_attrs_updates( + normalized_updates["cSkillAttrs"] + ) + if serialized_attrs: + normalized_updates["cSkillAttrs"] = serialized_attrs + else: + normalized_updates.pop("cSkillAttrs", None) + + if "cWebsiteLink" in normalized_updates: + normalized_websites = self._coerce_website_links( + normalized_updates["cWebsiteLink"] + ) + if normalized_websites: + normalized_updates["cWebsiteLink"] = normalized_websites + else: + normalized_updates.pop("cWebsiteLink", None) + + if candidate_email is not None: + if candidate_email.endswith("@508.dev"): + candidate_email = None + if candidate_email is not None: + existing_email_data = ( + pre_update_contact.get("emailAddressData") + if pre_update_contact + else None + ) + email_address_data = self._build_email_address_data( + email_candidate=candidate_email, + additional_emails=additional_emails, + existing_email_data=existing_email_data, + ) + if email_address_data: + normalized_updates["emailAddressData"] = email_address_data + elif additional_emails: + existing_email_data = ( + pre_update_contact.get("emailAddressData") + if pre_update_contact + else None + ) + email_address_data = self._build_email_address_data( + email_candidate=None, + additional_emails=additional_emails, + existing_email_data=existing_email_data, + ) + if email_address_data: + normalized_updates["emailAddressData"] = email_address_data + + if "emailAddressData" in normalized_updates: + normalized_email_data = self._coerce_email_address_data( + normalized_updates["emailAddressData"] + ) + if normalized_email_data is not None: + normalized_updates["emailAddressData"] = normalized_email_data + else: + normalized_updates.pop("emailAddressData", None) + + if "cSeniority" in normalized_updates: + normalized_updates["cSeniority"] = self._normalize_seniority( + normalized_updates.get("cSeniority") + ) + if not normalized_updates["cSeniority"]: + normalized_updates.pop("cSeniority", None) + if "cRoles" in normalized_updates: + normalized_roles = self._normalize_roles(normalized_updates["cRoles"]) + if normalized_roles: + normalized_updates["cRoles"] = normalized_roles + else: + normalized_updates.pop("cRoles", None) + if "cTimezone" in normalized_updates: + normalized_tz = self._normalize_timezone( + normalized_updates.get("cTimezone") + ) + if normalized_tz: + normalized_updates["cTimezone"] = normalized_tz + else: + normalized_updates.pop("cTimezone", None) + if "addressCity" in normalized_updates: + normalized_city = self._normalize_city( + normalized_updates.get("addressCity") + ) + if normalized_city: + normalized_updates["addressCity"] = normalized_city + else: + normalized_updates.pop("addressCity", None) + if "addressState" in normalized_updates: + normalized_state = self._normalize_state( + normalized_updates.get("addressState") + ) + if normalized_state: + normalized_updates["addressState"] = normalized_state + else: + normalized_updates.pop("addressState", None) + + allowed_fields = { + "emailAddressData", + "cGitHubUsername", + LINKEDIN_FIELD, + "cSeniority", + "addressCountry", + "cTimezone", + "addressCity", + "addressState", + "description", + "phoneNumber", + "cRoles", + "skills", + "cSkillAttrs", + "cWebsiteLink", + "cSocialLinks", + } + approved_updates: dict[str, Any] = { + field: value + for field, value in normalized_updates.items() + if field in allowed_fields and value + } + parsed_skills_for_apply = self._normalize_skills_for_apply( + approved_updates.get("skills") + ) + if parsed_skills_for_apply is not None: + approved_updates["skills"] = parsed_skills_for_apply + + link_applied = False + if link_discord: + discord_user_id = str(link_discord.get("user_id", "")).strip() + discord_username = str(link_discord.get("username", "")).strip() + if discord_user_id and discord_username: + approved_updates["cDiscordUserID"] = discord_user_id + approved_updates["cDiscordUsername"] = ( + f"{discord_username} (ID: {discord_user_id})" + ) + link_applied = True + + if not approved_updates: + return ResumeApplyResult( + contact_id=contact_id, + updated_fields=[], + updated_values={}, + success=False, + error="No valid profile fields provided", + ) + + # NOTE: cResumeLastProcessed is stored as UTC for CRM compatibility. + processed_at = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + crm_update_payload = dict(approved_updates) + crm_update_payload["cResumeLastProcessed"] = processed_at + + try: + self.crm.update_contact(contact_id, crm_update_payload) + verified_fields = self._verify_updated_fields( + contact_id=contact_id, + baseline_contact=pre_update_contact, + candidate_fields=list(approved_updates.keys()), + ) + if verified_fields is None: + verified_fields = sorted(approved_updates.keys()) + return ResumeApplyResult( + contact_id=contact_id, + updated_fields=verified_fields, + updated_values={ + field: approved_updates[field] + for field in verified_fields + if field in approved_updates + }, + link_discord_applied=link_applied, + success=bool(verified_fields), + error=None if verified_fields else "No fields were updated", + ) + except EspoAPIError as batch_error: + logger.warning( + "Resume profile batch update failed for contact_id=%s; applying fields individually. error=%s", + contact_id, + batch_error, + ) + + updated_fields: list[str] = [] + batch_errors: list[str] = [] + for field, value in approved_updates.items(): + try: + self.crm.update_contact(contact_id, {field: value}) + updated_fields.append(field) + except EspoAPIError as field_error: + batch_errors.append(f"{field}: {field_error}") + except Exception as field_error: + batch_errors.append(f"{field}: {field_error}") + + if updated_fields: + try: + self.crm.update_contact( + contact_id, {"cResumeLastProcessed": processed_at} + ) + except EspoAPIError as timestamp_error: + batch_errors.append(f"cResumeLastProcessed: {timestamp_error}") + except Exception as timestamp_error: + batch_errors.append(f"cResumeLastProcessed: {timestamp_error}") + + verified_fields = self._verify_updated_fields( + contact_id=contact_id, + baseline_contact=pre_update_contact, + candidate_fields=updated_fields, + ) + if verified_fields is not None: + updated_fields = verified_fields + warning_message = "; ".join(batch_errors) if batch_errors else None + + return ResumeApplyResult( + contact_id=contact_id, + updated_fields=sorted(updated_fields), + updated_values={ + field: approved_updates[field] + for field in sorted(updated_fields) + if field in approved_updates + }, + link_discord_applied=link_applied, + success=bool(updated_fields), + error=None + if updated_fields + else "Some fields did not persist after update", + warning=warning_message, + ) + + return ResumeApplyResult( + contact_id=contact_id, + updated_fields=[], + updated_values={}, + link_discord_applied=link_applied, + success=False, + error="; ".join(batch_errors) + if batch_errors + else "No fields were updated", + ) + except EspoAPIError as exc: + logger.error("EspoCRM apply failed contact_id=%s error=%s", contact_id, exc) + return ResumeApplyResult( + contact_id=contact_id, + updated_fields=[], + updated_values={}, + success=False, + error=str(exc), + ) + except Exception as exc: + logger.error( + "Unexpected apply error contact_id=%s error=%s", contact_id, exc + ) + return ResumeApplyResult( + contact_id=contact_id, + updated_fields=[], + updated_values={}, + success=False, + error=str(exc), + ) + + def _normalize_skills_for_apply(self, value: Any) -> list[str] | None: + """Normalize optional skills updates into an array-shaped payload.""" + if value is None: + return None + + if isinstance(value, str): + raw_skills = [item.strip() for item in value.split(",") if item.strip()] + elif isinstance(value, (list, tuple, set)): + raw_skills = [str(item).strip() for item in value if str(item).strip()] + else: + return None + + normalized = normalize_skill_list(raw_skills) + return normalized if normalized else None + + @staticmethod + def _normalize_compare_value(value: Any) -> str: + if value is None: + return "" + if isinstance(value, (dict, list)): + try: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + except Exception: + return str(value) + if isinstance(value, bool): + return str(value).lower() + return str(value).strip() + + def _coerce_skills_updates(self, value: Any) -> list[str]: + if isinstance(value, (list, tuple, set)): + raw_skills = [str(item).strip() for item in value if str(item).strip()] + elif isinstance(value, str): + raw_skills = [item.strip() for item in value.split(",") if item.strip()] + else: + return [] + + return normalize_skill_list(raw_skills) + + def _verify_updated_fields( + self, + *, + contact_id: str, + baseline_contact: dict[str, Any] | None, + candidate_fields: list[str], + ) -> list[str] | None: + if baseline_contact is None: + return None + + baseline: dict[str, str] = {} + for field_name in candidate_fields: + baseline[field_name] = self._normalize_compare_value( + baseline_contact.get(field_name, "") + ) + + try: + after_contact = self.crm.get_contact(contact_id) + except Exception as exc: + logger.debug( + "Failed to read post-update contact for verification contact_id=%s: %s", + contact_id, + exc, + ) + return None + if after_contact is baseline_contact: + return None + if not isinstance(after_contact, dict): + return None + + changed_fields: list[str] = [] + for field_name in candidate_fields: + after_value = self._normalize_compare_value( + after_contact.get(field_name, "") + ) + if after_value != baseline.get(field_name, ""): + changed_fields.append(field_name) + if not changed_fields: + return candidate_fields + return changed_fields + + def _collect_change( + self, + *, + crm_field: str, + label: str, + current: Any, + proposed: str | None, + proposed_updates: dict[str, Any], + proposed_changes: list[ResumeFieldChange], + skipped: list[ResumeSkipReason], + blocked_reason: str | None = None, + is_blocked: Callable[[str], bool] | None = None, + ) -> None: + if not proposed: + return + + if callable(is_blocked) and is_blocked(proposed): + skipped.append( + ResumeSkipReason( + field=crm_field, + value=proposed, + reason=blocked_reason or "Update blocked by policy", + ) + ) + return + + current_value = str(current).strip() if current is not None else None + if current_value and current_value == proposed: + return + + proposed_updates[crm_field] = proposed + proposed_changes.append( + ResumeFieldChange( + field=crm_field, + label=label, + current=current_value, + proposed=proposed, + reason="Extracted from uploaded resume", + ) + ) + + def _parse_existing_skills(self, value: Any) -> list[str]: + if value is None: + return [] + + if isinstance(value, list): + raw_skills = [str(item).strip() for item in value if str(item).strip()] + elif isinstance(value, (tuple, set)): + raw_skills = [str(item).strip() for item in value if str(item).strip()] + else: + raw_skills = [ + item.strip() for item in str(value).split(",") if item.strip() + ] + return self._dedupe_normalized_skills(raw_skills) + + @staticmethod + def _normalize_seniority(value: Any) -> str | None: + return normalize_seniority(value, empty_as_unknown=True) + + @staticmethod + def _normalize_country(value: Any) -> str | None: + return normalize_country(value) + + @staticmethod + def _normalize_city(value: Any) -> str | None: + return normalize_city(value, strip_parenthetical=False) + + @staticmethod + def _normalize_state(value: Any) -> str | None: + return normalize_state(value) + + @staticmethod + def _normalize_timezone(value: Any) -> str | None: + return normalize_timezone(value) + + @staticmethod + def _normalize_role(value: Any) -> str | None: + return normalize_role(value, ROLE_NORMALIZATION_MAP) + + @staticmethod + def _normalize_roles(value: Any) -> list[str]: + return normalize_roles(value, ROLE_NORMALIZATION_MAP) + + def _format_skills_with_strength( + self, + skills: list[str], + attrs: dict[str, int], + ) -> str: + deduped_skills = self._dedupe_normalized_skills(skills) + formatted: list[str] = [] + for raw_skill in deduped_skills: + skill = raw_skill.strip() + if not skill: + continue + strength = attrs.get(skill.casefold()) + if strength: + formatted.append(f"{skill} ({strength})") + else: + formatted.append(skill) + return ", ".join(formatted) + + def _dedupe_normalized_skills(self, value: Any) -> list[str]: + if value is None: + return [] + + if isinstance(value, str): + raw_skills = [item.strip() for item in value.split(",") if item.strip()] + elif isinstance(value, (list, tuple, set)): + raw_skills = [str(item).strip() for item in value if str(item).strip()] + else: + return [] + + return normalize_skill_list(raw_skills) + + def _normalize_skill(self, value: Any) -> str | None: + normalized = normalize_skill(str(value)) + return normalized or None + + def _coerce_profile_skill_result( + self, + extracted: ResumeExtractedProfile, + resume_text: str, + ) -> ExtractedSkills: + raw_skills = extracted.skills or [] + normalized_skills, normalized_attrs_raw = normalize_skill_payload( + skills_value=raw_skills, + skill_attrs_value=getattr(extracted, "skill_attrs", {}), + disallowed=DISALLOWED_RESUME_SKILLS, + ) + normalized_attrs = { + skill.casefold(): SkillAttributes(strength=strength) + for skill, strength in normalized_attrs_raw.items() + } + + if not normalized_attrs and normalized_skills: + for skill in normalized_skills: + normalized_attrs[skill.casefold()] = SkillAttributes(strength=3) + + if normalized_attrs or normalized_skills: + return ExtractedSkills( + skills=normalized_skills, + skill_attrs=normalized_attrs, + confidence=extracted.confidence, + source=extracted.source, + ) + + fallback = self.skills_extractor.extract_skills(resume_text) + fallback_skills = self._dedupe_normalized_skills(fallback.skills) + if fallback_skills or fallback.skill_attrs: + return ExtractedSkills( + skills=fallback_skills, + skill_attrs=fallback.skill_attrs, + confidence=fallback.confidence, + source=fallback.source, + ) + return fallback + + def _parse_skill_attrs(self, value: Any) -> dict[str, int]: + if value is None: + return {} + + candidate = value + if isinstance(value, str): + raw = value.strip() + if not raw: + return {} + candidate = self._decode_json_like(raw) + if candidate is None: + return {} + + if not isinstance(candidate, dict): + return {} + + parsed: dict[str, int] = {} + for raw_skill, raw_payload in candidate.items(): + normalized = self._normalize_skill(raw_skill) + if not normalized: + continue + skill = normalized.casefold() + + strength_value = raw_payload + if isinstance(raw_payload, dict): + strength_value = raw_payload.get("strength") + + try: + strength = int(float(strength_value)) + except Exception: + strength = 0 + parsed[skill] = max(1, min(5, strength)) if strength else 0 + + return {skill: strength for skill, strength in parsed.items() if strength > 0} + + def _merge_skill_attrs( + self, + *, + existing_attrs: dict[str, int], + extracted_attrs: dict[str, SkillAttributes], + merged_skills: list[str], + ) -> dict[str, int]: + merged: dict[str, int] = dict(existing_attrs) + + for skill, attrs in extracted_attrs.items(): + key = self._normalize_skill(skill) + if not key: + continue + key = key.casefold() + if key: + merged[key] = max(1, min(5, int(attrs.strength))) + + # Ensure every merged skill has a structured strength so attrs never shrink + # to a partial subset when extraction omitted some per-skill scores. + for raw_skill in merged_skills: + canonical = self._normalize_skill(raw_skill) + if not canonical: + continue + key = canonical.casefold() + if key not in merged: + merged[key] = DEFAULT_SKILL_STRENGTH + + return merged + + def _coerce_skill_attrs_updates(self, value: Any) -> str | None: + if value is None: + return None + + candidate = value + if isinstance(value, str): + raw = value.strip() + if not raw: + return None + candidate = self._decode_json_like(raw) + if candidate is None: + return None + + if not isinstance(candidate, dict): + return None + + parsed: dict[str, int] = {} + for raw_skill, raw_payload in candidate.items(): + skill = self._normalize_skill(raw_skill) + if not skill: + continue + strength_source = raw_payload + if isinstance(raw_payload, dict): + strength_source = raw_payload.get("strength") + try: + strength = int(float(strength_source)) + except Exception: + continue + if not 1 <= strength <= 5: + continue + parsed[skill] = strength + + return self._serialize_skill_attrs(parsed) + + @staticmethod + def _decode_json_like(raw: str) -> Any: + """Decode JSON-like strings, including double-encoded and repr payloads.""" + parsed: Any + try: + parsed = json.loads(raw) + except Exception: + try: + parsed = ast.literal_eval(raw) + except Exception: + return None + + if isinstance(parsed, str): + nested = parsed.strip() + if not nested: + return None + try: + reparsed = json.loads(nested) + except Exception: + try: + reparsed = ast.literal_eval(nested) + except Exception: + return parsed + return reparsed + + return parsed + + def _serialize_skill_attrs(self, attrs: dict[str, int]) -> str | None: + normalized: dict[str, dict[str, int]] = {} + for raw_skill, raw_strength in attrs.items(): + skill = self._normalize_skill(raw_skill) + if not skill: + continue + try: + strength = int(float(raw_strength)) + except Exception: + continue + clamped = max(1, min(5, strength)) + normalized[skill] = {"strength": clamped} + if not normalized: + return None + return json.dumps(normalized, sort_keys=True, separators=(",", ":")) + + def _coerce_website_links(self, value: Any) -> list[str]: + if value is None: + return [] + + if isinstance(value, str): + raw_values = [item for item in value.split(",") if item.strip()] + elif isinstance(value, (list, tuple, set)): + raw_values = list(value) + else: + return [] + + normalized: list[str] = [] + seen: set[str] = set() + for raw_value in raw_values: + if not isinstance(raw_value, str): + continue + normalized_link = self._normalize_website_url(raw_value.strip()) + if normalized_link is None: + continue + dedupe_key = normalized_website_identity_key(normalized_link) + if dedupe_key is None or dedupe_key in seen: + continue + seen.add(dedupe_key) + normalized.append(normalized_link) + + return normalized + + def _merge_website_links( + self, *, existing: list[str], extracted: list[str] + ) -> list[str]: + merged: list[str] = [] + seen: set[str] = set() + + for value in existing: + if not isinstance(value, str): + continue + normalized = self._normalize_website_url(value) + if not normalized: + continue + dedupe_key = normalized_website_identity_key(normalized) + if dedupe_key is None or dedupe_key in seen: + continue + seen.add(dedupe_key) + merged.append(normalized) + + for value in extracted: + if not isinstance(value, str): + continue + normalized = self._normalize_website_url(value) + if not normalized: + continue + dedupe_key = normalized_website_identity_key(normalized) + if dedupe_key is None or dedupe_key in seen: + continue + seen.add(dedupe_key) + merged.append(normalized) + + return merged + + @staticmethod + def _normalize_website_url(value: str) -> str | None: + return normalize_website_url(value, allow_scheme_less=True) + + def _normalize_email_address(self, value: Any) -> str | None: + if not isinstance(value, str): + return None + normalized = value.strip().lower() + if not normalized or "@" not in normalized: + return None + return normalized + + @staticmethod + def _coerce_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + + def _coerce_additional_emails(self, value: Any) -> list[str]: + if value is None: + return [] + + if isinstance(value, str): + raw_value = value.strip() + if not raw_value: + return [] + try: + parsed = json.loads(raw_value) + except Exception: + parsed = [raw_value] + else: + parsed = parsed if isinstance(parsed, list) else [parsed] + elif isinstance(value, (list, tuple, set)): + parsed = list(value) + else: + parsed = [value] + + deduped: list[str] = [] + seen: set[str] = set() + for item in parsed: + normalized = self._normalize_email_address(item) + if normalized is None: + continue + if normalized in seen: + continue + seen.add(normalized) + deduped.append(normalized) + return deduped + + def _coerce_email_address_data(self, value: Any) -> list[dict[str, Any]] | None: + parsed = self._parse_email_address_data(value) + return parsed if parsed else None + + def _parse_email_address_data(self, value: Any) -> list[dict[str, Any]]: + if value is None: + return [] + + candidate = value + if isinstance(value, str): + raw = value.strip() + if not raw: + return [] + try: + candidate = json.loads(raw) + except Exception: + return [] + + if not isinstance(candidate, list): + if isinstance(candidate, dict): + candidate = [candidate] + else: + return [] + + parsed: list[dict[str, Any]] = [] + for entry in candidate: + if not isinstance(entry, dict): + continue + raw_email = str( + entry.get("lower") + or entry.get("emailAddress") + or entry.get("email") + or "" + ).strip() + normalized_email = self._normalize_email_address(raw_email) + if normalized_email is None: + continue + + parsed.append( + { + "emailAddress": str( + entry.get("emailAddress", normalized_email) + ).strip(), + "lower": normalized_email, + "primary": self._coerce_bool(entry.get("primary")), + "optOut": self._coerce_bool(entry.get("optOut")), + "invalid": self._coerce_bool(entry.get("invalid")), + } + ) + + return parsed + + def _build_email_address_data( + self, + *, + email_candidate: str | None, + additional_emails: list[str] | None = None, + existing_email_data: Any, + ) -> list[dict[str, Any]]: + merged: dict[str, dict[str, Any]] = {} + + for entry in self._parse_email_address_data(existing_email_data): + merged[entry["lower"]] = entry + merged[entry["lower"]]["emailAddress"] = ( + str(entry.get("emailAddress", entry["lower"])).strip().lower() + ) + + candidate_lower = self._normalize_email_address(email_candidate) + if candidate_lower is None: + if not additional_emails: + return list(merged.values()) + candidate_lower = None + + if candidate_lower is not None: + for entry in merged.values(): + entry["primary"] = False + merged[candidate_lower] = { + "emailAddress": candidate_lower, + "lower": candidate_lower, + "primary": True, + "optOut": False, + "invalid": False, + } + + for extra in additional_emails or []: + normalized_extra = self._normalize_email_address(extra) + if normalized_extra is None or normalized_extra == candidate_lower: + continue + if normalized_extra in merged: + if candidate_lower is not None: + merged[normalized_extra]["primary"] = False + continue + merged[normalized_extra] = { + "emailAddress": normalized_extra, + "lower": normalized_extra, + "primary": False, + "optOut": False, + "invalid": False, + } + + return list(merged.values()) + + def _configured_model_name(self) -> str: + """Model identity used for idempotency/ledger keys.""" + if self.config.openai_api_key: + return self.config.resume_model + return "heuristic" + + def _record_processing_run( + self, + *, + contact_id: str, + attachment_id: str, + content_hash: str | None, + model_name: str, + status: str, + last_error: str | None = None, + ) -> None: + """Persist one processing result keyed by contact+attachment+version+model.""" + query = """ + INSERT INTO resume_processing_runs ( + contact_id, + attachment_id, + content_hash, + extractor_version, + model_name, + status, + last_error, + processed_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, NOW()) + ON CONFLICT (contact_id, attachment_id, extractor_version, model_name) + DO UPDATE SET + content_hash = EXCLUDED.content_hash, + status = EXCLUDED.status, + last_error = EXCLUDED.last_error, + processed_at = NOW(); + """ + if not self.config.postgres_url: + return + try: + with connect(self.config.postgres_url) as conn: + with conn.cursor() as cursor: + cursor.execute( + query, + ( + contact_id, + attachment_id, + content_hash, + self.config.resume_extractor_version, + model_name, + status, + last_error, + ), + ) + except Exception as exc: + logger.warning( + "Failed to persist resume processing run contact_id=%s attachment_id=%s " + "version=%s model=%s status=%s error=%s", + contact_id, + attachment_id, + self.config.resume_extractor_version, + model_name, + status, + exc, + ) diff --git a/packages/shared/src/five08/resume_skills_extractor.py b/packages/shared/src/five08/resume_skills_extractor.py new file mode 100644 index 00000000..b546623c --- /dev/null +++ b/packages/shared/src/five08/resume_skills_extractor.py @@ -0,0 +1,216 @@ +"""Skills extraction from resume text.""" + +import json +import logging +import re +from typing import Any + +from five08.skills import ( + DISALLOWED_RESUME_SKILLS, + normalize_skill, + normalize_skill_payload, +) +from five08.resume_processing_models import ExtractedSkills, SkillAttributes + +logger = logging.getLogger(__name__) + +try: # pragma: no cover - import success depends on installed dependencies + from openai import OpenAI as OpenAIClient +except Exception: # pragma: no cover + OpenAIClient = None # type: ignore[misc,assignment] + +COMMON_SKILLS = { + "python", + "javascript", + "typescript", + "java", + "go", + "rust", + "node", + "docker", + "kubernetes", + "amazon web services", + "google cloud", + "azure", + "postgresql", + "mysql", + "redis", + "react", + "django", + "flask", + "fastapi", + "git", + "linux", + "product management", + "go to market", + "ab testing", + "search engine optimization", + "search engine marketing", + "customer relationship management", + "google analytics", + "product marketing", + "content marketing", +} + +DISALLOWED_SKILLS = DISALLOWED_RESUME_SKILLS + +DEFAULT_SKILL_STRENGTH = 3 + + +class SkillsExtractor: + """Extract skills with LLM when configured, fallback heuristics otherwise.""" + + def __init__( + self, + *, + model: str, + openai_api_key: str | None, + openai_base_url: str | None, + ) -> None: + self.model = model + self.client: Any = None + + if openai_api_key and OpenAIClient is not None: + self.client = OpenAIClient( + api_key=openai_api_key, + base_url=openai_base_url, + ) + + def extract_skills(self, resume_text: str) -> ExtractedSkills: + """Extract skills from resume text.""" + if self.client is None: + return self._extract_skills_heuristic(resume_text) + + prompt = self._create_prompt(resume_text) + try: + response = self.client.chat.completions.create( + model=self.model, + messages=[ + { + "role": "system", + "content": ( + "You extract professional skills from resumes for a CRM. " + "Focus on white-collar skills for product development orgs: " + "engineering, product, data, design, growth, and marketing. " + "Return JSON only, no prose. " + "Normalize skills to concise canonical names, lowercase. " + "Provide a strength from 1-5 when known, where 5 is strongest. " + "If uncertain, you may omit it or leave it blank. " + "Bias 3 for simple mentions, 4-5 for recent/current project usage, " + "and 1-2 for weak, outdated, or minimal exposure." + ), + }, + {"role": "user", "content": prompt}, + ], + temperature=0.1, + max_tokens=1200, + ) + content = response.choices[0].message.content + if not content: + raise ValueError("LLM returned empty content") + + parsed = self._parse_llm_json(content) + confidence = float(parsed.get("confidence", 0.7)) + return self._normalize_extracted_payload( + skills_value=parsed.get("skills", []), + skill_attrs_value=parsed.get("skill_attrs", {}), + confidence=confidence, + source=self.model, + ) + except Exception as exc: + logger.warning("LLM skills extraction failed, using fallback: %s", exc) + return self._extract_skills_heuristic(resume_text) + + def _extract_skills_heuristic(self, resume_text: str) -> ExtractedSkills: + """Simple keyword and token-based extraction fallback.""" + lowered = resume_text.lower() + token_matches = re.findall(r"\b[a-z][a-z0-9+#\-.]{1,24}\b", lowered) + detected: set[str] = set() + for token in token_matches: + canonical = self._normalize_skill_name(token) + if canonical in COMMON_SKILLS and canonical not in DISALLOWED_SKILLS: + detected.add(canonical) + for skill in COMMON_SKILLS: + if " " not in skill or skill in DISALLOWED_SKILLS: + continue + if re.search(rf"\b{re.escape(skill)}\b", lowered): + detected.add(skill) + + sorted_skills = sorted(detected) + return ExtractedSkills( + skills=sorted_skills, + skill_attrs={ + skill: SkillAttributes(strength=DEFAULT_SKILL_STRENGTH) + for skill in sorted_skills + }, + confidence=0.45 if sorted_skills else 0.2, + source="heuristic", + ) + + def _create_prompt(self, resume_text: str) -> str: + """Prompt template for LLM extraction.""" + snippet = resume_text[:8000] + return ( + "Analyze the resume and extract a concise skill list.\n" + "Use white-collar/product-development relevance only: engineering, product, " + "data, design, growth, and marketing.\n" + "Exclude personal traits and vague soft skills unless role-critical.\n" + "Return JSON with this exact schema:\n" + '{"skills": ["skill1", "skill2", "skill3 (4)"], ' + '"confidence": 0.8}\n' + "Rules:\n" + "- skills must be lowercase canonical names with minimal punctuation\n" + '- prefer forms like "nodejs", "ab testing", "go to market"\n' + "- optional strength may be included inline for a skill in parentheses, e.g. skill (4)\n" + "- if strength is uncertain, omit the suffix or use an empty suffix, e.g. skill ()\n" + "- strength is integer 1-5 (5 strongest), and should be assigned per above.\n" + "- use 3 when a skill is simply mentioned without strong context\n" + "- use 4 or 5 when usage is clearly current or recent in project work\n" + "- use 2 for older, side, or weak mentions and 1 for very weak/outdated evidence\n" + "- no extra keys\n\n" + f"Resume:\n{snippet}" + ) + + def _parse_llm_json(self, content: str) -> dict[str, Any]: + raw = content.strip() + if raw.startswith("```"): + lines = [line for line in raw.splitlines() if not line.startswith("```")] + raw = "\n".join(lines).strip() + + parsed = json.loads(raw) + if not isinstance(parsed, dict): + raise ValueError("skills extraction output was not a JSON object") + return parsed + + def _normalize_extracted_payload( + self, + *, + skills_value: Any, + skill_attrs_value: Any, + confidence: float, + source: str, + ) -> ExtractedSkills: + deduped_skills, normalized_attrs = normalize_skill_payload( + skills_value=skills_value, + skill_attrs_value=skill_attrs_value, + disallowed=DISALLOWED_SKILLS, + ) + ordered_skills = sorted(deduped_skills) + attrs_map = { + skill: SkillAttributes(strength=strength) + for skill, strength in normalized_attrs.items() + } + + return ExtractedSkills( + skills=ordered_skills, + skill_attrs=attrs_map, + confidence=max(0.0, min(1.0, confidence)), + source=source, + ) + + def _normalize_skill_name(self, value: str) -> str: + return normalize_skill(value) + + def canonicalize_skill(self, value: str) -> str: + """Public helper for consistent skill normalization across processors.""" + return self._normalize_skill_name(value) diff --git a/tests/unit/test_crm.py b/tests/unit/test_crm.py index c0a24ad5..f07cd110 100644 --- a/tests/unit/test_crm.py +++ b/tests/unit/test_crm.py @@ -353,6 +353,41 @@ def test_format_seniority_label(self, raw, expected): """Seniority labels should normalize consistent display strings.""" assert _format_seniority_label(raw) == expected + @pytest.mark.asyncio + async def test_resume_apply_confirmation_shows_partial_warning( + self, crm_cog, mock_interaction + ): + """Successful partial applies should render any returned warning.""" + mock_interaction.message = None + crm_cog._apply_resume_profile_direct = AsyncMock( + return_value={ + "success": True, + "updated_fields": ["cGitHubUsername"], + "updated_values": {"cGitHubUsername": "wumichaelm"}, + "warning": "phoneNumber: phone rejected", + } + ) + + view = ResumeUpdateConfirmationView( + crm_cog=crm_cog, + requester_id=123, + contact_id="contact-1", + contact_name="Test User", + proposed_updates={"cGitHubUsername": "wumichaelm"}, + ) + button = next( + child + for child in view.children + if isinstance(child, discord.ui.Button) and child.label == "Confirm Updates" + ) + + await button.callback(mock_interaction) + + final_send = mock_interaction.followup.send.call_args_list[-1] + embed = final_send.kwargs["embed"] + warning_field = next(field for field in embed.fields if field.name == "Warning") + assert warning_field.value == "phoneNumber: phone rejected" + @pytest.mark.parametrize( ("payload", "expected"), [ @@ -2482,8 +2517,8 @@ async def test_assign_onboarder_success_keeps_state_when_not_pending( { "id": "contact123", "name": "John Doe", - "cOnboardingCoordinator": "old", - "cOnboardingStatus": "onboarded", + "cOnboarder": "old", + "cOnboardingState": "onboarded", }, {"id": "contact123"}, ] @@ -2493,7 +2528,7 @@ async def test_assign_onboarder_success_keeps_state_when_not_pending( ) payload = crm_cog.espo_api.request.call_args_list[1][0][2] - assert payload == {"cOnboardingCoordinator": "jane"} + assert payload == {"cOnboarder": "jane"} message = mock_interaction.followup.send.call_args[0][0] assert "onboarding state left unchanged" in message @@ -2621,7 +2656,7 @@ async def test_assign_onboarder_missing_onboarder_field_records_error( crm_cog.espo_api.request.assert_called_once_with("GET", "Contact/contact123") message = mock_interaction.followup.send.call_args[0][0] assert ( - "Could not locate a known onboarder field for this CRM contact." in message + "Could not locate the `cOnboarder` field for this CRM contact." in message ) crm_cog._audit_command.assert_called_once() audit_kwargs = crm_cog._audit_command.call_args.kwargs @@ -2675,13 +2710,13 @@ async def test_view_onboarding_queue_lists_open_entries( "cOnboarder": "mentorA", "type": "Member", }, - {"id": "c2", "name": "Bob", "cOnboardingStatus": "onboarded"}, - {"id": "c3", "name": "Cara", "cOnboarding": "waitlist"}, - {"id": "c4", "name": "Drew", "cOnboarding": "rejected"}, + {"id": "c2", "name": "Bob", "cOnboardingState": "onboarded"}, + {"id": "c3", "name": "Cara", "cOnboardingState": "waitlist"}, + {"id": "c4", "name": "Drew", "cOnboardingState": "rejected"}, { "id": "c5", "name": "Eli", - "cOnboardingStatus": "", + "cOnboardingState": "", "type": "Candidate / Member", }, ] @@ -2696,8 +2731,7 @@ async def test_view_onboarding_queue_lists_open_entries( "maxSize": 200, "select": ( "id,name,emailAddress,cDiscordUsername,cDiscordUserID," - "cOnboardingState,cOnboardingStatus,cOnboarding," - "cOnboarder,cOnboardingCoordinator,cOnboardingUpdatedAt" + "cOnboardingState,cOnboarder,cOnboardingUpdatedAt" ), }, ) @@ -2737,7 +2771,7 @@ async def test_view_onboarding_queue_empty_when_only_excluded( "list": [ {"id": "c1", "name": "Bob", "cOnboardingState": "onboarded"}, {"id": "c2", "name": "Cara", "cOnboardingState": "waitlist"}, - {"id": "c3", "name": "Drew", "cOnboarding": "rejected"}, + {"id": "c3", "name": "Drew", "cOnboardingState": "rejected"}, ] } @@ -2750,8 +2784,7 @@ async def test_view_onboarding_queue_empty_when_only_excluded( "maxSize": 200, "select": ( "id,name,emailAddress,cDiscordUsername,cDiscordUserID," - "cOnboardingState,cOnboardingStatus,cOnboarding," - "cOnboarder,cOnboardingCoordinator,cOnboardingUpdatedAt" + "cOnboardingState,cOnboarder,cOnboardingUpdatedAt" ), }, ) @@ -2808,18 +2841,14 @@ async def test_view_onboarding_queue_uses_pagination_for_large_queues( assert "view" in send_kwargs assert send_kwargs["view"].total_pages > 1 - def test_format_onboarding_updated_at_normalizes_timezone(self, crm_cog): - """Timestamps should be normalized consistently for display.""" - assert crm_cog._format_onboarding_updated_at(0) == "1970-01-01 00:00 UTC" - assert ( - crm_cog._format_onboarding_updated_at("2026-03-03T12:00:00-05:00") - == "2026-03-03 17:00 UTC" - ) - assert crm_cog._format_onboarding_updated_at("2026-03-03T12:00:00") == ( - "2026-03-03 12:00" + def test_format_onboarding_updated_at_uses_espo_formats(self, crm_cog): + """Espo date and datetime values should be formatted directly.""" + assert crm_cog._format_onboarding_updated_at("2026-03-03 12:00:00") == ( + "2026-03-03 12:00 UTC" ) - assert crm_cog._format_onboarding_updated_at("2026-03-03T00:00:00Z") == ( - "2026-03-03" + assert crm_cog._format_onboarding_updated_at("2026-03-03") == "2026-03-03" + assert crm_cog._format_onboarding_updated_at("2026-03-03T12:00:00Z") == ( + "2026-03-03T12:00:00Z" ) @pytest.mark.asyncio @@ -3672,24 +3701,19 @@ async def test_search_contacts_by_field_includes_requested_field_and_excludes_de """Search-by-field includes the requested field and excludes the default.""" crm_cog.espo_api.request.return_value = {"list": []} - with patch.object( - crm_cog, "_configured_linkedin_field", return_value="cLinkedIn" - ) as configured_field: - configured_linkedin_field = crm_cog._configured_linkedin_field() - await crm_cog._search_contacts_by_field( - field=configured_linkedin_field, value="https://linkedin.com/in/test" - ) + await crm_cog._search_contacts_by_field( + field="cLinkedIn", value="https://linkedin.com/in/test" + ) call = crm_cog.espo_api.request.call_args assert call.args[0] == "GET" assert call.args[1] == "Contact" params = call.args[2] - configured_field.assert_called_once() - assert params["where"][0]["attribute"] == configured_linkedin_field + assert params["where"][0]["attribute"] == "cLinkedIn" assert params["where"][0]["value"] == "https://linkedin.com/in/test" assert params["where"][0]["type"] == "equals" select_fields = params["select"].split(",") - assert configured_linkedin_field in select_fields + assert "cLinkedIn" in select_fields assert "cLinkedInUrl" not in select_fields @pytest.mark.asyncio @@ -4073,21 +4097,14 @@ async def test_update_contact_requires_updates(self, crm_cog, mock_interaction): assert "Provide at least one of" in message @pytest.mark.asyncio - async def test_update_contact_uses_configured_linkedin_field( - self, crm_cog, mock_interaction - ): - """Configured LinkedIn custom field should flow through update payload and embed.""" + async def test_update_contact_uses_clinkedin_field(self, crm_cog, mock_interaction): + """LinkedIn updates should flow through the cLinkedIn field and embed.""" mock_interaction.user.id = 123456789 - with ( - patch.object( - crm_cog, "_configured_linkedin_field", return_value="cLinkedIn" - ), - patch.object( - crm_cog, - "_find_contact_by_discord_id", - new=AsyncMock(return_value={"id": "contact123", "name": "Test User"}), - ), + with patch.object( + crm_cog, + "_find_contact_by_discord_id", + new=AsyncMock(return_value={"id": "contact123", "name": "Test User"}), ): crm_cog.espo_api.request.return_value = {"id": "contact123"} @@ -4159,6 +4176,45 @@ async def test_update_contact_upload_resume_only(self, crm_cog, mock_interaction assert kwargs["contact"]["id"] == "contact123" assert kwargs["target_scope"] == "self" + @pytest.mark.asyncio + async def test_update_contact_resume_rejects_txt_file( + self, crm_cog, mock_interaction + ): + """Resume uploads should reject TXT files.""" + mock_interaction.user.id = 123456789 + resume_file = Mock() + resume_file.filename = "resume.txt" + resume_file.size = 1024 + + crm_cog._audit_command = Mock() + + with ( + patch( + "five08.discord_bot.cogs.crm.settings.api_shared_secret", + "test-shared-secret", + ), + patch.object( + crm_cog, + "_upload_resume_attachment_to_contact", + new=AsyncMock(), + ) as mock_upload, + ): + await crm_cog.update_contact.callback( + crm_cog, mock_interaction, resume=resume_file + ) + + mock_upload.assert_not_awaited() + message = mock_interaction.followup.send.call_args.args[0] + assert "Invalid file type. Upload a PDF or DOCX file." in message + crm_cog._audit_command.assert_called_once() + audit_kwargs = crm_cog._audit_command.call_args.kwargs + assert audit_kwargs["action"] == "crm.update_contact" + assert audit_kwargs["result"] == "denied" + assert audit_kwargs["metadata"] == { + "filename": "resume.txt", + "reason": "invalid_file_type", + } + @pytest.mark.asyncio async def test_update_contact_unexpected_exception(self, crm_cog, mock_interaction): """Unexpected exceptions should return a useful message.""" @@ -4389,15 +4445,16 @@ async def test_check_existing_resume_api_error(self, crm_cog, mock_interaction): def test_build_resume_create_contact_payload_sets_email_field_by_domain( self, crm_cog ): - """Test that resume payload writes either emailAddress or c508Email.""" + """Payload should prefer one non-508 email when available.""" with ( patch.object( crm_cog, "_extract_resume_contact_hints", return_value={ - "emails": ["person@example.com"], - "github_usernames": [], - "linkedin_urls": [], + "emails": ["person@508.dev", "person@example.com"], + "github_usernames": ["personhub"], + "linkedin_urls": ["https://linkedin.com/in/person"], + "phone": "+1 555-0100", }, ), patch.object( @@ -4411,15 +4468,21 @@ def test_build_resume_create_contact_payload_sets_email_field_by_domain( assert payload["firstName"] == "Person" assert payload["lastName"] == "Example" assert "c508Email" not in payload + assert "cGitHubUsername" not in payload + assert "cLinkedIn" not in payload + assert "phoneNumber" not in payload + def test_build_resume_create_contact_payload_falls_back_to_508_email(self, crm_cog): + """When only 508 email exists, keep just that single identifier.""" with ( patch.object( crm_cog, "_extract_resume_contact_hints", return_value={ "emails": ["person@508.dev"], - "github_usernames": [], - "linkedin_urls": [], + "github_usernames": ["personhub"], + "linkedin_urls": ["https://linkedin.com/in/person508"], + "phone": "+1 555-0100", }, ), patch.object( @@ -4433,23 +4496,27 @@ def test_build_resume_create_contact_payload_sets_email_field_by_domain( assert payload["firstName"] == "Person" assert payload["lastName"] == "Unknown" assert "emailAddress" not in payload + assert "cGitHubUsername" not in payload + assert "cLinkedIn" not in payload + assert "phoneNumber" not in payload - def test_build_resume_create_contact_payload_populates_prospect_details( + def test_build_resume_create_contact_payload_uses_first_non_email_identifier( self, crm_cog ): - """Test creating prospect payload includes richer parsed fields.""" + """If no email exists, choose exactly one fallback identifier in priority order.""" with ( patch.object( crm_cog, "_extract_resume_contact_hints", return_value={ - "emails": ["jane@example.com"], - "github_usernames": ["janedoe"], + "emails": [], + "github_usernames": [" janedoe ", "backupuser"], "linkedin_urls": ["https://linkedin.com/in/janedoe"], "phone": "+1 555-0100", "address_country": "Canada", "seniority_level": "senior", "skills": ["Python", " fastapi ", ""], + "description": "Senior backend engineer", }, ), patch.object(crm_cog, "_extract_resume_name_hint", return_value="Jane Doe"), @@ -4457,15 +4524,19 @@ def test_build_resume_create_contact_payload_populates_prospect_details( payload = crm_cog._build_resume_create_contact_payload(b"resume") assert payload["type"] == "Prospect" assert payload["name"] == "Jane Doe" - assert payload["emailAddress"] == "jane@example.com" assert payload["cGitHubUsername"] == "janedoe" - assert payload["cLinkedIn"] == "https://linkedin.com/in/janedoe" - assert payload["phoneNumber"] == "+1 555-0100" - assert payload["addressCountry"] == "Canada" - assert payload["cSeniority"] == "senior" - assert payload["skills"] == ["Python", "fastapi"] assert payload["firstName"] == "Jane" assert payload["lastName"] == "Doe" + assert "emailAddress" not in payload + assert "c508Email" not in payload + assert "cLinkedIn" not in payload + assert "phoneNumber" not in payload + assert "addressCountry" not in payload + assert "cTimezone" not in payload + assert "addressCity" not in payload + assert "cSeniority" not in payload + assert "description" not in payload + assert "skills" not in payload def test_build_resume_create_contact_payload_single_name_uses_unknown_last( self, crm_cog @@ -4562,10 +4633,12 @@ def test_build_inference_lookup_summary_falls_back_to_parsed_identifiers( """Test lookup summary fallback uses parsed identifiers with cleanup.""" with ( patch.object(crm_cog, "_format_inferred_attempts", return_value=""), - patch.object( - crm_cog, - "_extract_resume_contact_hints", - return_value={ + patch.object(crm_cog, "_extract_resume_contact_hints") as extract_hints, + ): + summary = crm_cog._build_inference_lookup_summary( + file_content=b"resume", + attempts=[], + hints={ "emails": [ " jane@example.com ", "jane@example.com", @@ -4579,10 +4652,6 @@ def test_build_inference_lookup_summary_falls_back_to_parsed_identifiers( "https://linkedin.com/in/jane", ], }, - ), - ): - summary = crm_cog._build_inference_lookup_summary( - file_content=b"resume", attempts=[] ) assert ( @@ -4591,15 +4660,15 @@ def test_build_inference_lookup_summary_falls_back_to_parsed_identifiers( + "emails: `jane@example.com`, `second@example.com`; " + "github usernames: `janedoe`; linkedin URLs: `https://linkedin.com/in/jane`" ) + extract_hints.assert_not_called() - def test_build_inference_lookup_summary_with_non_dict_hints(self, crm_cog): - """Test non-dict parsed contact hints produce empty summary safely.""" + def test_build_inference_lookup_summary_with_empty_hints(self, crm_cog): + """Empty hints should produce an empty lookup summary.""" with ( patch.object(crm_cog, "_format_inferred_attempts", return_value=""), - patch.object(crm_cog, "_extract_resume_contact_hints", return_value=None), ): summary = crm_cog._build_inference_lookup_summary( - file_content=b"resume", attempts=[] + file_content=b"resume", attempts=[], hints={} ) assert summary == "" @@ -4608,22 +4677,20 @@ def test_build_resume_parsed_identity_summary_includes_name_and_email( self, crm_cog ): """Parsed name and email are included in resume identity summary.""" - with patch.object( - crm_cog, - "_extract_resume_contact_hints", - return_value={ - "name": "Jane Doe", - "emails": ["jane@example.com", "ignored@alt.example"], - }, - ): + with patch.object(crm_cog, "_extract_resume_contact_hints") as extract_hints: summary = crm_cog._build_resume_parsed_identity_summary( - file_content=b"resume" + file_content=b"resume", + hints={ + "name": "Jane Doe", + "emails": ["jane@example.com", "ignored@alt.example"], + }, ) assert ( summary == "\nParsed contact details: name=`Jane Doe`, email=`jane@example.com`" ) + extract_hints.assert_not_called() def test_build_resume_parsed_identity_summary_ignores_heading_name(self, crm_cog): """Heading-like parsed names should fall back to heuristic name extraction.""" @@ -4887,7 +4954,7 @@ async def test_upload_resume_self_not_linked_records_error( async def test_upload_resume_invalid_file_type_records_error( self, crm_cog, mock_interaction ): - """Uploading non-PDF/DOC/DOCX/TXT files should be recorded as an error.""" + """Uploading unsupported file types should be recorded as an error.""" mock_interaction.user.id = 101 resume_file = Mock() @@ -4910,9 +4977,7 @@ async def test_upload_resume_invalid_file_type_records_error( ) message = mock_interaction.followup.send.call_args[0][0] - assert ( - "Invalid file type. Please upload a PDF, DOC, DOCX, or TXT file." in message - ) + assert "Invalid file type. Upload a PDF or DOCX file." in message crm_cog._audit_command.assert_called_once() audit_kwargs = crm_cog._audit_command.call_args.kwargs assert audit_kwargs["action"] == "crm.upload_resume" @@ -4922,6 +4987,37 @@ async def test_upload_resume_invalid_file_type_records_error( "reason": "invalid_file_type", } + @pytest.mark.asyncio + async def test_upload_resume_rejects_doc_file(self, crm_cog, mock_interaction): + """DOC files should be rejected when they are not parser-supported.""" + mock_interaction.user.id = 101 + + resume_file = Mock() + resume_file.filename = "candidate.doc" + resume_file.size = 1024 + + crm_cog._audit_command = Mock() + + await crm_cog.upload_resume.callback( + crm_cog, + mock_interaction, + resume_file, + None, + False, + None, + ) + + message = mock_interaction.followup.send.call_args[0][0] + assert "Invalid file type. Upload a PDF or DOCX file." in message + crm_cog._audit_command.assert_called_once() + audit_kwargs = crm_cog._audit_command.call_args.kwargs + assert audit_kwargs["action"] == "crm.upload_resume" + assert audit_kwargs["result"] == "error" + assert audit_kwargs["metadata"] == { + "filename": "candidate.doc", + "reason": "invalid_file_type", + } + @pytest.mark.asyncio async def test_upload_resume_file_too_large_records_error( self, crm_cog, mock_interaction @@ -5146,6 +5242,79 @@ async def test_upload_resume_no_matching_inferred_contact_shows_name_and_email( ) assert "view" in mock_interaction.followup.send.call_args.kwargs + @pytest.mark.asyncio + async def test_upload_resume_reuses_inferred_hints_in_failure_summaries( + self, crm_cog, mock_interaction + ): + """Inference failure UI should pass through already-parsed hints.""" + mock_interaction.user.id = 101 + mock_interaction.user.name = "Requester" + steering_role = Mock() + steering_role.name = "Steering Committee" + mock_interaction.user.roles = [steering_role] + + resume_file = Mock() + resume_file.filename = "candidate.pdf" + resume_file.size = 1024 + resume_file.read = AsyncMock(return_value=b"resume-bytes") + + inferred_hints = { + "name": "Jane Doe", + "emails": ["jane@example.com"], + } + crm_cog._build_inference_lookup_summary_async = AsyncMock(return_value="") + crm_cog._build_resume_parsed_identity_summary_async = AsyncMock( + return_value="\nParsed contact details: name=`Jane Doe`, email=`jane@example.com`" + ) + + with ( + patch.object( + crm_cog, + "_infer_contact_from_resume", + new=AsyncMock( + return_value=( + None, + { + "reason": "no_matching_contact", + "hints": inferred_hints, + }, + ) + ), + ), + patch.object( + crm_cog, + "_find_contact_by_discord_id", + new=AsyncMock(return_value=None), + ), + patch( + "five08.discord_bot.cogs.crm.check_user_roles_with_hierarchy", + return_value=True, + ), + patch( + "five08.discord_bot.cogs.crm.settings.api_shared_secret", + "test-shared-secret", + ), + ): + await crm_cog.upload_resume.callback( + crm_cog, + mock_interaction, + resume_file, + None, + False, + None, + ) + + assert ( + crm_cog._build_inference_lookup_summary_async.await_args.kwargs["hints"] + == inferred_hints + ) + assert ( + crm_cog._build_resume_parsed_identity_summary_async.await_args.kwargs[ + "hints" + ] + == inferred_hints + ) + @pytest.mark.asyncio async def test_resume_create_contact_view_logs_create_failure( self, crm_cog, mock_interaction @@ -5587,37 +5756,32 @@ async def test_reprocess_confirmation_view_calls_reprocess_preview( ) @pytest.mark.asyncio - async def test_run_resume_extract_and_preview_uses_refresh_token_for_reprocess( + async def test_run_resume_extract_and_preview_calls_direct_extract_for_reprocess( self, crm_cog, mock_interaction ): - """Explicit reprocess actions should bypass cached extract jobs.""" - crm_cog._enqueue_resume_extract_job = AsyncMock(return_value="job-123") - crm_cog._wait_for_backend_job_result = AsyncMock( - return_value={ - "status": "succeeded", - "result": {"success": False, "error": "boom"}, - } + """Reprocess should call the direct extract path with the latest attachment.""" + crm_cog._extract_resume_profile_direct = AsyncMock( + return_value={"success": False, "error": "boom"} ) crm_cog._build_resume_extract_debug_file = Mock(return_value=Mock()) crm_cog._audit_command = Mock() - with patch( - "five08.discord_bot.cogs.crm.uuid4", - return_value=Mock(hex="refresh-token-123"), - ): - await crm_cog._run_resume_extract_and_preview( - mock_interaction, - contact_id="contact123", - contact_name="Candidate User", - attachment_id="resume123", - filename="candidate.pdf", - link_member=None, - action="crm.reprocess_resume", - status_message="🔄 Reprocessing resume and extracting profile fields now...", - ) + await crm_cog._run_resume_extract_and_preview( + mock_interaction, + contact_id="contact123", + contact_name="Candidate User", + attachment_id="resume123", + filename="candidate.pdf", + link_member=None, + action="crm.reprocess_resume", + status_message="🔄 Reprocessing resume and extracting profile fields now...", + ) - kwargs = crm_cog._enqueue_resume_extract_job.await_args.kwargs - assert kwargs["refresh_token"] == "refresh-token-123" + crm_cog._extract_resume_profile_direct.assert_awaited_once_with( + contact_id="contact123", + attachment_id="resume123", + filename="candidate.pdf", + ) @pytest.mark.asyncio async def test_build_match_candidates_posting_fetches_jd_links_from_text( diff --git a/tests/unit/test_resume_profile_processor.py b/tests/unit/test_resume_profile_processor.py index 9b662454..730ac1fc 100644 --- a/tests/unit/test_resume_profile_processor.py +++ b/tests/unit/test_resume_profile_processor.py @@ -2,13 +2,33 @@ import json from datetime import datetime +from types import SimpleNamespace from unittest.mock import Mock +from five08.clients.espo import EspoAPIError +from five08.resume_profile_processor import ResumeProcessorConfig from five08.worker.crm.resume_profile_processor import ResumeProfileProcessor from five08.worker.models import ExtractedSkills, ResumeExtractedProfile +def test_resume_processor_config_filters_unsupported_extensions() -> None: + """Shared config should clamp settings to the supported resume formats.""" + config = ResumeProcessorConfig.from_settings( + SimpleNamespace( + espo_base_url="https://crm.example.com", + espo_api_key="secret", + allowed_file_types="pdf,docx,txt", + max_file_size_mb=12, + ) + ) + + assert config.allowed_file_extensions == {"pdf", "docx"} + assert config.allowed_attachment_suffixes == frozenset({".pdf", ".docx"}) + assert config.allowed_file_extensions_label == "PDF or DOCX" + assert config.max_file_size_bytes == 12 * 1024 * 1024 + + def test_extract_profile_proposal_filters_508_email() -> None: """Extract proposal should skip @508.dev email updates by policy.""" processor = ResumeProfileProcessor() @@ -500,58 +520,6 @@ def test_extract_profile_proposal_normalizes_unknown_seniority_to_unknown() -> N assert result.proposed_updates["cSeniority"] == "unknown" -def test_extract_profile_proposal_uses_configured_linkedin_field( - monkeypatch: object, -) -> None: - """LinkedIn proposal should respect configurable CRM field mapping.""" - processor = ResumeProfileProcessor() - processor.crm = Mock() - processor.extractor = Mock() - processor.skills_extractor = Mock() - processor.document_processor = Mock() - processor._record_processing_run = Mock() - processor.skills_extractor.canonicalize_skill.side_effect = lambda v: ( - str(v).strip().lower() - ) - - monkeypatch.setattr( - "five08.worker.crm.resume_profile_processor.settings.crm_linkedin_field", - "cLinkedInProfile", - ) - - processor.crm.get_contact.return_value = { - "emailAddress": "member@example.com", - "cLinkedInProfile": "https://linkedin.com/in/old", - } - processor.crm.download_attachment.return_value = b"resume-bytes" - processor.document_processor.extract_text.return_value = "resume text" - processor.document_processor.get_content_hash.return_value = "hash-linkedin-config" - processor.extractor.extract.return_value = ResumeExtractedProfile( - email=None, - github_username=None, - linkedin_url="https://linkedin.com/in/new", - phone=None, - confidence=0.9, - source="gpt-4o-mini", - ) - processor.skills_extractor.extract_skills.return_value = ExtractedSkills( - skills=[], - skill_attrs={}, - confidence=0.8, - source="gpt-4o-mini", - ) - - result = processor.extract_profile_proposal( - contact_id="contact-linkedin", - attachment_id="att-linkedin", - filename="resume.pdf", - ) - - assert result.success is True - assert result.proposed_updates["cLinkedInProfile"] == "https://linkedin.com/in/new" - assert "cLinkedIn" not in result.proposed_updates - - def test_apply_profile_updates_appends_resume_email_as_primary_emailAddressData() -> ( None ): @@ -951,46 +919,104 @@ def test_apply_profile_updates_normalizes_unknown_seniority_to_unknown() -> None assert payload["cSeniority"] == "unknown" -def test_apply_profile_updates_allows_configured_linkedin_field( - monkeypatch: object, -) -> None: - """Apply should accept LinkedIn updates using configured CRM field name.""" +def test_apply_profile_updates_normalizes_skill_aliases_for_api_payload() -> None: + """Alias-heavy skills should be normalized into shared canonical forms.""" processor = ResumeProfileProcessor() processor.crm = Mock() - monkeypatch.setattr( - "five08.worker.crm.resume_profile_processor.settings.crm_linkedin_field", - "cLinkedInProfile", + result = processor.apply_profile_updates( + contact_id="contact-6", + updates={ + "skills": ["Node.js", "Node.js", "node"], + "cSkillAttrs": {"Node.js": {"strength": 4}, "node": {"strength": 5}}, + }, ) + assert result.success is True + payload = processor.crm.update_contact.call_args[0][1] + assert payload["skills"] == ["node"] + assert json.loads(payload["cSkillAttrs"]) == {"node": {"strength": 5}} + + +def test_apply_profile_updates_accepts_link_only_updates() -> None: + """Link-only submissions should still persist Discord linkage.""" + processor = ResumeProfileProcessor() + processor.crm = Mock() + result = processor.apply_profile_updates( - contact_id="contact-linkedin", - updates={"cLinkedInProfile": "https://linkedin.com/in/new"}, + contact_id="contact-link-only", + updates={}, + link_discord={"user_id": "123", "username": "member#0001"}, ) assert result.success is True + assert result.link_discord_applied is True + assert result.updated_fields == ["cDiscordUserID", "cDiscordUsername"] payload = processor.crm.update_contact.call_args[0][1] - assert payload["cLinkedInProfile"] == "https://linkedin.com/in/new" - assert "cLinkedIn" not in payload + assert payload["cDiscordUserID"] == "123" + assert payload["cDiscordUsername"] == "member#0001 (ID: 123)" -def test_apply_profile_updates_normalizes_skill_aliases_for_api_payload() -> None: - """Alias-heavy skills should be normalized into shared canonical forms.""" +def test_apply_profile_updates_returns_warning_for_partial_success() -> None: + """Fallback field updates should surface partial failures without dropping successes.""" processor = ResumeProfileProcessor() processor.crm = Mock() + processor._verify_updated_fields = Mock(return_value=["cGitHubUsername"]) + processor.crm.update_contact.side_effect = [ + EspoAPIError("batch failed"), + None, + EspoAPIError("phone rejected"), + None, + ] result = processor.apply_profile_updates( - contact_id="contact-6", + contact_id="contact-partial", updates={ - "skills": ["Node.js", "Node.js", "node"], - "cSkillAttrs": {"Node.js": {"strength": 4}, "node": {"strength": 5}}, + "cGitHubUsername": "new-gh", + "phoneNumber": "14155551234", }, ) assert result.success is True - payload = processor.crm.update_contact.call_args[0][1] - assert payload["skills"] == ["node"] - assert json.loads(payload["cSkillAttrs"]) == {"node": {"strength": 5}} + assert result.updated_fields == ["cGitHubUsername"] + assert result.updated_values == {"cGitHubUsername": "new-gh"} + assert result.warning == "phoneNumber: phone rejected" + assert result.error is None + timestamp_call = processor.crm.update_contact.call_args_list[-1] + assert timestamp_call.args[0] == "contact-partial" + assert set(timestamp_call.args[1].keys()) == {"cResumeLastProcessed"} + assert ( + datetime.strptime( + timestamp_call.args[1]["cResumeLastProcessed"], "%Y-%m-%d %H:%M:%S" + ) + is not None + ) + + +def test_apply_profile_updates_does_not_report_failed_fields_as_updated() -> None: + """Failed writes should not be echoed back as updated fields or values.""" + processor = ResumeProfileProcessor() + processor.crm = Mock() + processor.crm.update_contact.side_effect = [ + EspoAPIError("batch failed"), + EspoAPIError("github rejected"), + EspoAPIError("phone rejected"), + ] + + result = processor.apply_profile_updates( + contact_id="contact-failed", + updates={ + "cGitHubUsername": "new-gh", + "phoneNumber": "14155551234", + }, + ) + + assert result.success is False + assert result.updated_fields == [] + assert result.updated_values == {} + assert ( + result.error == "cGitHubUsername: github rejected; phoneNumber: phone rejected" + ) def test_extract_profile_proposal_records_failed_run() -> None: diff --git a/tests/unit/test_skills_extractor.py b/tests/unit/test_skills_extractor.py index 99b1e907..1b25b51e 100644 --- a/tests/unit/test_skills_extractor.py +++ b/tests/unit/test_skills_extractor.py @@ -1,5 +1,6 @@ """Unit tests for heuristic skills extraction.""" +from five08.resume_skills_extractor import SkillsExtractor as SharedSkillsExtractor from five08.worker.crm.skills_extractor import SkillsExtractor @@ -25,6 +26,22 @@ def test_heuristic_extractor_includes_two_letter_go_skill() -> None: assert "python" in result.skills +def test_shared_heuristic_extractor_detects_multiword_phrases() -> None: + """Shared heuristic fallback should detect plain multi-word skills.""" + extractor = SharedSkillsExtractor( + model="heuristic", + openai_api_key=None, + openai_base_url=None, + ) + result = extractor._extract_skills_heuristic( + "Led product management and go to market planning for a customer relationship management platform." + ) + + assert "product management" in result.skills + assert "go to market" in result.skills + assert "customer relationship management" in result.skills + + def test_normalize_extracted_payload_canonicalizes_and_validates_strength() -> None: """LLM payload normalization should map aliases and ignore out-of-range strengths.""" extractor = SkillsExtractor() diff --git a/tests/unit/test_worker_config.py b/tests/unit/test_worker_config.py index dcee2a87..e350650d 100644 --- a/tests/unit/test_worker_config.py +++ b/tests/unit/test_worker_config.py @@ -168,7 +168,6 @@ def test_fixed_worker_defaults_ignore_legacy_env_vars( espo_api_key="test-key", ) - assert settings.crm_linkedin_field == "cLinkedIn" assert settings.crm_intake_completed_field == "" assert settings.parsed_resume_keywords == {"resume", "cv", "curriculum"} assert settings.oidc_http_timeout_seconds == 8.0