From 62f47b3266db83ee8d777f7b05208da1fc875fe7 Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Thu, 19 Dec 2024 15:07:29 +0530 Subject: [PATCH 1/7] fixes for prompt sudio coverage --- .../output_manager_util.py | 35 +++++++------------ .../document-parser/DocumentParser.jsx | 25 +------------ .../manage-docs-modal/ManageDocsModal.jsx | 27 ++++++++++---- .../prompt-card/PromptCardItems.jsx | 22 ++++++++---- frontend/src/hooks/usePromptOutput.js | 5 +-- 5 files changed, 51 insertions(+), 63 deletions(-) diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py index 4a0352099..6e945d5e1 100644 --- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py +++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py @@ -1,4 +1,3 @@ -from django.db.models import Count from prompt_studio.prompt_studio_output_manager_v2.models import ( PromptStudioOutputManager, ) @@ -11,41 +10,33 @@ def get_coverage( profile_manager_id: str, prompt_id: str = None, is_single_pass: bool = False, - ) -> dict[str, int]: + ) -> dict[str, list[str]]: """ Method to fetch coverage data for given tool and profile manager. Args: - tool (CustomTool): The tool instance or ID for which coverage is fetched. + tool_id (str): The ID of the tool for which coverage is fetched. profile_manager_id (str): The ID of the profile manager for which coverage is calculated. prompt_id (Optional[str]): The ID of the prompt (optional). - is_single_pass (Optional[bool]): Singlepass enabled or not + is_single_pass (Optional[bool]): Singlepass enabled or not. If provided, coverage is fetched for the specific prompt. Returns: - dict[str, int]: A dictionary containing coverage information. + dict[str, list[str]]: A dictionary containing coverage information. Keys are formatted as "coverage__". - Values are the count of documents associated with each prompt + Values are lists of document IDs associated with each prompt and profile combination. """ # TODO: remove singlepass reference - prompt_outputs = ( - PromptStudioOutputManager.objects.filter( - tool_id=tool_id, - profile_manager_id=profile_manager_id, - prompt_id=prompt_id, - is_single_pass_extract=is_single_pass, - ) - .values("prompt_id", "profile_manager_id") - .annotate(document_count=Count("document_manager_id")) - ) + prompt_outputs = PromptStudioOutputManager.objects.filter( + tool_id=tool_id, + profile_manager_id=profile_manager_id, + prompt_id=prompt_id, + is_single_pass_extract=is_single_pass, + ).values("prompt_id", "profile_manager_id", "document_manager_id") - coverage = {} + coverage = [] for prompt_output in prompt_outputs: - prompt_key = str(prompt_output["prompt_id"]) - profile_key = str(prompt_output["profile_manager_id"]) - coverage[f"coverage_{prompt_key}_{profile_key}"] = prompt_output[ - "document_count" - ] + coverage.append(str(prompt_output["document_manager_id"])) return coverage diff --git a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx index b075c7e4e..43f64d111 100644 --- a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx +++ b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx @@ -180,29 +180,6 @@ function DocumentParser({ return outputs; }; - const getPromptCoverageCount = (promptId) => { - const keys = Object.keys(promptOutputs || {}); - const coverageKey = `coverage_${promptId}`; - const outputs = {}; - if (!keys?.length) { - details?.prompts?.forEach((prompt) => { - if (prompt?.coverage) { - const key = Object.keys(prompt?.coverage)[0]; - if (key?.startsWith(coverageKey)) { - outputs[key] = prompt?.coverage[key]; - } - } - }); - return outputs; - } - keys?.forEach((key) => { - if (key?.startsWith(coverageKey)) { - outputs[key] = promptOutputs[key]; - } - }); - return outputs; - }; - if (!details?.prompts?.length) { if (isSimplePromptStudio && SpsPromptsEmptyState) { return ; @@ -230,7 +207,7 @@ function DocumentParser({ outputs={getPromptOutputs(item?.prompt_id)} enforceTypeList={enforceTypeList} setUpdatedPromptsCopy={setUpdatedPromptsCopy} - coverageCountData={getPromptCoverageCount(item?.prompt_id)} + coverageCountData={item?.coverage} isChallenge={isChallenge} />
diff --git a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx index 3d9d53e7d..281ccf05a 100644 --- a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx +++ b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx @@ -33,6 +33,7 @@ import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper"; import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader"; import "./ManageDocsModal.css"; import usePostHogEvents from "../../../hooks/usePostHogEvents"; +import { usePromptOutputStore } from "../../../store/prompt-output-store"; let SummarizeStatusTitle = null; try { @@ -90,6 +91,7 @@ function ManageDocsModal({ const axiosPrivate = useAxiosPrivate(); const handleException = useExceptionHandler(); const { setPostHogCustomEvent } = usePostHogEvents(); + const { promptOutputs, updatePromptOutput } = usePromptOutputStore(); const successIndex = ( @@ -543,21 +545,32 @@ function ManageDocsModal({ ); updateCustomTool({ listOfDocs: newListOfDocs }); - if (newListOfDocs?.length === 1 && selectedDoc?.document_id !== docId) { - const doc = newListOfDocs[1]; + if (selectedDoc?.document_id === docId) { + const doc = newListOfDocs[0]; handleDocChange(doc); } - - if (docId === selectedDoc?.document_id) { - updateCustomTool({ selectedDoc: "" }); - handleUpdateTool({ output: "" }); - } + const updatedPromptOutput = removeIdFromCoverage(promptOutputs, docId); + updatePromptOutput(updatedPromptOutput); }) .catch((err) => { setAlertDetails(handleException(err, "Failed to delete")); }); }; + const removeIdFromCoverage = (data, idToRemove) => { + return Object.entries(data).reduce((updatedData, [key, value]) => { + // Create a new object for the current entry + updatedData[key] = { + ...value, + // Update the coverage array if it exists + coverage: value.coverage + ? value.coverage.filter((id) => id !== idToRemove) + : value.coverage, + }; + return updatedData; + }, {}); + }; + return ( 1; const divRef = useRef(null); const [enforceType, setEnforceType] = useState(""); - const profileId = singlePassExtractMode - ? defaultLlmProfile - : selectedLlmProfileId || defaultLlmProfile; - const coverageKey = generateCoverageKey(promptDetails?.prompt_id, profileId); + const promptId = promptDetails?.prompt_id; + const docId = selectedDoc?.document_id; + const promptProfile = promptDetails?.profile_manager || defaultLlmProfile; + const promptOutputKey = generatePromptOutputKey( + promptId, + docId, + promptProfile, + singlePassExtractMode, + true + ); + const promptCoverage = + promptOutputs[promptOutputKey]?.coverage || coverageCountData; useEffect(() => { if (enforceType !== promptDetails?.enforce_type) { @@ -213,7 +223,7 @@ function PromptCardItems({ )} - Coverage: {coverageCountData[coverageKey] || 0} of{" "} + Coverage: {promptCoverage?.length || 0} of{" "} {listOfDocs?.length || 0} docs diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.js index ad32591e3..5517ec968 100644 --- a/frontend/src/hooks/usePromptOutput.js +++ b/frontend/src/hooks/usePromptOutput.js @@ -91,7 +91,6 @@ const usePromptOutput = () => { let isTokenUsageForSinglePassAdded = false; const tokenUsageDetails = {}; - data.forEach((item) => { const promptId = item?.prompt_id; const docId = item?.document_manager; @@ -109,7 +108,6 @@ const usePromptOutput = () => { isSinglePass, true ); - const coverageKey = `coverage_${item?.prompt_id}_${llmProfile}`; outputs[key] = { runId: item?.run_id, promptOutputId: item?.prompt_output_id, @@ -119,8 +117,8 @@ const usePromptOutput = () => { tokenUsage: item?.token_usage, output: item?.output, timer, + coverage: item?.coverage, }; - outputs[coverageKey] = item?.coverage[coverageKey] || 0; if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded) return; @@ -150,7 +148,6 @@ const usePromptOutput = () => { ); tokenUsageDetails[tokenUsageId] = item?.token_usage; }); - if (isReset) { setPromptOutput(outputs); setTokenUsage(tokenUsageDetails); From b5553b79182c3e9ec8738f9aa88e8593f69431ae Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Thu, 19 Dec 2024 15:08:05 +0530 Subject: [PATCH 2/7] fixed prompt studio local variable issue --- .../prompt_studio_core_v2/serializers.py | 62 +++++++++++++------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/backend/prompt_studio/prompt_studio_core_v2/serializers.py b/backend/prompt_studio/prompt_studio_core_v2/serializers.py index bd9e32262..1f3f7e98d 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_core_v2/serializers.py @@ -44,48 +44,70 @@ class Meta: def to_representation(self, instance): # type: ignore data = super().to_representation(instance) + default_profile = None + + # Fetch summarize LLM profile try: - profile_manager = ProfileManager.objects.get( + summarize_profile = ProfileManager.objects.get( prompt_studio_tool=instance, is_summarize_llm=True ) - data[TSKeys.SUMMARIZE_LLM_PROFILE] = profile_manager.profile_id + data[TSKeys.SUMMARIZE_LLM_PROFILE] = summarize_profile.profile_id except ObjectDoesNotExist: logger.info( - "Summarize LLM profile doesnt exist for prompt tool %s", + "Summarize LLM profile doesn't exist for prompt tool %s", str(instance.tool_id), ) + + # Fetch default LLM profile try: - profile_manager = ProfileManager.get_default_llm_profile(instance) - data[TSKeys.DEFAULT_PROFILE] = profile_manager.profile_id + default_profile = ProfileManager.get_default_llm_profile(instance) + data[TSKeys.DEFAULT_PROFILE] = default_profile.profile_id except DefaultProfileError: logger.warning( - "Default LLM profile doesnt exist for prompt tool %s", + "Default LLM profile doesn't exist for prompt tool %s", str(instance.tool_id), ) - prompt_instance: ToolStudioPrompt = ToolStudioPrompt.objects.filter( + + # Fetch prompt instances + prompt_instances = ToolStudioPrompt.objects.filter( tool_id=data.get(TSKeys.TOOL_ID) ).order_by("sequence_number") - data[TSKeys.PROMPTS] = [] + + if not prompt_instances.exists(): + data[TSKeys.PROMPTS] = [] + return data + + # Process prompt instances output: list[Any] = [] - # Appending prompt instances of the tool for FE Processing - if prompt_instance.count() != 0: - for prompt in prompt_instance: - profile_manager_id = prompt.prompt_id - if instance.single_pass_extraction_mode: - # use projects default profile - profile_manager_id = profile_manager.profile_id - prompt_serializer = ToolStudioPromptSerializer(prompt) + for prompt in prompt_instances: + prompt_serializer = ToolStudioPromptSerializer(prompt) + serialized_data = prompt_serializer.data + + # Determine coverage + coverage: list[Any] = [] + profile_manager_id = prompt.profile_manager + if default_profile and instance.single_pass_extraction_mode: + profile_manager_id = default_profile.profile_id + + if profile_manager_id: coverage = OutputManagerUtils.get_coverage( data.get(TSKeys.TOOL_ID), profile_manager_id, prompt.prompt_id, instance.single_pass_extraction_mode, ) - serialized_data = prompt_serializer.data - serialized_data["coverage"] = coverage - output.append(serialized_data) - data[TSKeys.PROMPTS] = output + else: + logger.info( + "Skipping coverage calculation for prompt %s " + "due to missing profile ID", + str(prompt.prompt_key), + ) + + # Add coverage to serialized data + serialized_data["coverage"] = coverage + output.append(serialized_data) + data[TSKeys.PROMPTS] = output data["created_by_email"] = instance.created_by.email return data From b37efddc6463f01e0246d69772647ea112df19c0 Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Thu, 19 Dec 2024 15:19:32 +0530 Subject: [PATCH 3/7] updated return type --- .../prompt_studio_output_manager_v2/output_manager_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py index 6e945d5e1..b5b6a957d 100644 --- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py +++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py @@ -10,7 +10,7 @@ def get_coverage( profile_manager_id: str, prompt_id: str = None, is_single_pass: bool = False, - ) -> dict[str, list[str]]: + ) -> list[str]: """ Method to fetch coverage data for given tool and profile manager. From 2234a6ce4a270a7dfb124c24df1d7c27cdc79f1e Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Thu, 19 Dec 2024 16:28:55 +0530 Subject: [PATCH 4/7] added optional chaining --- .../custom-tools/manage-docs-modal/ManageDocsModal.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx index 281ccf05a..0688ac07e 100644 --- a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx +++ b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx @@ -563,9 +563,9 @@ function ManageDocsModal({ updatedData[key] = { ...value, // Update the coverage array if it exists - coverage: value.coverage - ? value.coverage.filter((id) => id !== idToRemove) - : value.coverage, + coverage: value?.coverage + ? value?.coverage?.filter((id) => id !== idToRemove) + : value?.coverage, }; return updatedData; }, {}); From a423d7568d5ec8f1d143068d417a5f737cfe124a Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Thu, 19 Dec 2024 16:30:55 +0530 Subject: [PATCH 5/7] added missing type --- backend/prompt_studio/prompt_studio_core_v2/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/prompt_studio/prompt_studio_core_v2/serializers.py b/backend/prompt_studio/prompt_studio_core_v2/serializers.py index 1f3f7e98d..131d57a1f 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_core_v2/serializers.py @@ -69,7 +69,7 @@ def to_representation(self, instance): # type: ignore ) # Fetch prompt instances - prompt_instances = ToolStudioPrompt.objects.filter( + prompt_instances: ToolStudioPrompt = ToolStudioPrompt.objects.filter( tool_id=data.get(TSKeys.TOOL_ID) ).order_by("sequence_number") From cee369e4b72f2fb70da7b25d3ed39d5080d48509 Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Thu, 19 Dec 2024 16:38:53 +0530 Subject: [PATCH 6/7] added comment for error suppression --- backend/prompt_studio/prompt_studio_core_v2/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/prompt_studio/prompt_studio_core_v2/serializers.py b/backend/prompt_studio/prompt_studio_core_v2/serializers.py index 131d57a1f..d6e79483a 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_core_v2/serializers.py @@ -63,6 +63,7 @@ def to_representation(self, instance): # type: ignore default_profile = ProfileManager.get_default_llm_profile(instance) data[TSKeys.DEFAULT_PROFILE] = default_profile.profile_id except DefaultProfileError: + # To make it compatible with older projects error suppressed with warning. logger.warning( "Default LLM profile doesn't exist for prompt tool %s", str(instance.tool_id), From 825f51ee0bb65164647848f64b0c39f80e94f23d Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Thu, 19 Dec 2024 17:50:39 +0530 Subject: [PATCH 7/7] handled edge cases --- .../manage-docs-modal/ManageDocsModal.jsx | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx index 0688ac07e..d563e8a1e 100644 --- a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx +++ b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx @@ -549,7 +549,12 @@ function ManageDocsModal({ const doc = newListOfDocs[0]; handleDocChange(doc); } - const updatedPromptOutput = removeIdFromCoverage(promptOutputs, docId); + const updatedPromptDetails = removeIdFromCoverage(details, docId); + const updatedPromptOutput = removeIdFromCoverageOfPromptOutput( + promptOutputs, + docId + ); + updateCustomTool({ details: updatedPromptDetails }); updatePromptOutput(updatedPromptOutput); }) .catch((err) => { @@ -558,6 +563,17 @@ function ManageDocsModal({ }; const removeIdFromCoverage = (data, idToRemove) => { + if (data.prompts && Array.isArray(data.prompts)) { + data.prompts.forEach((prompt) => { + if (Array.isArray(prompt.coverage)) { + prompt.coverage = prompt.coverage.filter((id) => id !== idToRemove); + } + }); + } + return data; // Return the updated data + }; + + const removeIdFromCoverageOfPromptOutput = (data, idToRemove) => { return Object.entries(data).reduce((updatedData, [key, value]) => { // Create a new object for the current entry updatedData[key] = {