File content type support (#725)

* File content type support

* Fix broken import

* Fix lint

* Fix lint

* fix unmocked test

apps/gateway/package.json

@@ -21,7 +21,7 @@
     "@latitude-data/constants": "workspace:^",
     "@latitude-data/core": "workspace:^",
     "@latitude-data/env": "workspace:^",
-    "@latitude-data/promptl": "^0.3.2",
+    "promptl-ai": "^0.3.3",
     "@sentry/cli": "^2.37.0",
     "@sentry/node": "^8.30.0",
     "@t3-oss/env-core": "^0.10.1",

apps/web/next.config.mjs

@@ -13,7 +13,7 @@ const nextConfig = {
   transpilePackages: INTERNAL_PACKAGES,
   serverExternalPackages: [
     '@latitude-data/sdk',
-    '@latitude-data/promptl',
+    'promptl-ai',
     '@latitude-data/compiler',
     '@sentry/nextjs',
     'bullmq',

apps/web/package.json

@@ -21,7 +21,7 @@
     "@latitude-data/constants": "workspace:*",
     "@latitude-data/core": "workspace:*",
     "@latitude-data/env": "workspace:^",
-    "@latitude-data/promptl": "^0.3.2",
+    "promptl-ai": "^0.3.3",
     "@latitude-data/sdk": "workspace:*",
     "@latitude-data/web-ui": "workspace:*",
     "@lucia-auth/adapter-drizzle": "^1.0.7",
@@ -31,7 +31,7 @@
     "@sentry/utils": "^8.30.0",
     "@sindresorhus/slugify": "^2.2.1",
     "@t3-oss/env-nextjs": "^0.10.1",
-    "ai": "^3.2.42",
+    "ai": "^4.0.18",
     "bullmq": "^5.8.5",
     "date-fns": "^3.6.0",
     "drizzle-orm": "^0.33.0",

apps/web/src/actions/documents/runDocumentInBatchAction.ts

@@ -4,7 +4,7 @@ import { readMetadata } from '@latitude-data/compiler'
 import { publisher } from '@latitude-data/core/events/publisher'
 import { setupJobs } from '@latitude-data/core/jobs'
 import { CommitsRepository } from '@latitude-data/core/repositories'
-import { scan } from '@latitude-data/promptl'
+import { scan } from 'promptl-ai'
 import { z } from 'zod'
 
 import {

apps/web/src/actions/evaluations/runBatch.ts

@@ -4,7 +4,7 @@ import { readMetadata } from '@latitude-data/compiler'
 import { publisher } from '@latitude-data/core/events/publisher'
 import { setupJobs } from '@latitude-data/core/jobs'
 import { EvaluationsRepository } from '@latitude-data/core/repositories'
-import { scan } from '@latitude-data/promptl'
+import { scan } from 'promptl-ai'
 import { nanoid } from 'nanoid'
 import { z } from 'zod'
 

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/EvaluationStats.tsx

@@ -8,7 +8,7 @@ import {
   EvaluationMetadataType,
 } from '@latitude-data/core/browser'
 import { ConnectedDocumentWithMetadata } from '@latitude-data/core/repositories'
-import { scan } from '@latitude-data/promptl'
+import { scan } from 'promptl-ai'
 import { Skeleton, Text } from '@latitude-data/web-ui'
 import { formatCostInMillicents } from '$/app/_lib/formatUtils'
 import useConnectedDocuments from '$/stores/connectedDocuments'

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Preview.tsx

@@ -18,7 +18,7 @@ import {
   Adapters,
   ConversationMetadata,
   Chain as PromptlChain,
-} from '@latitude-data/promptl'
+} from 'promptl-ai'
 import {
   Alert,
   Button,

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/index.tsx

@@ -4,7 +4,7 @@ import { useState } from 'react'
 
 import { useDocumentParameters } from '$/hooks/useDocumentParameters'
 import { DocumentVersion } from '@latitude-data/core/browser'
-import type { ConversationMetadata } from '@latitude-data/promptl'
+import type { ConversationMetadata } from 'promptl-ai'
 import {
   AppLocalStorage,
   SplitPane,

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentationModal/index.tsx

@@ -11,7 +11,7 @@ import {
 
 import { readMetadata } from '@latitude-data/compiler'
 import { ApiKey } from '@latitude-data/core/browser'
-import { scan, type ConversationMetadata } from '@latitude-data/promptl'
+import { scan, type ConversationMetadata } from 'promptl-ai'
 import { Modal } from '@latitude-data/web-ui'
 import { useCurrentDocument } from '$/app/providers/DocumentProvider'
 import useDocumentVersions from '$/stores/documentVersions'

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/batch/_components/RunPromptInBatchModal/index.tsx

@@ -4,7 +4,7 @@ import { useCallback, useEffect, useMemo, useState } from 'react'
 
 import { readMetadata } from '@latitude-data/compiler'
 import { Dataset, DocumentVersion } from '@latitude-data/core/browser'
-import { scan, type ConversationMetadata } from '@latitude-data/promptl'
+import { scan, type ConversationMetadata } from 'promptl-ai'
 import {
   Button,
   CloseTrigger,

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Actions/CreateBatchEvaluationModal/useRunBatchForm.ts

@@ -1,7 +1,7 @@
 import { useCallback, useMemo, useState } from 'react'
 
 import { Dataset, DocumentVersion } from '@latitude-data/core/browser'
-import type { ConversationMetadata } from '@latitude-data/promptl'
+import type { ConversationMetadata } from 'promptl-ai'
 import { SelectOption, useCurrentCommit } from '@latitude-data/web-ui'
 import { useMappedParametersFromLocalStorage } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/batch/_components/RunPromptInBatchModal/useMappedParametersFromLocalStorage'
 import useDatasets from '$/stores/datasets'

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx

@@ -6,7 +6,7 @@ import {
   EvaluationResultableType,
 } from '@latitude-data/core/browser'
 import { env } from '@latitude-data/env'
-import { scan } from '@latitude-data/promptl'
+import { scan } from 'promptl-ai'
 import {
   Breadcrumb,
   BreadcrumbItem,

apps/web/src/app/(public)/share/d/[publishedDocumentUuid]/_components/SharedDocument/index.tsx

@@ -11,7 +11,7 @@ import {
 import { capitalize } from 'lodash-es'
 
 import { PublishedDocument } from '@latitude-data/core/browser'
-import { ConversationMetadata } from '@latitude-data/promptl'
+import { ConversationMetadata } from 'promptl-ai'
 import { Button, Card, CardContent, cn, TextArea } from '@latitude-data/web-ui'
 
 import { Container } from '../Container'

apps/web/src/app/(public)/share/d/[publishedDocumentUuid]/_components/SharedDocument/usePrompt.ts

@@ -46,6 +46,7 @@ function getDeltas({
 
     return content.flatMap((c) => {
       if (c.type === ContentType.image) return []
+      if (c.type === ContentType.file) return []
       if (c.type === ContentType.toolCall)
         return splitInWords(JSON.stringify(c))
       if (c.type === ContentType.toolResult)

apps/web/src/components/EditorHeader/index.tsx

@@ -9,7 +9,7 @@ import {
   findFirstModelForProvider,
   ProviderApiKey,
 } from '@latitude-data/core/browser'
-import type { ConversationMetadata } from '@latitude-data/promptl'
+import type { ConversationMetadata } from 'promptl-ai'
 import {
   AppLocalStorage,
   DropdownMenu,

apps/web/src/hooks/useDocumentParameters/index.ts

@@ -2,7 +2,7 @@ import { useCallback, useMemo } from 'react'
 
 import { recalculateInputs } from '$/hooks/useDocumentParameters/recalculateInputs'
 import { DocumentLog, ParameterType } from '@latitude-data/core/browser'
-import type { ConversationMetadata } from '@latitude-data/promptl'
+import type { ConversationMetadata } from 'promptl-ai'
 import { AppLocalStorage, useLocalStorage } from '@latitude-data/web-ui'
 
 export const INPUT_SOURCE = {

apps/web/src/hooks/useDocumentParameters/recalculateInputs/index.test.ts

@@ -1,5 +1,5 @@
 import { ParameterType } from '@latitude-data/core/browser'
-import type { ConversationMetadata } from '@latitude-data/promptl'
+import type { ConversationMetadata } from 'promptl-ai'
 import { describe, expect, it } from 'vitest'
 
 import { recalculateInputs } from './index'

apps/web/src/hooks/useDocumentParameters/recalculateInputs/index.ts

@@ -1,6 +1,6 @@
 import { Inputs, InputSource } from '$/hooks/useDocumentParameters'
 import { ParameterType } from '@latitude-data/core/browser'
-import type { ConversationMetadata } from '@latitude-data/promptl'
+import type { ConversationMetadata } from 'promptl-ai'
 
 const ParameterTypes = Object.values(ParameterType) as string[]
 

apps/web/src/hooks/useMetadata.ts

@@ -2,7 +2,7 @@
 
 import { useEffect, useRef, useState } from 'react'
 
-import type { ConversationMetadata } from '@latitude-data/promptl'
+import type { ConversationMetadata } from 'promptl-ai'
 import { useDebouncedCallback } from 'use-debounce'
 
 import type { ReadMetadataWorkerProps } from '../workers/readMetadata'

apps/web/src/workers/readMetadata.ts

@@ -1,6 +1,6 @@
 import { readMetadata } from '@latitude-data/compiler'
 import { type DocumentVersion } from '@latitude-data/core/browser'
-import { scan } from '@latitude-data/promptl'
+import { scan } from 'promptl-ai'
 
 export type ReadMetadataWorkerProps = Parameters<typeof readMetadata>[0] & {
   promptlVersion: number

docs/promptl/advanced/chains.mdx

@@ -159,7 +159,7 @@ To run a step, execute the `step` method of the chain instance. The first time `
 
 ### Example: Using the Chain Class
 ```javascript
-import { Chain } from '@latitude-data/promptl';
+import { Chain } from 'promptl-ai';
 import OpenAI from 'openai';
 
 // Initialize the OpenAI client

docs/promptl/advanced/snippets.mdx

@@ -76,7 +76,7 @@ function getPrompt(relativePath, currentPath) {
 ### Using the `resolveFn`
 Pass your `resolveFn` to the `render` function:
 ```javascript
-import { render } from '@latitude-data/promptl';
+import { render } from 'promptl-ai';
 import getPrompt from './getPrompt'; // Import your custom resolve function
 
 const { messages, config } = await render({

docs/promptl/syntax/messages.mdx

@@ -105,6 +105,7 @@ Messages can contain different types of content. By default, all plain text is c
 
 - **Text (default)**: `<content type="text">` or `<content-text>`.
 - **Image**: `<content type="image">` or `<content-image>`. Add the image URL or base64-encoded string – depending on your provider's requirements – as the content inside the tag.
+- **File**: `<content type="file">` or `<content-file>`. Add the file URL or base64-encoded string – depending on your provider's requirements – as the content inside the tag. Requires a MIME type `mime` attribute to specify the file type.
 - **Tool Call**: `<content type="tool-call">` or `<tool-call>`. Represents a tool invocation with attributes like `id`, `name`, and `arguments` (optional). Only allowed inside assistant messages.
 
 <Note>
@@ -116,6 +117,9 @@ Messages can contain different types of content. By default, all plain text is c
 <user>
   Take a look at this image:
   <content-image>[image url]</content>
+
+  And this PDF:
+  <content-file mime="application/pdf">[file url]</content>
 </user>
 
 <assistant>

docs/promptl/usage/adapters/anthropic.mdx

@@ -22,7 +22,7 @@ Here’s how to use PromptL with Anthropic’s API:
 
 ```typescript
 import Anthropic from '@anthropic-ai/sdk'
-import { Adapters, render } from '@latitude-data/promptl'
+import { Adapters, render } from 'promptl-ai'
 
 const prompt = `
 ---

docs/promptl/usage/adapters/openai.mdx

@@ -14,7 +14,7 @@ PromptL seamlessly integrates with OpenAI's API. By default, PromptL formats pro
 Here’s how to generate a response from OpenAI using PromptL:
 
 ```typescript
-import { render } from '@latitude-data/promptl'
+import { render } from 'promptl-ai'
 import OpenAI from 'openai'
 
 const prompt = `

docs/promptl/usage/adapters/overview.mdx

@@ -30,7 +30,7 @@ By using an adapter, you don’t need to worry about these differences—PromptL
 Here’s how to use an adapter in your project. For this example, we’ll use OpenAI:
 
 ```javascript
-import { render, Adapter } from '@latitude-data/promptl';
+import { render, Adapter } from 'promptl-ai';
 
 const { messages, config } = await render({
   prompt,

docs/promptl/usage/quick-start.mdx

@@ -41,7 +41,7 @@ Here’s how to use PromptL to generate a dynamic prompt and interact with an LL
 
 ### Example Code
 ```javascript
-import { render } from '@latitude-data/promptl'
+import { render } from 'promptl-ai'
 import OpenAI from 'openai'
 
 // Define your PromptL prompt
@@ -84,7 +84,7 @@ console.log(response.choices[0].message.content)
 For production environments, add error handling to manage unexpected issues:
 
 ```javascript
-import { render } from '@latitude-data/promptl'
+import { render } from 'promptl-ai'
 import OpenAI from 'openai'
 
 async function generateResponse(prompt, parameters) {

examples/chat-app/package.json

@@ -9,13 +9,13 @@
     "dev": "tsx watch server.ts"
   },
   "dependencies": {
-    "@ai-sdk/openai": "^0.0.54",
+    "@ai-sdk/openai": "^1.0.8",
     "@anthropic-ai/sdk": "^0.32.1",
     "@latitude-data/sdk": "workspace:^",
     "@latitude-data/telemetry": "workspace:^",
     "@types/morgan": "^1.9.9",
     "@vercel/otel": "^1.10.0",
-    "ai": "^4.0.9",
+    "ai": "^4.0.18",
     "axios": "^1.7.8",
     "dotenv": "^16.4.5",
     "express": "^4.21.1",

examples/sdks/typescript/package.json

@@ -8,12 +8,12 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@ai-sdk/openai": "^0.0.54",
+    "@ai-sdk/openai": "^1.0.8",
     "@anthropic-ai/sdk": "^0.32.1",
     "@latitude-data/sdk": "workspace:^",
     "@latitude-data/telemetry": "workspace:^",
     "@vercel/otel": "^1.10.0",
-    "ai": "^4.0.3",
+    "ai": "^4.0.18",
     "openai": "^4.68.4",
     "zod": "^3.23.8"
   }

package.json

@@ -37,11 +37,11 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/anthropic": "^0.0.48",
-    "@ai-sdk/azure": "^0.0.32",
-    "@ai-sdk/google": "^0.0.51",
-    "@ai-sdk/mistral": "^0.0.38",
-    "@ai-sdk/openai": "^0.0.54",
+    "@ai-sdk/anthropic": "^1.0.5",
+    "@ai-sdk/azure": "^1.0.10",
+    "@ai-sdk/google": "^1.0.10",
+    "@ai-sdk/mistral": "^1.0.5",
+    "@ai-sdk/openai": "^1.0.8",
     "@aws-sdk/client-s3": "^3.645.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@monaco-editor/react": "^4.6.0",
@@ -59,7 +59,7 @@
     "@react-email/components": "^0.0.23",
     "@sindresorhus/slugify": "^2.2.1",
     "@t3-oss/env-core": "^0.11.1",
-    "ai": "^3.2.42",
+    "ai": "^4.0.18",
     "argon2": "^0.41.0",
     "bullmq": "^5.12.11",
     "class-variance-authority": "^0.7.0",

packages/compiler/src/types/message.ts

@@ -1,6 +1,7 @@
 export enum ContentType {
   text = 'text',
   image = 'image',
+  file = 'file', // Not supported but it is here because almost all code uses legacy compiler types
   toolCall = 'tool-call',
   toolResult = 'tool-result',
 }
@@ -34,6 +35,13 @@ export type ImageContent = IMessageContent & {
   image: string | Uint8Array | Buffer | ArrayBuffer | URL
 }
 
+// Not supported but it is here because almost all code uses legacy compiler types
+export type FileContent = IMessageContent & {
+  type: ContentType.file
+  file: string | Uint8Array | Buffer | ArrayBuffer | URL
+  mimeType: string
+}
+
 export type ToolContent = {
   type: ContentType.toolResult
   toolCallId: string
@@ -52,6 +60,7 @@ export type ToolRequestContent = {
 export type MessageContent =
   | TextContent
   | ImageContent
+  | FileContent // Not supported but it is here because almost all code uses legacy compiler types
   | ToolContent
   | ToolRequestContent