diff --git a/ea-scripts/ExcaliAI.md b/ea-scripts/ExcaliAI.md index bf8a88e..7c374d2 100644 --- a/ea-scripts/ExcaliAI.md +++ b/ea-scripts/ExcaliAI.md @@ -3,10 +3,9 @@ ![](https://raw.githubusercontent.com/zsviczian/obsidian-excalidraw-plugin/master/images/scripts-draw-a-ui.jpg) ```js*/ -let previewImg, previewDiv; let dirty=false; -if(!ea.verifyMinimumPluginVersion || !ea.verifyMinimumPluginVersion("2.0.11")) { +if(!ea.verifyMinimumPluginVersion || !ea.verifyMinimumPluginVersion("2.0.12")) { new Notice("This script requires a newer version of Excalidraw. Please install the latest version."); return; } @@ -27,41 +26,58 @@ const outputTypes = { "image-gen": { instruction: "Return a single message with the generated image prompt in a codeblock", blocktype: "image" + }, + "image-edit": { + instruction: "", + blocktype: "image" } } const systemPrompts = { "Challenge my thinking": { prompt: `Your task is to interpret a screenshot of a whiteboard, translating its ideas into a Mermaid graph. The whiteboard will encompass thoughts on a subject. Within the mind map, distinguish ideas that challenge, dispute, or contradict the whiteboard content. Additionally, include concepts that expand, complement, or advance the user's thinking. Utilize the Mermaid graph diagram type and present the resulting Mermaid diagram within a code block. Ensure the Mermaid script excludes the use of parentheses ().`, - type: "mermaid" + type: "mermaid", + help: "Translate your image and optional text prompt into a Mermaid mindmap. If there are conversion errors, edit the Mermaid script under 'More Tools'." }, "Convert sketch to shapes": { prompt: `Given an image featuring various geometric shapes drawn by the user, your objective is to analyze the input and generate SVG code that accurately represents these shapes. Your output will be the SVG code enclosed in an HTML code block.`, - type: "svg" + type: "svg", + help: "Convert selected scribbles into shapes; works better with fewer shapes. Experimental and may not produce good drawings." }, - "Excalidraw sketch": { + "Create a simple Excalidraw icon": { prompt: `Given a description of an SVG image from the user, your objective is to generate the corresponding SVG code. Avoid incorporating textual elements within the generated SVG. Your output should be the resulting SVG code enclosed in an HTML code block.`, - type: "svg" + type: "svg", + help: "Convert text prompts into simple icons inserted as Excalidraw elements. Expect only a text prompt. Experimental and may not produce good drawings." + }, + "Edit an image": { + prompt: null, + type: "image-edit", + help: "Image elements will be used as the Image. Shapes on top of the image will be the Mask. Use the prompt to instruct Dall-e about the changes. Dall-e-2 model will be used." }, "Generate an image from image and prompt": { prompt: "Your task involves receiving an image and a textual prompt from the user. Your goal is to craft a detailed, accurate, and descriptive narrative of the image, tailored for effective image generation. Utilize the user-provided text prompt to inform and guide your depiction of the image. Ensure the resulting image remains text-free.", - type: "image-gen" + type: "image-gen", + help: "Generate an image based on the drawing and prompt using ChatGPT-Vision and Dall-e. Provide a contextual text-prompt for accurate interpretation." }, "Generate an image from prompt": { prompt: null, - type: "image-gen" + type: "image-gen", + help: "Send only the text prompt to OpenAI. Provide a detailed description; OpenAI will enrich your prompt automatically. To avoid it, start your prompt like this 'DO NOT add any detail, just use it AS-IS:'" }, "Generate an image to illustrate a quote": { prompt: "Your task involves transforming a user-provided quote into a detailed and imaginative illustration. Craft a visual representation that captures the essence of the quote and resonates well with a broad audience. If the Author's name is provided, aim to establish a connection between the illustration and the Author. This can be achieved by referencing a well-known story from the Author, situating the image in the Author's era or setting, or employing other creative methods of association. Additionally, provide preferences for styling, such as the chosen medium and artistic direction, to guide the image creation process. Ensure the resulting image remains text-free. Your task output should comprise a descriptive and detailed narrative aimed at facilitating the creation of a captivating illustration from the quote.", - type: "image-gen" + type: "image-gen", + help: "ExcaliAI will create an image prompt to illustrate your text input - a quote - with GPT, then generate an image using Dall-e. In case you include the Author's name, GPT will try to generate an image that in some way references the Author." }, "Visual brainstorm": { prompt: "Your objective is to interpret a screenshot of a whiteboard, creating an image aimed at sparking further thoughts on the subject. The whiteboard will present diverse ideas about a specific topic. Your generated image should achieve one of two purposes: highlighting concepts that challenge, dispute, or contradict the whiteboard content, or introducing ideas that expand, complement, or enrich the user's thinking. You have the option to include multiple tiles in the resulting image, resembling a sequence akin to a comic strip. Ensure that the image remains devoid of text.", - type: "image-gen" + type: "image-gen", + help: "Use ChatGPT Visions and Dall-e to create an image based on your text prompt and image to spark new ideas." }, "Wireframe to code": { prompt: `You are an expert tailwind developer. A user will provide you with a low-fidelity wireframe of an application and you will return a single html file that uses tailwind to create the website. Use creative license to make the application more fleshed out. Write the necessary javascript code. If you need to insert an image, use placehold.co to create a placeholder image.`, - type: "html" + type: "html", + help: "Use GPT Visions to interpret the wireframe and generate a web application. YOu may copy the resulting code from the active embeddable's top left menu." }, } @@ -85,23 +101,30 @@ if(!OPENAI_API_KEY || OPENAI_API_KEY === "") { return; } -const imageModel = ea.plugin.settings.openAIDefaultImageGenerationModel; let userPrompt = settings["User Prompt"] ?? ""; let agentTask = settings["Agent's Task"]; let imageSize = settings["Image Size"]??"1024x1024"; -const validSizes = imageModel === "dall-e-2" - ? [`256x256`, `512x512`, `1024x1024`] - : (imageModel === "dall-e-3" - ? [`1024x1024`, `1792x1024`, `1024x1792`] - : [`1024x1024`]) -if(!validSizes.includes(imageSize)) { - imageSize = "1024x1024"; - dirty = true; -} if(!systemPrompts.hasOwnProperty(agentTask)) { agentTask = Object.keys(systemPrompts)[0]; } +let imageModel, valideSizes; + +const setImageModelAndSizes = () => { + imageModel = systemPrompts[agentTask].type === "image-edit" + ? "dall-e-2" + : ea.plugin.settings.openAIDefaultImageGenerationModel; + validSizes = imageModel === "dall-e-2" + ? [`256x256`, `512x512`, `1024x1024`] + : (imageModel === "dall-e-3" + ? [`1024x1024`, `1792x1024`, `1024x1792`] + : [`1024x1024`]) + if(!validSizes.includes(imageSize)) { + imageSize = "1024x1024"; + dirty = true; + } +} +setImageModelAndSizes(); // -------------------------------------- // Generate Image Blob From Selected Excalidraw Elements @@ -120,51 +143,114 @@ const calculateImageScale = (elements) => { ); } -const generateCanvasDataURL = async (view, makeSquare=false) => { +const createMask = async (dataURL) => { + return new Promise((resolve, reject) => { + const img = new Image(); + + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = img.width; + canvas.height = img.height; + + const ctx = canvas.getContext('2d'); + ctx.drawImage(img, 0, 0); + + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const data = imageData.data; + + for (let i = 0; i < data.length; i += 4) { + // If opaque (alpha > 0), make it transparent + if (data[i + 3] > 0) { + data[i + 3] = 0; // Set alpha to 0 (transparent) + } else if (data[i + 3] === 0) { + // If fully transparent, make it red + data[i] = 255; // Red + data[i + 1] = 0; // Green + data[i + 2] = 0; // Blue + data[i + 3] = 255; // make it opaque + } + } + + ctx.putImageData(imageData, 0, 0); + const maskDataURL = canvas.toDataURL(); + + resolve(maskDataURL); + }; + + img.onerror = error => { + reject(error); + }; + + img.src = dataURL; + }); +} + +//https://platform.openai.com/docs/api-reference/images/createEdit +//dall-e-2 image edit only works on square images +//if targetDalleImageEdit === true then the image and the mask will be returned in two separate dataURLs +let squareBB; + +const generateCanvasDataURL = async (view, targetDalleImageEdit=false) => { + let PADDING = 5; await view.forceSave(true); //to ensure recently embedded PNG and other images are saved to file const viewElements = ea.getViewSelectedElements(); if(viewElements.length === 0) { - return; + return {imageDataURL: null, maskDataURL: null} ; } ea.copyViewElementsToEAforEditing(viewElements, true); //copying the images objects over to EA for PNG generation - if(makeSquare) { - const bb = ea.getBoundingBox(viewElements); + let maskDataURL; + const loader = ea.getEmbeddedFilesLoader(false); + let scale = calculateImageScale(ea.getElements()); + const bb = ea.getBoundingBox(viewElements); + if(ea.getElements() + .filter(el=>el.type==="image") + .some(el=>Math.round(el.width) === Math.round(bb.width) && Math.round(el.height) === Math.round(bb.height)) + ) { PADDING = 0; } + + let exportSettings = {withBackground: true, withTheme: true}; + + if(targetDalleImageEdit) { + PADDING = 0; const strokeColor = ea.style.strokeColor; const backgroundColor = ea.style.backgroundColor; ea.style.backgroundColor = "transparent"; ea.style.strokeColor = "transparent"; - //deliberately not adding a rect if width === height + let rectID; if(bb.height > bb.width) { - ea.addRect(bb.topX-(bb.height-bb.width)/2, bb.topY,bb.height, bb.height); + rectID = ea.addRect(bb.topX-(bb.height-bb.width)/2, bb.topY,bb.height, bb.height); } if(bb.width > bb.height) { - ea.addRect(bb.topX, bb.topY-(bb.width-bb.height)/2,bb.width, bb.width); + rectID = ea.addRect(bb.topX, bb.topY-(bb.width-bb.height)/2,bb.width, bb.width); } + if(bb.height === bb.width) { + rectID = ea.addRect(bb.topX, bb.topY, bb.width, bb.height); + } + const rect = ea.getElement(rectID); + squareBB = {topX: rect.x-PADDING, topY: rect.y-PADDING, width: rect.width + 2*PADDING, height: rect.height + 2*PADDING}; ea.style.strokeColor = strokeColor; ea.style.backgroundColor = backgroundColor; - } - const scale = calculateImageScale(ea.getElements()); + ea.getElements().filter(el=>el.type === "image").forEach(el=>{el.isDeleted = true}); - const loader = ea.getEmbeddedFilesLoader(false); - const exportSettings = { - withBackground: true, - withTheme: true, - }; - - const dataURL = - await ea.createPNGBase64( - null, - scale, - exportSettings, - loader, - "light", + dalleWidth = parseInt(imageSize.split("x")[0]); + scale = dalleWidth/squareBB.width; + exportSettings = {withBackground: false, withTheme: true}; + maskDataURL= await ea.createPNGBase64( + null, scale, exportSettings, loader, "light", PADDING ); + maskDataURL = await createMask(maskDataURL) + ea.getElements().filter(el=>el.type === "image").forEach(el=>{el.isDeleted = false}); + ea.getElements().filter(el=>el.type !== "image" && el.id !== rectID).forEach(el=>{el.isDeleted = true}); + } + + const imageDataURL = await ea.createPNGBase64( + null, scale, exportSettings, loader, "light", PADDING + ); ea.clear(); - return dataURL; + return {imageDataURL, maskDataURL}; } -let imageDataURL = await generateCanvasDataURL(ea.targetView); +let {imageDataURL, maskDataURL} = await generateCanvasDataURL(ea.targetView, systemPrompts[agentTask].type === "image-edit"); // -------------------------------------- // Support functions - embeddable spinner and error @@ -242,6 +328,7 @@ const generateImage = async(text, spinnerID, bb) => { n:1, }, }; + const result = await ea.postOpenAI(requestObject); console.log({result, json:result?.json}); @@ -257,8 +344,8 @@ const generateImage = async(text, spinnerID, bb) => { const revisedPrompt = result.json.data[0].revised_prompt; if(revisedPrompt) { ea.style.fontSize = 16; - const rectID = ea.addText(imageEl.x, imageEl.y + imageEl.height + 50, revisedPrompt, { - width: imageEl.width, + const rectID = ea.addText(imageEl.x+15, imageEl.y + imageEl.height + 50, revisedPrompt, { + width: imageEl.width-30, textAlign: "center", textVerticalAlign: "top", box: true, @@ -285,12 +372,17 @@ const run = async (text) => { const systemPrompt = systemPrompts[agentTask]; const outputType = outputTypes[systemPrompt.type]; const isImageGenRequest = outputType.blocktype === "image"; - - const requestObject = { - ...imageDataURL ? {image: imageDataURL} : {}, - ...(text && text.trim() !== "") ? {text} : {}, - systemPrompt: systemPrompt.prompt, - instruction: outputType.instruction, + const isImageEditRequest = systemPrompt.type === "image-edit"; + + if(isImageEditRequest) { + if(!text) { + new Notice("You must provide a text prompt with instructions for how the image should be modified"); + return; + } + if(!imageDataURL || !maskDataURL) { + new Notice("You must provide an image and a mask"); + return; + } } //place spinner next to selected elements @@ -312,11 +404,29 @@ const run = async (text) => { isEACompleted = true; }); - if(isImageGenRequest && !systemPrompt.prompt) { + if(isImageGenRequest && !systemPrompt.prompt && !isImageEditRequest) { generateImage(text,spinnerID,bb); return; } - + + const requestObject = isImageEditRequest + ? { + ...imageDataURL ? {image: imageDataURL} : {}, + ...(text && text.trim() !== "") ? {text} : {}, + imageGenerationProperties: { + size: imageSize, + //quality: "standard", //not supported by dall-e-2 + n:1, + mask: maskDataURL, + }, + } + : { + ...imageDataURL ? {image: imageDataURL} : {}, + ...(text && text.trim() !== "") ? {text} : {}, + systemPrompt: systemPrompt.prompt, + instruction: outputType.instruction, + } + //Get result from GPT const result = await ea.postOpenAI(requestObject); console.log({result, json:result?.json}); @@ -330,7 +440,25 @@ const run = async (text) => { await errorMessage(spinnerID, "Unexpected issue with ExcalidrawAutomate"); return; } - + + if(isImageEditRequest) { + if(!result?.json?.data?.[0]?.url) { + await errorMessage(spinnerID, result?.json?.error?.message); + return; + } + + const spinner = ea.getElement(spinnerID) + spinner.isDeleted = true; + const imageID = await ea.addImage(spinner.x, spinner.y, result.json.data[0].url); + await ea.addElementsToView(false, true, true); + ea.getExcalidrawAPI().setToast({ + message: IMAGE_WARNING, + duration: 15000, + closable: true + }); + return; + } + if(!result?.json?.hasOwnProperty("choices")) { await errorMessage(spinnerID, result?.json?.error?.message); return; @@ -389,8 +517,27 @@ const run = async (text) => { // -------------------------------------- // User Interface // -------------------------------------- +let previewDiv; const fragWithHTML = (html) => createFragment((frag) => (frag.createDiv().innerHTML = html)); -const isImageGenerationTask = () => systemPrompts[agentTask].type === "image-gen"; +const isImageGenerationTask = () => systemPrompts[agentTask].type === "image-gen" || systemPrompts[agentTask].type === "image-edit"; +const addPreviewImage = () => { + if(!previewDiv) return; + previewDiv.empty(); + previewDiv.createEl("img",{ + attr: { + style: `max-width: 100%;max-height: 30vh;`, + src: imageDataURL, + } + }); + if(maskDataURL) { + previewDiv.createEl("img",{ + attr: { + style: `max-width: 100%;max-height: 30vh;`, + src: maskDataURL, + } + }); + } +} const configModal = new ea.obsidian.Modal(app); configModal.modalEl.style.width="100%"; @@ -400,19 +547,32 @@ configModal.onOpen = async () => { const contentEl = configModal.contentEl; contentEl.createEl("h1", {text: "ExcaliAI"}); - let systemPromptTextArea, systemPromptDiv, imageSizeSetting; + let systemPromptTextArea, systemPromptDiv, imageSizeSetting, imageSizeSettingDropdown, helpEl; new ea.obsidian.Setting(contentEl) - .setName("Select Prompt") + .setName("What would you like to do?") .addDropdown(dropdown=>{ Object.keys(systemPrompts).forEach(key=>dropdown.addOption(key,key)); dropdown .setValue(agentTask) - .onChange(value => { + .onChange(async (value) => { dirty = true; + const prevTask = agentTask; agentTask = value; + if( + (systemPrompts[prevTask].type === "image-edit" && systemPrompts[value].type !== "image-edit") || + (systemPrompts[prevTask].type !== "image-edit" && systemPrompts[value].type === "image-edit") + ) { + ({imageDataURL, maskDataURL} = await generateCanvasDataURL(ea.targetView, systemPrompts[value].type === "image-edit")); + addPreviewImage(); + setImageModelAndSizes(); + while (imageSizeSettingDropdown.selectEl.options.length > 0) { imageSizeSettingDropdown.selectEl.remove(0); } + validSizes.forEach(size=>imageSizeSettingDropdown.addOption(size,size)); + imageSizeSettingDropdown.setValue(imageSize); + } imageSizeSetting.settingEl.style.display = isImageGenerationTask() ? "" : "none"; const prompt = systemPrompts[value].prompt; + helpEl.innerHTML = `Help: ` + systemPrompts[value].help; if(prompt) { systemPromptDiv.style.display = ""; systemPromptTextArea.setValue(systemPrompts[value].prompt); @@ -422,6 +582,9 @@ configModal.onOpen = async () => { }); }) + helpEl = contentEl.createEl("p"); + helpEl.innerHTML = `Help: ` + systemPrompts[agentTask].help; + systemPromptDiv = contentEl.createDiv(); systemPromptDiv.createEl("h4", {text: "Customize System Prompt"}); systemPromptDiv.createEl("span", {text: "Unless you know what you are doing I do not recommend changing the system prompt"}) @@ -461,12 +624,17 @@ configModal.onOpen = async () => { .setDesc(fragWithHTML("⚠️ Important ⚠️: " + IMAGE_WARNING)) .addDropdown(dropdown=>{ validSizes.forEach(size=>dropdown.addOption(size,size)); + imageSizeSettingDropdown = dropdown; dropdown - .setValue(imageSize) - .onChange(value => { - dirty = true; - imageSize = value; - }); + .setValue(imageSize) + .onChange(async (value) => { + dirty = true; + imageSize = value; + if(systemPrompts[agentTask].type === "image-edit") { + ({imageDataURL, maskDataURL} = await generateCanvasDataURL(ea.targetView, true)); + addPreviewImage(); + } + }); }) imageSizeSetting.settingEl.style.display = isImageGenerationTask() ? "" : "none"; @@ -476,14 +644,9 @@ configModal.onOpen = async () => { style: "text-align: center;", } }); - previewImg = previewDiv.createEl("img",{ - attr: { - style: `max-width: 100%;max-height: 30vh;`, - src: imageDataURL, - } - }); + addPreviewImage(); } else { - contentEl.createEl("h4", {text: "No elements are selected"}); + contentEl.createEl("h4", {text: "No elements are selected from your canvas"}); contentEl.createEl("span", {text: "Because there are no Excalidraw elements selected on the canvas, only the text prompt will be sent to OpenAI."}); } diff --git a/ea-scripts/GPT-Draw-a-UI.md b/ea-scripts/GPT-Draw-a-UI.md index bf8a88e..68663ea 100644 --- a/ea-scripts/GPT-Draw-a-UI.md +++ b/ea-scripts/GPT-Draw-a-UI.md @@ -3,10 +3,9 @@ ![](https://raw.githubusercontent.com/zsviczian/obsidian-excalidraw-plugin/master/images/scripts-draw-a-ui.jpg) ```js*/ -let previewImg, previewDiv; let dirty=false; -if(!ea.verifyMinimumPluginVersion || !ea.verifyMinimumPluginVersion("2.0.11")) { +if(!ea.verifyMinimumPluginVersion || !ea.verifyMinimumPluginVersion("2.0.12")) { new Notice("This script requires a newer version of Excalidraw. Please install the latest version."); return; } @@ -27,41 +26,58 @@ const outputTypes = { "image-gen": { instruction: "Return a single message with the generated image prompt in a codeblock", blocktype: "image" + }, + "image-edit": { + instruction: "", + blocktype: "image" } } const systemPrompts = { "Challenge my thinking": { prompt: `Your task is to interpret a screenshot of a whiteboard, translating its ideas into a Mermaid graph. The whiteboard will encompass thoughts on a subject. Within the mind map, distinguish ideas that challenge, dispute, or contradict the whiteboard content. Additionally, include concepts that expand, complement, or advance the user's thinking. Utilize the Mermaid graph diagram type and present the resulting Mermaid diagram within a code block. Ensure the Mermaid script excludes the use of parentheses ().`, - type: "mermaid" + type: "mermaid", + help: "Translate your image and optional text prompt into a Mermaid mindmap. If there are conversion errors, edit the Mermaid script under 'More Tools'." }, "Convert sketch to shapes": { prompt: `Given an image featuring various geometric shapes drawn by the user, your objective is to analyze the input and generate SVG code that accurately represents these shapes. Your output will be the SVG code enclosed in an HTML code block.`, - type: "svg" + type: "svg", + help: "Convert selected scribbles into shapes; works better with fewer shapes. Experimental and may not produce good drawings." }, - "Excalidraw sketch": { + "Create a simple Excalidraw icon": { prompt: `Given a description of an SVG image from the user, your objective is to generate the corresponding SVG code. Avoid incorporating textual elements within the generated SVG. Your output should be the resulting SVG code enclosed in an HTML code block.`, - type: "svg" + type: "svg", + help: "Convert text prompts into simple icons inserted as Excalidraw elements. Expect only a text prompt. Experimental and may not produce good drawings." + }, + "Edit an image": { + prompt: null, + type: "image-edit", + help: "Image elements will be used as the Image. Shapes on top of the image will be the Mask. Use the prompt to instruct Dall-e about the changes. Dall-e-2 model will be used." }, "Generate an image from image and prompt": { prompt: "Your task involves receiving an image and a textual prompt from the user. Your goal is to craft a detailed, accurate, and descriptive narrative of the image, tailored for effective image generation. Utilize the user-provided text prompt to inform and guide your depiction of the image. Ensure the resulting image remains text-free.", - type: "image-gen" + type: "image-gen", + help: "Generate an image based on the drawing and prompt using ChatGPT-Vision and Dall-e. Provide a contextual text-prompt for accurate interpretation." }, "Generate an image from prompt": { prompt: null, - type: "image-gen" + type: "image-gen", + help: "Send only the text prompt to OpenAI. Provide a detailed description; OpenAI will enrich your prompt automatically. To avoid it, start your prompt like this 'DO NOT add any detail, just use it AS-IS:'" }, "Generate an image to illustrate a quote": { prompt: "Your task involves transforming a user-provided quote into a detailed and imaginative illustration. Craft a visual representation that captures the essence of the quote and resonates well with a broad audience. If the Author's name is provided, aim to establish a connection between the illustration and the Author. This can be achieved by referencing a well-known story from the Author, situating the image in the Author's era or setting, or employing other creative methods of association. Additionally, provide preferences for styling, such as the chosen medium and artistic direction, to guide the image creation process. Ensure the resulting image remains text-free. Your task output should comprise a descriptive and detailed narrative aimed at facilitating the creation of a captivating illustration from the quote.", - type: "image-gen" + type: "image-gen", + help: "ExcaliAI will create an image prompt to illustrate your text input - a quote - with GPT, then generate an image using Dall-e. In case you include the Author's name, GPT will try to generate an image that in some way references the Author." }, "Visual brainstorm": { prompt: "Your objective is to interpret a screenshot of a whiteboard, creating an image aimed at sparking further thoughts on the subject. The whiteboard will present diverse ideas about a specific topic. Your generated image should achieve one of two purposes: highlighting concepts that challenge, dispute, or contradict the whiteboard content, or introducing ideas that expand, complement, or enrich the user's thinking. You have the option to include multiple tiles in the resulting image, resembling a sequence akin to a comic strip. Ensure that the image remains devoid of text.", - type: "image-gen" + type: "image-gen", + help: "Use ChatGPT Visions and Dall-e to create an image based on your text prompt and image to spark new ideas." }, "Wireframe to code": { prompt: `You are an expert tailwind developer. A user will provide you with a low-fidelity wireframe of an application and you will return a single html file that uses tailwind to create the website. Use creative license to make the application more fleshed out. Write the necessary javascript code. If you need to insert an image, use placehold.co to create a placeholder image.`, - type: "html" + type: "html", + help: "Use GPT Visions to interpret the wireframe and generate a web application. YOu may copy the resulting code from the active embeddable's top left menu." }, } @@ -85,23 +101,30 @@ if(!OPENAI_API_KEY || OPENAI_API_KEY === "") { return; } -const imageModel = ea.plugin.settings.openAIDefaultImageGenerationModel; let userPrompt = settings["User Prompt"] ?? ""; let agentTask = settings["Agent's Task"]; let imageSize = settings["Image Size"]??"1024x1024"; -const validSizes = imageModel === "dall-e-2" - ? [`256x256`, `512x512`, `1024x1024`] - : (imageModel === "dall-e-3" - ? [`1024x1024`, `1792x1024`, `1024x1792`] - : [`1024x1024`]) -if(!validSizes.includes(imageSize)) { - imageSize = "1024x1024"; - dirty = true; -} if(!systemPrompts.hasOwnProperty(agentTask)) { agentTask = Object.keys(systemPrompts)[0]; } +let imageModel, valideSizes; + +const setImageModelAndSizes = () => { + imageModel = systemPrompts[agentTask].type === "image-edit" + ? "dall-e-2" + : ea.plugin.settings.openAIDefaultImageGenerationModel; + validSizes = imageModel === "dall-e-2" + ? [`256x256`, `512x512`, `1024x1024`] + : (imageModel === "dall-e-3" + ? [`1024x1024`, `1792x1024`, `1024x1792`] + : [`1024x1024`]) + if(!validSizes.includes(imageSize)) { + imageSize = "1024x1024"; + dirty = true; + } +} +setImageModelAndSizes(); // -------------------------------------- // Generate Image Blob From Selected Excalidraw Elements @@ -120,51 +143,114 @@ const calculateImageScale = (elements) => { ); } -const generateCanvasDataURL = async (view, makeSquare=false) => { +const createMask = async (dataURL) => { + return new Promise((resolve, reject) => { + const img = new Image(); + + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = img.width; + canvas.height = img.height; + + const ctx = canvas.getContext('2d'); + ctx.drawImage(img, 0, 0); + + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const data = imageData.data; + + for (let i = 0; i < data.length; i += 4) { + // If opaque (alpha > 0), make it transparent + if (data[i + 3] > 0) { + data[i + 3] = 0; // Set alpha to 0 (transparent) + } else if (data[i + 3] === 0) { + // If fully transparent, make it red + data[i] = 255; // Red + data[i + 1] = 0; // Green + data[i + 2] = 0; // Blue + data[i + 3] = 255; // make it opaque + } + } + + ctx.putImageData(imageData, 0, 0); + const maskDataURL = canvas.toDataURL(); + + resolve(maskDataURL); + }; + + img.onerror = error => { + reject(error); + }; + + img.src = dataURL; + }); +} + +//https://platform.openai.com/docs/api-reference/images/createEdit +//dall-e-2 image edit only works on square images +//if targetDalleImageEdit === true then the image and the mask will be returned in two separate dataURLs +let squareBB; + +const generateCanvasDataURL = async (view, targetDalleImageEdit=false) => { + let PADDING = 5; await view.forceSave(true); //to ensure recently embedded PNG and other images are saved to file const viewElements = ea.getViewSelectedElements(); if(viewElements.length === 0) { - return; + return {imageDataURL: null, maskDataURL: null} ; } ea.copyViewElementsToEAforEditing(viewElements, true); //copying the images objects over to EA for PNG generation - if(makeSquare) { - const bb = ea.getBoundingBox(viewElements); + let maskDataURL; + const loader = ea.getEmbeddedFilesLoader(false); + let scale = calculateImageScale(ea.getElements()); + const bb = ea.getBoundingBox(viewElements); + if(ea.getElements() + .filter(el=>el.type==="image") + .some(el=>Math.round(el.width) === Math.round(bb.width) && Math.round(el.height) === Math.round(bb.height)) + ) { PADDING = 0; } + + let exportSettings = {withBackground: true, withTheme: true}; + + if(targetDalleImageEdit) { + PADDING = 0; const strokeColor = ea.style.strokeColor; const backgroundColor = ea.style.backgroundColor; ea.style.backgroundColor = "transparent"; ea.style.strokeColor = "transparent"; - //deliberately not adding a rect if width === height + let rectID; if(bb.height > bb.width) { - ea.addRect(bb.topX-(bb.height-bb.width)/2, bb.topY,bb.height, bb.height); + rectID = ea.addRect(bb.topX-(bb.height-bb.width)/2, bb.topY,bb.height, bb.height); } if(bb.width > bb.height) { - ea.addRect(bb.topX, bb.topY-(bb.width-bb.height)/2,bb.width, bb.width); + rectID = ea.addRect(bb.topX, bb.topY-(bb.width-bb.height)/2,bb.width, bb.width); } + if(bb.height === bb.width) { + rectID = ea.addRect(bb.topX, bb.topY, bb.width, bb.height); + } + const rect = ea.getElement(rectID); + squareBB = {topX: rect.x-PADDING, topY: rect.y-PADDING, width: rect.width + 2*PADDING, height: rect.height + 2*PADDING}; ea.style.strokeColor = strokeColor; ea.style.backgroundColor = backgroundColor; - } - const scale = calculateImageScale(ea.getElements()); + ea.getElements().filter(el=>el.type === "image").forEach(el=>{el.isDeleted = true}); - const loader = ea.getEmbeddedFilesLoader(false); - const exportSettings = { - withBackground: true, - withTheme: true, - }; - - const dataURL = - await ea.createPNGBase64( - null, - scale, - exportSettings, - loader, - "light", + dalleWidth = parseInt(imageSize.split("x")[0]); + scale = dalleWidth/squareBB.width; + exportSettings = {withBackground: false, withTheme: true}; + maskDataURL= await ea.createPNGBase64( + null, scale, exportSettings, loader, "light", PADDING ); + maskDataURL = await createMask(maskDataURL) + ea.getElements().filter(el=>el.type === "image").forEach(el=>{el.isDeleted = false}); + ea.getElements().filter(el=>el.type !== "image" && el.id !== rectID).forEach(el=>{el.isDeleted = true}); + } + + const imageDataURL = await ea.createPNGBase64( + null, scale, exportSettings, loader, "light", PADDING + ); ea.clear(); - return dataURL; + return {imageDataURL, maskDataURL}; } -let imageDataURL = await generateCanvasDataURL(ea.targetView); +let {imageDataURL, maskDataURL} = await generateCanvasDataURL(ea.targetView, systemPrompts[agentTask].type === "image-edit"); // -------------------------------------- // Support functions - embeddable spinner and error @@ -242,6 +328,7 @@ const generateImage = async(text, spinnerID, bb) => { n:1, }, }; + const result = await ea.postOpenAI(requestObject); console.log({result, json:result?.json}); @@ -257,8 +344,8 @@ const generateImage = async(text, spinnerID, bb) => { const revisedPrompt = result.json.data[0].revised_prompt; if(revisedPrompt) { ea.style.fontSize = 16; - const rectID = ea.addText(imageEl.x, imageEl.y + imageEl.height + 50, revisedPrompt, { - width: imageEl.width, + const rectID = ea.addText(imageEl.x+15, imageEl.y + imageEl.height + 50, revisedPrompt, { + width: imageEl.width-30, textAlign: "center", textVerticalAlign: "top", box: true, @@ -285,12 +372,17 @@ const run = async (text) => { const systemPrompt = systemPrompts[agentTask]; const outputType = outputTypes[systemPrompt.type]; const isImageGenRequest = outputType.blocktype === "image"; - - const requestObject = { - ...imageDataURL ? {image: imageDataURL} : {}, - ...(text && text.trim() !== "") ? {text} : {}, - systemPrompt: systemPrompt.prompt, - instruction: outputType.instruction, + const isImageEditRequest = systemPrompt.type === "image-edit"; + + if(isImageEditRequest) { + if(!text) { + new Notice("You must provide a text prompt with instructions for how the image should be modified"); + return; + } + if(!imageDataURL || !maskDataURL) { + new Notice("You must provide an image and a mask"); + return; + } } //place spinner next to selected elements @@ -312,11 +404,29 @@ const run = async (text) => { isEACompleted = true; }); - if(isImageGenRequest && !systemPrompt.prompt) { + if(isImageGenRequest && !systemPrompt.prompt && !isImageEditRequest) { generateImage(text,spinnerID,bb); return; } - + + const requestObject = isImageEditRequest + ? { + ...imageDataURL ? {image: imageDataURL} : {}, + ...(text && text.trim() !== "") ? {text} : {}, + imageGenerationProperties: { + size: imageSize, + //quality: "standard", //not supported by dall-e-2 + n:1, + mask: maskDataURL, + }, + } + : { + ...imageDataURL ? {image: imageDataURL} : {}, + ...(text && text.trim() !== "") ? {text} : {}, + systemPrompt: systemPrompt.prompt, + instruction: outputType.instruction, + } + //Get result from GPT const result = await ea.postOpenAI(requestObject); console.log({result, json:result?.json}); @@ -330,7 +440,25 @@ const run = async (text) => { await errorMessage(spinnerID, "Unexpected issue with ExcalidrawAutomate"); return; } - + + if(isImageEditRequest) { + if(!result?.json?.data?.[0]?.url) { + await errorMessage(spinnerID, result?.json?.error?.message); + return; + } + + const spinner = ea.getElement(spinnerID) + spinner.isDeleted = true; + const imageID = await ea.addImage(spinner.x, spinner.y, result.json.data[0].url); + await ea.addElementsToView(false, true, true); + ea.getExcalidrawAPI().setToast({ + message: IMAGE_WARNING, + duration: 15000, + closable: true + }); + return; + } + if(!result?.json?.hasOwnProperty("choices")) { await errorMessage(spinnerID, result?.json?.error?.message); return; @@ -389,8 +517,27 @@ const run = async (text) => { // -------------------------------------- // User Interface // -------------------------------------- +let previewDiv; const fragWithHTML = (html) => createFragment((frag) => (frag.createDiv().innerHTML = html)); -const isImageGenerationTask = () => systemPrompts[agentTask].type === "image-gen"; +const isImageGenerationTask = () => systemPrompts[agentTask].type === "image-gen" || systemPrompts[agentTask].type === "image-edit"; +const addPreviewImage = () => { + if(!previewDiv) return; + previewDiv.empty(); + previewDiv.createEl("img",{ + attr: { + style: `max-width: 100%;max-height: 30vh;`, + src: imageDataURL, + } + }); + if(maskDataURL) { + previewDiv.createEl("img",{ + attr: { + style: `max-width: 100%;max-height: 30vh;`, + src: maskDataURL, + } + }); + } +} const configModal = new ea.obsidian.Modal(app); configModal.modalEl.style.width="100%"; @@ -400,19 +547,32 @@ configModal.onOpen = async () => { const contentEl = configModal.contentEl; contentEl.createEl("h1", {text: "ExcaliAI"}); - let systemPromptTextArea, systemPromptDiv, imageSizeSetting; + let systemPromptTextArea, systemPromptDiv, imageSizeSetting, imageSizeSettingDropdown, helpEl; new ea.obsidian.Setting(contentEl) - .setName("Select Prompt") + .setName("What would you like to do?") .addDropdown(dropdown=>{ Object.keys(systemPrompts).forEach(key=>dropdown.addOption(key,key)); dropdown .setValue(agentTask) - .onChange(value => { + .onChange(async (value) => { dirty = true; + const prevTask = agentTask; agentTask = value; + if( + (systemPrompts[prevTask].type === "image-edit" && systemPrompts[value].type !== "image-edit") || + (systemPrompts[prevTask].type !== "image-edit" && systemPrompts[value].type === "image-edit") + ) { + ({imageDataURL, maskDataURL} = await generateCanvasDataURL(ea.targetView, systemPrompts[value].type === "image-edit")); + addPreviewImage(); + setImageModelAndSizes(); + while (imageSizeSettingDropdown.selectEl.options.length > 0) { imageSizeSettingDropdown.selectEl.remove(0); } + validSizes.forEach(size=>imageSizeSettingDropdown.addOption(size,size)); + imageSizeSettingDropdown.setValue(imageSize); + } imageSizeSetting.settingEl.style.display = isImageGenerationTask() ? "" : "none"; const prompt = systemPrompts[value].prompt; + helpEl.innerHTML = `Help: ` + systemPrompts[value].help; if(prompt) { systemPromptDiv.style.display = ""; systemPromptTextArea.setValue(systemPrompts[value].prompt); @@ -422,6 +582,9 @@ configModal.onOpen = async () => { }); }) + helpEl = contentEl.createEl("p"); + helpEl.innerHTML = `Help: ` + systemPrompts[agentTask].help; + systemPromptDiv = contentEl.createDiv(); systemPromptDiv.createEl("h4", {text: "Customize System Prompt"}); systemPromptDiv.createEl("span", {text: "Unless you know what you are doing I do not recommend changing the system prompt"}) @@ -461,12 +624,17 @@ configModal.onOpen = async () => { .setDesc(fragWithHTML("⚠️ Important ⚠️: " + IMAGE_WARNING)) .addDropdown(dropdown=>{ validSizes.forEach(size=>dropdown.addOption(size,size)); + imageSizeSettingDropdown = dropdown; dropdown - .setValue(imageSize) - .onChange(value => { - dirty = true; - imageSize = value; - }); + .setValue(imageSize) + .onChange(async (value) => { + dirty = true; + imageSize = value; + if(systemPrompts[agentTask].type === "image-edit") { + ({imageDataURL, maskDataURL} = await generateCanvasDataURL(ea.targetView, true)); + addPreviewImage(); + } + }); }) imageSizeSetting.settingEl.style.display = isImageGenerationTask() ? "" : "none"; @@ -476,14 +644,9 @@ configModal.onOpen = async () => { style: "text-align: center;", } }); - previewImg = previewDiv.createEl("img",{ - attr: { - style: `max-width: 100%;max-height: 30vh;`, - src: imageDataURL, - } - }); + addPreviewImage(); } else { - contentEl.createEl("h4", {text: "No elements are selected"}); + contentEl.createEl("h4", {text: "No elements are selected from your canvas"}); contentEl.createEl("span", {text: "Because there are no Excalidraw elements selected on the canvas, only the text prompt will be sent to OpenAI."}); } diff --git a/ea-scripts/directory-info.json b/ea-scripts/directory-info.json index 8621e1d..b4f292b 100644 --- a/ea-scripts/directory-info.json +++ b/ea-scripts/directory-info.json @@ -1 +1 @@ -[{"fname":"Mindmap connector.md","mtime":1658686599427},{"fname":"Mindmap connector.svg","mtime":1658686599427},{"fname":"Add Connector Point.md","mtime":1645305706000},{"fname":"Add Connector Point.svg","mtime":1645944722000},{"fname":"Add Link to Existing File and Open.md","mtime":1647807918345},{"fname":"Add Link to Existing File and Open.svg","mtime":1645964261000},{"fname":"Add Link to New Page and Open.md","mtime":1654168862138},{"fname":"Add Link to New Page and Open.svg","mtime":1645960639000},{"fname":"Add Next Step in Process.md","mtime":1688304760357},{"fname":"Add Next Step in Process.svg","mtime":1645960639000},{"fname":"Box Each Selected Groups.md","mtime":1645305706000},{"fname":"Box Each Selected Groups.svg","mtime":1645967510000},{"fname":"Box Selected Elements.md","mtime":1645305706000},{"fname":"Box Selected Elements.svg","mtime":1645960639000},{"fname":"Change shape of selected elements.md","mtime":1652701169236},{"fname":"Change shape of selected elements.svg","mtime":1645960775000},{"fname":"Connect elements.md","mtime":1645305706000},{"fname":"Connect elements.svg","mtime":1645960639000},{"fname":"Convert freedraw to line.md","mtime":1645305706000},{"fname":"Convert freedraw to line.svg","mtime":1645960639000},{"fname":"Convert selected text elements to sticky notes.md","mtime":1670169501383},{"fname":"Convert selected text elements to sticky notes.svg","mtime":1645960639000},{"fname":"Convert text to link with folder and alias.md","mtime":1641639819000},{"fname":"Convert text to link with folder and alias.svg","mtime":1645960639000},{"fname":"Copy Selected Element Styles to Global.md","mtime":1642232088000},{"fname":"Copy Selected Element Styles to Global.svg","mtime":1645960639000},{"fname":"Create new markdown file and embed into active drawing.md","mtime":1640866935000},{"fname":"Create new markdown file and embed into active drawing.svg","mtime":1645960639000},{"fname":"Darken background color.md","mtime":1663059051059},{"fname":"Darken background color.svg","mtime":1645960639000},{"fname":"Elbow connectors.md","mtime":1671126911490},{"fname":"Elbow connectors.svg","mtime":1645960639000},{"fname":"Expand rectangles horizontally keep text centered.md","mtime":1646563692000},{"fname":"Expand rectangles horizontally keep text centered.svg","mtime":1645967510000},{"fname":"Expand rectangles horizontally.md","mtime":1644950235000},{"fname":"Expand rectangles horizontally.svg","mtime":1645967510000},{"fname":"Expand rectangles vertically keep text centered.md","mtime":1646563692000},{"fname":"Expand rectangles vertically keep text centered.svg","mtime":1645967510000},{"fname":"Expand rectangles vertically.md","mtime":1658686599427},{"fname":"Expand rectangles vertically.svg","mtime":1645967510000},{"fname":"Fixed horizontal distance between centers.md","mtime":1646743234000},{"fname":"Fixed horizontal distance between centers.svg","mtime":1645960639000},{"fname":"Fixed inner distance.md","mtime":1646743234000},{"fname":"Fixed inner distance.svg","mtime":1645960639000},{"fname":"Fixed spacing.md","mtime":1646743234000},{"fname":"Fixed spacing.svg","mtime":1645967510000},{"fname":"Fixed vertical distance between centers.md","mtime":1646743234000},{"fname":"Fixed vertical distance between centers.svg","mtime":1645967510000},{"fname":"Fixed vertical distance.md","mtime":1646743234000},{"fname":"Fixed vertical distance.svg","mtime":1645967510000},{"fname":"Lighten background color.md","mtime":1663059051059},{"fname":"Lighten background color.svg","mtime":1645959546000},{"fname":"Modify background color opacity.md","mtime":1644924415000},{"fname":"Modify background color opacity.svg","mtime":1645944722000},{"fname":"Normalize Selected Arrows.md","mtime":1670403743278},{"fname":"Normalize Selected Arrows.svg","mtime":1645960639000},{"fname":"Organic Line.md","mtime":1672920172531},{"fname":"Organic Line.svg","mtime":1645964261000},{"fname":"Organic Line Legacy.md","mtime":1690607372668},{"fname":"Organic Line Legacy.svg","mtime":1690607372668},{"fname":"README.md","mtime":1645175700000},{"fname":"Repeat Elements.md","mtime":1663059051059},{"fname":"Repeat Elements.svg","mtime":1645960639000},{"fname":"Reverse arrows.md","mtime":1645305706000},{"fname":"Reverse arrows.svg","mtime":1645960639000},{"fname":"Scribble Helper.md","mtime":1682228345043},{"fname":"Scribble Helper.svg","mtime":1645944722000},{"fname":"Select Elements of Type.md","mtime":1643464321000},{"fname":"Select Elements of Type.svg","mtime":1645960639000},{"fname":"Set Dimensions.md","mtime":1645305706000},{"fname":"Set Dimensions.svg","mtime":1645944722000},{"fname":"Set Font Family.md","mtime":1645305706000},{"fname":"Set Font Family.svg","mtime":1645944722000},{"fname":"Set Grid.md","mtime":1693725826368},{"fname":"Set Grid.svg","mtime":1645960639000},{"fname":"Set Link Alias.md","mtime":1645305706000},{"fname":"Set Link Alias.svg","mtime":1645960639000},{"fname":"Set Stroke Width of Selected Elements.md","mtime":1645305706000},{"fname":"Set Stroke Width of Selected Elements.svg","mtime":1645960639000},{"fname":"Set Text Alignment.md","mtime":1645305706000},{"fname":"Set Text Alignment.svg","mtime":1645960639000},{"fname":"Set background color of unclosed line object by adding a shadow clone.md","mtime":1681665030892},{"fname":"Set background color of unclosed line object by adding a shadow clone.svg","mtime":1645960639000},{"fname":"Split text by lines.md","mtime":1645305706000},{"fname":"Split text by lines.svg","mtime":1645944722000},{"fname":"Zoom to Fit Selected Elements.md","mtime":1640770602000},{"fname":"Zoom to Fit Selected Elements.svg","mtime":1645960639000},{"fname":"directory-info.json","mtime":1646583437000},{"fname":"index-new.md","mtime":1645986149000},{"fname":"index.md","mtime":1645175700000},{"fname":"Grid Selected Images.md","mtime":1701630797839},{"fname":"Grid Selected Images.svg","mtime":1649614401982},{"fname":"Palette loader.md","mtime":1686511890942},{"fname":"Palette loader.svg","mtime":1649614401982},{"fname":"Rename Image.md","mtime":1663678478785},{"fname":"Rename Image.svg","mtime":1663678478785},{"fname":"Text Arch.md","mtime":1664095143846},{"fname":"Text Arch.svg","mtime":1670403743278},{"fname":"Deconstruct selected elements into new drawing.md","mtime":1693733190088},{"fname":"Deconstruct selected elements into new drawing.svg","mtime":1668541145255},{"fname":"Slideshow.md","mtime":1700511998048},{"fname":"Slideshow.svg","mtime":1670017348333},{"fname":"Auto Layout.md","mtime":1670403743278},{"fname":"Auto Layout.svg","mtime":1670175947081},{"fname":"Uniform size.md","mtime":1670175947081},{"fname":"Uniform size.svg","mtime":1670175947081},{"fname":"Mindmap format.md","mtime":1684484694228},{"fname":"Mindmap format.svg","mtime":1674944958059},{"fname":"Text to Sticky Notes.md","mtime":1678537561724},{"fname":"Text to Sticky Notes.svg","mtime":1678537561724},{"fname":"Folder Note Core - Make Current Drawing a Folder.md","mtime":1678973697470},{"fname":"Folder Note Core - Make Current Drawing a Folder.svg","mtime":1678973697470},{"fname":"Invert colors.md","mtime":1678973697470},{"fname":"Invert colors.svg","mtime":1678973697470},{"fname":"Auto Draw for Pen.md","mtime":1680418321236},{"fname":"Auto Draw for Pen.svg","mtime":1680418321236},{"fname":"Hardware Eraser Support.md","mtime":1680418321236},{"fname":"Hardware Eraser Support.svg","mtime":1680418321236},{"fname":"PDF Page Text to Clipboard.md","mtime":1683984041712},{"fname":"PDF Page Text to Clipboard.svg","mtime":1680418321236},{"fname":"Excalidraw Collaboration Frame.md","mtime":1687881495985},{"fname":"Excalidraw Collaboration Frame.svg","mtime":1687881495985},{"fname":"Create DrawIO file.md","mtime":1688243858267},{"fname":"Create DrawIO file.svg","mtime":1688243858267},{"fname":"Ellipse Selected Elements.md","mtime":1690131476331},{"fname":"Ellipse Selected Elements.svg","mtime":1690131476331},{"fname":"Select Similar Elements.md","mtime":1691270949338},{"fname":"Select Similar Elements.svg","mtime":1691270949338},{"fname":"Toggle Grid.md","mtime":1692125382945},{"fname":"Toggle Grid.svg","mtime":1692124753386},{"fname":"Split Ellipse.md","mtime":1693134104356},{"fname":"Split Ellipse.svg","mtime":1693134104356},{"fname":"Text Aura.md","mtime":1693731979540},{"fname":"Text Aura.svg","mtime":1693731979540},{"fname":"Boolean Operations.md","mtime":1695746839537},{"fname":"Boolean Operations.svg","mtime":1695746839537},{"fname":"Concatenate lines.md","mtime":1696175301525},{"fname":"Concatenate lines.svg","mtime":1696175301525},{"fname":"GPT-Draw-a-UI.md","mtime":1703176663558},{"fname":"GPT-Draw-a-UI.svg","mtime":1700511998048},{"fname":"ExcaliAI.md","mtime":1703176663558},{"fname":"ExcaliAI.svg","mtime":1701011028767},{"fname":"Repeat Texts.md","mtime":1701969627758},{"fname":"Repeat Texts.svg","mtime":1701969627758},{"fname":"Relative Font Size Cycle.md","mtime":1701969627758},{"fname":"Relative Font Size Cycle.svg","mtime":1701969627758},{"fname":"Golden Ratio.md","mtime":1702812404286},{"fname":"Golden Ratio.svg","mtime":1702812404286}] \ No newline at end of file +[{"fname":"Mindmap connector.md","mtime":1658686599427},{"fname":"Mindmap connector.svg","mtime":1658686599427},{"fname":"Add Connector Point.md","mtime":1645305706000},{"fname":"Add Connector Point.svg","mtime":1645944722000},{"fname":"Add Link to Existing File and Open.md","mtime":1647807918345},{"fname":"Add Link to Existing File and Open.svg","mtime":1645964261000},{"fname":"Add Link to New Page and Open.md","mtime":1654168862138},{"fname":"Add Link to New Page and Open.svg","mtime":1645960639000},{"fname":"Add Next Step in Process.md","mtime":1688304760357},{"fname":"Add Next Step in Process.svg","mtime":1645960639000},{"fname":"Box Each Selected Groups.md","mtime":1645305706000},{"fname":"Box Each Selected Groups.svg","mtime":1645967510000},{"fname":"Box Selected Elements.md","mtime":1645305706000},{"fname":"Box Selected Elements.svg","mtime":1645960639000},{"fname":"Change shape of selected elements.md","mtime":1652701169236},{"fname":"Change shape of selected elements.svg","mtime":1645960775000},{"fname":"Connect elements.md","mtime":1645305706000},{"fname":"Connect elements.svg","mtime":1645960639000},{"fname":"Convert freedraw to line.md","mtime":1645305706000},{"fname":"Convert freedraw to line.svg","mtime":1645960639000},{"fname":"Convert selected text elements to sticky notes.md","mtime":1670169501383},{"fname":"Convert selected text elements to sticky notes.svg","mtime":1645960639000},{"fname":"Convert text to link with folder and alias.md","mtime":1641639819000},{"fname":"Convert text to link with folder and alias.svg","mtime":1645960639000},{"fname":"Copy Selected Element Styles to Global.md","mtime":1642232088000},{"fname":"Copy Selected Element Styles to Global.svg","mtime":1645960639000},{"fname":"Create new markdown file and embed into active drawing.md","mtime":1640866935000},{"fname":"Create new markdown file and embed into active drawing.svg","mtime":1645960639000},{"fname":"Darken background color.md","mtime":1663059051059},{"fname":"Darken background color.svg","mtime":1645960639000},{"fname":"Elbow connectors.md","mtime":1671126911490},{"fname":"Elbow connectors.svg","mtime":1645960639000},{"fname":"Expand rectangles horizontally keep text centered.md","mtime":1646563692000},{"fname":"Expand rectangles horizontally keep text centered.svg","mtime":1645967510000},{"fname":"Expand rectangles horizontally.md","mtime":1644950235000},{"fname":"Expand rectangles horizontally.svg","mtime":1645967510000},{"fname":"Expand rectangles vertically keep text centered.md","mtime":1646563692000},{"fname":"Expand rectangles vertically keep text centered.svg","mtime":1645967510000},{"fname":"Expand rectangles vertically.md","mtime":1658686599427},{"fname":"Expand rectangles vertically.svg","mtime":1645967510000},{"fname":"Fixed horizontal distance between centers.md","mtime":1646743234000},{"fname":"Fixed horizontal distance between centers.svg","mtime":1645960639000},{"fname":"Fixed inner distance.md","mtime":1646743234000},{"fname":"Fixed inner distance.svg","mtime":1645960639000},{"fname":"Fixed spacing.md","mtime":1646743234000},{"fname":"Fixed spacing.svg","mtime":1645967510000},{"fname":"Fixed vertical distance between centers.md","mtime":1646743234000},{"fname":"Fixed vertical distance between centers.svg","mtime":1645967510000},{"fname":"Fixed vertical distance.md","mtime":1646743234000},{"fname":"Fixed vertical distance.svg","mtime":1645967510000},{"fname":"Lighten background color.md","mtime":1663059051059},{"fname":"Lighten background color.svg","mtime":1645959546000},{"fname":"Modify background color opacity.md","mtime":1644924415000},{"fname":"Modify background color opacity.svg","mtime":1645944722000},{"fname":"Normalize Selected Arrows.md","mtime":1670403743278},{"fname":"Normalize Selected Arrows.svg","mtime":1645960639000},{"fname":"Organic Line.md","mtime":1672920172531},{"fname":"Organic Line.svg","mtime":1645964261000},{"fname":"Organic Line Legacy.md","mtime":1690607372668},{"fname":"Organic Line Legacy.svg","mtime":1690607372668},{"fname":"README.md","mtime":1645175700000},{"fname":"Repeat Elements.md","mtime":1663059051059},{"fname":"Repeat Elements.svg","mtime":1645960639000},{"fname":"Reverse arrows.md","mtime":1645305706000},{"fname":"Reverse arrows.svg","mtime":1645960639000},{"fname":"Scribble Helper.md","mtime":1682228345043},{"fname":"Scribble Helper.svg","mtime":1645944722000},{"fname":"Select Elements of Type.md","mtime":1643464321000},{"fname":"Select Elements of Type.svg","mtime":1645960639000},{"fname":"Set Dimensions.md","mtime":1645305706000},{"fname":"Set Dimensions.svg","mtime":1645944722000},{"fname":"Set Font Family.md","mtime":1645305706000},{"fname":"Set Font Family.svg","mtime":1645944722000},{"fname":"Set Grid.md","mtime":1693725826368},{"fname":"Set Grid.svg","mtime":1645960639000},{"fname":"Set Link Alias.md","mtime":1645305706000},{"fname":"Set Link Alias.svg","mtime":1645960639000},{"fname":"Set Stroke Width of Selected Elements.md","mtime":1645305706000},{"fname":"Set Stroke Width of Selected Elements.svg","mtime":1645960639000},{"fname":"Set Text Alignment.md","mtime":1645305706000},{"fname":"Set Text Alignment.svg","mtime":1645960639000},{"fname":"Set background color of unclosed line object by adding a shadow clone.md","mtime":1681665030892},{"fname":"Set background color of unclosed line object by adding a shadow clone.svg","mtime":1645960639000},{"fname":"Split text by lines.md","mtime":1645305706000},{"fname":"Split text by lines.svg","mtime":1645944722000},{"fname":"Zoom to Fit Selected Elements.md","mtime":1640770602000},{"fname":"Zoom to Fit Selected Elements.svg","mtime":1645960639000},{"fname":"directory-info.json","mtime":1646583437000},{"fname":"index-new.md","mtime":1645986149000},{"fname":"index.md","mtime":1645175700000},{"fname":"Grid Selected Images.md","mtime":1701630797839},{"fname":"Grid Selected Images.svg","mtime":1649614401982},{"fname":"Palette loader.md","mtime":1686511890942},{"fname":"Palette loader.svg","mtime":1649614401982},{"fname":"Rename Image.md","mtime":1663678478785},{"fname":"Rename Image.svg","mtime":1663678478785},{"fname":"Text Arch.md","mtime":1664095143846},{"fname":"Text Arch.svg","mtime":1670403743278},{"fname":"Deconstruct selected elements into new drawing.md","mtime":1693733190088},{"fname":"Deconstruct selected elements into new drawing.svg","mtime":1668541145255},{"fname":"Slideshow.md","mtime":1700511998048},{"fname":"Slideshow.svg","mtime":1670017348333},{"fname":"Auto Layout.md","mtime":1670403743278},{"fname":"Auto Layout.svg","mtime":1670175947081},{"fname":"Uniform size.md","mtime":1670175947081},{"fname":"Uniform size.svg","mtime":1670175947081},{"fname":"Mindmap format.md","mtime":1684484694228},{"fname":"Mindmap format.svg","mtime":1674944958059},{"fname":"Text to Sticky Notes.md","mtime":1678537561724},{"fname":"Text to Sticky Notes.svg","mtime":1678537561724},{"fname":"Folder Note Core - Make Current Drawing a Folder.md","mtime":1678973697470},{"fname":"Folder Note Core - Make Current Drawing a Folder.svg","mtime":1678973697470},{"fname":"Invert colors.md","mtime":1678973697470},{"fname":"Invert colors.svg","mtime":1678973697470},{"fname":"Auto Draw for Pen.md","mtime":1680418321236},{"fname":"Auto Draw for Pen.svg","mtime":1680418321236},{"fname":"Hardware Eraser Support.md","mtime":1680418321236},{"fname":"Hardware Eraser Support.svg","mtime":1680418321236},{"fname":"PDF Page Text to Clipboard.md","mtime":1683984041712},{"fname":"PDF Page Text to Clipboard.svg","mtime":1680418321236},{"fname":"Excalidraw Collaboration Frame.md","mtime":1687881495985},{"fname":"Excalidraw Collaboration Frame.svg","mtime":1687881495985},{"fname":"Create DrawIO file.md","mtime":1688243858267},{"fname":"Create DrawIO file.svg","mtime":1688243858267},{"fname":"Ellipse Selected Elements.md","mtime":1690131476331},{"fname":"Ellipse Selected Elements.svg","mtime":1690131476331},{"fname":"Select Similar Elements.md","mtime":1691270949338},{"fname":"Select Similar Elements.svg","mtime":1691270949338},{"fname":"Toggle Grid.md","mtime":1692125382945},{"fname":"Toggle Grid.svg","mtime":1692124753386},{"fname":"Split Ellipse.md","mtime":1693134104356},{"fname":"Split Ellipse.svg","mtime":1693134104356},{"fname":"Text Aura.md","mtime":1693731979540},{"fname":"Text Aura.svg","mtime":1693731979540},{"fname":"Boolean Operations.md","mtime":1695746839537},{"fname":"Boolean Operations.svg","mtime":1695746839537},{"fname":"Concatenate lines.md","mtime":1696175301525},{"fname":"Concatenate lines.svg","mtime":1696175301525},{"fname":"GPT-Draw-a-UI.md","mtime":1703324727900},{"fname":"GPT-Draw-a-UI.svg","mtime":1700511998048},{"fname":"ExcaliAI.md","mtime":1703324727900},{"fname":"ExcaliAI.svg","mtime":1701011028767},{"fname":"Repeat Texts.md","mtime":1701969627758},{"fname":"Repeat Texts.svg","mtime":1701969627758},{"fname":"Relative Font Size Cycle.md","mtime":1701969627758},{"fname":"Relative Font Size Cycle.svg","mtime":1701969627758},{"fname":"Golden Ratio.md","mtime":1702812404286},{"fname":"Golden Ratio.svg","mtime":1702812404286}] \ No newline at end of file diff --git a/manifest.json b/manifest.json index 675fcad..f4929b4 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-excalidraw-plugin", "name": "Excalidraw", - "version": "2.0.11", + "version": "2.0.12", "minAppVersion": "1.1.6", "description": "An Obsidian plugin to edit and view Excalidraw drawings", "author": "Zsolt Viczian", diff --git a/package.json b/package.json index 34820af..8bde416 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "author": "", "license": "MIT", "dependencies": { - "@zsviczian/excalidraw": "0.17.1-obsidian-6", + "@zsviczian/excalidraw": "0.17.1-obsidian-8", "chroma-js": "^2.4.2", "clsx": "^2.0.0", "colormaster": "^1.2.1", diff --git a/rollup.config.js b/rollup.config.js index f261c5b..a936fff 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -74,14 +74,14 @@ const BUILD_CONFIG = { output: { dir: DIST_FOLDER, entryFileNames: 'main.js', - sourcemap: isProd?false:'inline', + //sourcemap: isProd?false:'inline', format: 'cjs', exports: 'default', }, plugins: [ typescript2({ tsconfig: isProd ? "tsconfig.json" : "tsconfig.dev.json", - inlineSources: !isProd + //inlineSources: !isProd }), replace({ preventAssignment: true, diff --git a/src/ExcalidrawView.ts b/src/ExcalidrawView.ts index 9fa4850..48ef79c 100644 --- a/src/ExcalidrawView.ts +++ b/src/ExcalidrawView.ts @@ -24,6 +24,7 @@ import { import { AppState, BinaryFileData, + DataURL, ExcalidrawImperativeAPI, Gesture, LibraryItems, @@ -52,6 +53,7 @@ import { restore, obsidianToExcalidrawMap, MAX_IMAGE_SIZE, + fileid, } from "./constants/constants"; import ExcalidrawPlugin from "./main"; import { @@ -135,6 +137,7 @@ import { getMermaidText, shouldRenderMermaid } from "./utils/MermaidUtils"; import { nanoid } from "nanoid"; import { CustomMutationObserver, isDebugMode } from "./utils/DebugHelper"; import { extractCodeBlocks, postOpenAI } from "./utils/AIUtils"; +import { Mutable } from "@zsviczian/excalidraw/types/excalidraw/utility-types"; declare const PLUGIN_VERSION:string; @@ -3985,6 +3988,52 @@ export default class ExcalidrawView extends TextFileView { } } + public getSingleSelectedImageWithURL(): {imageEl: ExcalidrawImageElement, embeddedFile: EmbeddedFile} { + if(!this.excalidrawAPI) return null; + const els = this.getViewSelectedElements().filter(el=>el.type==="image"); + if(els.length !== 1) { + return null; + } + const el = els[0] as ExcalidrawImageElement; + const imageFile = this.excalidrawData.getFile(el.fileId); + if(!imageFile.isHyperLink) return null; + return {imageEl: el, embeddedFile: imageFile}; + } + + public async convertImageElWithURLToLocalFile(data: {imageEl: ExcalidrawImageElement, embeddedFile: EmbeddedFile}) { + const {imageEl, embeddedFile} = data; + const imageDataURL = embeddedFile.getImage(false); + if(!imageDataURL && !imageDataURL.startsWith("data:")) { + new Notice("Image not found"); + return false; + } + const ea = getEA(this) as ExcalidrawAutomate; + ea.copyViewElementsToEAforEditing([imageEl]); + const eaEl = ea.getElement(imageEl.id) as Mutable; + eaEl.fileId = fileid() as FileId; + if(!eaEl.link) {eaEl.link = embeddedFile.hyperlink}; + let dataURL = embeddedFile.getImage(false); + if(!dataURL.startsWith("data:")) { + new Notice("Attempting to download image from URL. This may take a long while. The operation will time out after max 1 minute"); + dataURL = await getDataURLFromURL(dataURL, embeddedFile.mimeType, 30000); + if(!dataURL.startsWith("data:")) { + new Notice("Failed. Could not download image!"); + return false; + } + } + const files: BinaryFileData[] = []; + files.push({ + mimeType: embeddedFile.mimeType, + id: eaEl.fileId, + dataURL: dataURL as DataURL, + created: embeddedFile.mtime, + }); + const api = this.excalidrawAPI as ExcalidrawImperativeAPI; + api.addFiles(files); + await ea.addElementsToView(false,true); + new Notice("Image successfully converted to local file"); + } + private async instantiateExcalidraw( initdata: { elements: any, @@ -4086,6 +4135,19 @@ export default class ExcalidrawView extends TextFileView { } } + const img = this.getSingleSelectedImageWithURL(); + if(img) { + contextMenuActions.push([ + renderContextMenuAction( + t("CONVERT_URL_TO_FILE"), + () => { + this.convertImageElWithURLToLocalFile(img); + }, + onClose + ), + ]); + } + contextMenuActions.push([ renderContextMenuAction( t("UNIVERSAL_ADD_FILE"), diff --git a/src/customEmbeddable.tsx b/src/customEmbeddable.tsx index 136e983..4ff71c2 100644 --- a/src/customEmbeddable.tsx +++ b/src/customEmbeddable.tsx @@ -33,7 +33,8 @@ const getTheme = (view: ExcalidrawView, theme:string): string => view.excalidraw //required to control the video //-------------------------------------------------------------------------------- export const renderWebView = (src: string, view: ExcalidrawView, id: string, appState: UIAppState):JSX.Element =>{ - if(DEVICE.isDesktop) { + const isDataURL = src.startsWith("data:"); + if(DEVICE.isDesktop && !isDataURL) { return ( view.updateEmbeddableRef(id, ref)} @@ -55,11 +56,12 @@ export const renderWebView = (src: string, view: ExcalidrawView, id: string, app title="Excalidraw Embedded Content" allowFullScreen={true} allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" - src={src} + src={isDataURL ? null : src} style={{ overflow: "hidden", borderRadius: "var(--embeddable-radius)", }} + srcDoc={isDataURL ? atob(src.split(',')[1]) : null} /> ); } diff --git a/src/dialogs/Messages.ts b/src/dialogs/Messages.ts index b47091d..3a87eea 100644 --- a/src/dialogs/Messages.ts +++ b/src/dialogs/Messages.ts @@ -17,6 +17,20 @@ I develop this plugin as a hobby, spending my free time doing this. If you find
`, +"2.0.12":` +## Fixed +- Stencil library not working [#1516](https://github.com/zsviczian/obsidian-excalidraw-plugin/issues/1516), [#1517](https://github.com/zsviczian/obsidian-excalidraw-plugin/issues/1517) +- The new convert image from URL to Local File feature did not work in two situations: + - When the embedded image is downloaded from a very slow server (e.g. OpenAIs temp image server) + - On Android +- The postToOpenAI function did not work in all situations on Android. +- ExcaliAI wireframe to code did not display correctly on Android +- Tooltips kept popping up on Android. + +## New +- Added "Save image from URL to local file" to the right-click context menu +- Further [ExcaliAI](https://github.com/zsviczian/obsidian-excalidraw-plugin/blob/master/ea-scripts/ExcaliAI.md) improvements including support for image editing with image mask +`, "2.0.11":` ## Fixed - Resolved an Obsidian performance issue caused by simultaneous installations of Excalidraw and the Minimal theme. Optimized Excalidraw CSS loading into Obsidian since April 2021, resulting in noticeable performance improvements. ([#1456](https://github.com/zsviczian/obsidian-excalidraw-plugin/issues/1456)) diff --git a/src/dialogs/ScriptInstallPrompt.ts b/src/dialogs/ScriptInstallPrompt.ts index 68bcc96..81fa698 100644 --- a/src/dialogs/ScriptInstallPrompt.ts +++ b/src/dialogs/ScriptInstallPrompt.ts @@ -88,7 +88,8 @@ export class ScriptInstallPrompt extends Modal { this.close(); return; } - await MarkdownRenderer.renderMarkdown( + await MarkdownRenderer.render( + this.plugin.app, source, this.contentDiv, "", diff --git a/src/main.ts b/src/main.ts index f2bda16..ec28034 100644 --- a/src/main.ts +++ b/src/main.ts @@ -801,37 +801,10 @@ export default class ExcalidrawPlugin extends Plugin { checkCallback: (checking: boolean) => { const view = this.app.workspace.getActiveViewOfType(ExcalidrawView); if(!view) return false; - if(!view.excalidrawAPI) return false; - const els = view.getViewSelectedElements().filter(el=>el.type==="image"); - if(els.length !== 1) { - if(checking) return false; - new Notice("Select a single image element and try again"); - return false; - } - const el = els[0] as ExcalidrawImageElement; - const imageFile = view.excalidrawData.getFile(el.fileId); - if(!imageFile.isHyperLink) return false; + const img = view.getSingleSelectedImageWithURL(); + if(!img) return false; if(checking) return true; - const imageDataURL = imageFile.getImage(false); - if(!imageDataURL) { - new Notice("Image not found"); - return false; - } - const ea = getEA(view) as ExcalidrawAutomate; - ea.copyViewElementsToEAforEditing([el]); - const eaEl = ea.getElement(el.id) as Mutable; - eaEl.fileId = fileid() as FileId; - if(!eaEl.link) {eaEl.link = imageFile.hyperlink}; - const files: BinaryFileData[] = []; - files.push({ - mimeType: imageFile.mimeType, - id: eaEl.fileId, - dataURL: imageFile.getImage(false) as DataURL, - created: imageFile.mtime, - }); - const api = view.excalidrawAPI as ExcalidrawImperativeAPI; - api.addFiles(files); - ea.addElementsToView(false,true); + view.convertImageElWithURLToLocalFile(img); }, }); diff --git a/src/menu/ObsidianMenu.tsx b/src/menu/ObsidianMenu.tsx index d307e82..05278a7 100644 --- a/src/menu/ObsidianMenu.tsx +++ b/src/menu/ObsidianMenu.tsx @@ -2,7 +2,7 @@ import { AppState, ExcalidrawImperativeAPI } from "@zsviczian/excalidraw/types/e import clsx from "clsx"; import { TFile } from "obsidian"; import * as React from "react"; -import { VIEW_TYPE_EXCALIDRAW } from "src/constants/constants"; +import { DEVICE, VIEW_TYPE_EXCALIDRAW } from "src/constants/constants"; import { PenSettingsModal } from "src/dialogs/PenSettingsModal"; import ExcalidrawView from "src/ExcalidrawView"; import { PenStyle } from "src/PenTypes"; @@ -142,7 +142,7 @@ export class ObsidianMenu { >
-
+
{icon}
diff --git a/src/utils/AIUtils.ts b/src/utils/AIUtils.ts index b1fe5d2..c702e40 100644 --- a/src/utils/AIUtils.ts +++ b/src/utils/AIUtils.ts @@ -1,4 +1,4 @@ -import { AnyARecord } from "dns"; +import { DEVICE } from "../constants/constants"; import { Notice, RequestUrlResponse, requestUrl } from "obsidian"; import ExcalidrawPlugin from "src/main"; @@ -62,21 +62,15 @@ const handleImageEditPrompt = async (request: AIRequest) : Promise res.blob()) - .then((blob) => new File([blob], 'image.png', { type: 'image/png' })); - body.append('image', imageFile); + const imageBlob = await fetch(image).then((res) => res.blob()); + body.append('image', imageBlob, 'image.png'); } if (imageGenerationProperties.mask) { - const maskFile = await fetch(imageGenerationProperties.mask) - .then((res) => res.blob()) - .then((blob) => new File([blob], 'mask.png', { type: 'image/png' })); - body.append('mask', maskFile); + const maskBlob = await fetch(imageGenerationProperties.mask).then((res) => res.blob()); + body.append('mask', maskBlob, 'masik.png'); } - Boolean(image) && body.append("image", image); - imageGenerationProperties.size && body.append("size", imageGenerationProperties.size); imageGenerationProperties.n && body.append("n", String(imageGenerationProperties.n)); @@ -88,10 +82,8 @@ const handleImageEditPrompt = async (request: AIRequest) : Promise