feat: rewrite d2c to not require token (#8269)

This commit is contained in:
David Luzar 2024-08-20 18:06:22 +02:00 committed by GitHub
parent fb4bb29aa5
commit b5d7f5b4ba
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 282 additions and 564 deletions

View file

@ -45,7 +45,6 @@ import {
frameToolIcon,
mermaidLogoIcon,
laserPointerToolIcon,
OpenAIIcon,
MagicIcon,
} from "./icons";
import { KEYS } from "../keys";
@ -400,7 +399,7 @@ export const ShapesSwitcher = ({
>
{t("toolBar.mermaidToExcalidraw")}
</DropdownMenu.Item>
{app.props.aiEnabled !== false && (
{app.props.aiEnabled !== false && app.plugins.diagramToCode && (
<>
<DropdownMenu.Item
onSelect={() => app.onMagicframeToolSelect()}
@ -410,20 +409,6 @@ export const ShapesSwitcher = ({
{t("toolBar.magicframe")}
<DropdownMenu.Item.Badge>AI</DropdownMenu.Item.Badge>
</DropdownMenu.Item>
<DropdownMenu.Item
onSelect={() => {
trackEvent("ai", "open-settings", "d2c");
app.setOpenDialog({
name: "settings",
source: "settings",
tab: "diagram-to-code",
});
}}
icon={OpenAIIcon}
data-testid="toolbar-magicSettings"
>
{t("toolBar.magicSettings")}
</DropdownMenu.Item>
</>
)}
</DropdownMenu.Content>

View file

@ -83,7 +83,6 @@ import {
ZOOM_STEP,
POINTER_EVENTS,
TOOL_TYPE,
EDITOR_LS_KEYS,
isIOS,
supportsResizeObserver,
DEFAULT_COLLISION_THRESHOLD,
@ -183,6 +182,7 @@ import type {
ExcalidrawIframeElement,
ExcalidrawEmbeddableElement,
Ordered,
MagicGenerationData,
ExcalidrawNonSelectionElement,
ExcalidrawArrowElement,
} from "../element/types";
@ -257,6 +257,7 @@ import type {
UnsubscribeCallback,
EmbedsValidationStatus,
ElementsPendingErasure,
GenerateDiagramToCode,
NullableGridSize,
} from "../types";
import {
@ -405,13 +406,9 @@ import {
} from "../cursor";
import { Emitter } from "../emitter";
import { ElementCanvasButtons } from "../element/ElementCanvasButtons";
import type { MagicCacheData } from "../data/magic";
import { diagramToHTML } from "../data/magic";
import { exportToBlob } from "../../utils/export";
import { COLOR_PALETTE } from "../colors";
import { ElementCanvasButton } from "./MagicButton";
import { MagicIcon, copyIcon, fullscreenIcon } from "./icons";
import { EditorLocalStorage } from "../data/EditorLocalStorage";
import FollowMode from "./FollowMode/FollowMode";
import { Store, StoreAction } from "../store";
import { AnimationFrameHandler } from "../animation-frame-handler";
@ -1018,7 +1015,7 @@ class App extends React.Component<AppProps, AppState> {
if (isIframeElement(el)) {
src = null;
const data: MagicCacheData = (el.customData?.generationData ??
const data: MagicGenerationData = (el.customData?.generationData ??
this.magicGenerations.get(el.id)) || {
status: "error",
message: "No generation data",
@ -1559,10 +1556,6 @@ class App extends React.Component<AppProps, AppState> {
}
app={this}
isCollaborating={this.props.isCollaborating}
openAIKey={this.OPENAI_KEY}
isOpenAIKeyPersisted={this.OPENAI_KEY_IS_PERSISTED}
onOpenAIAPIKeyChange={this.onOpenAIKeyChange}
onMagicSettingsConfirm={this.onMagicSettingsConfirm}
>
{this.props.children}
</LayerUI>
@ -1807,7 +1800,7 @@ class App extends React.Component<AppProps, AppState> {
private magicGenerations = new Map<
ExcalidrawIframeElement["id"],
MagicCacheData
MagicGenerationData
>();
private updateMagicGeneration = ({
@ -1815,7 +1808,7 @@ class App extends React.Component<AppProps, AppState> {
data,
}: {
frameElement: ExcalidrawIframeElement;
data: MagicCacheData;
data: MagicGenerationData;
}) => {
if (data.status === "pending") {
// We don't wanna persist pending state to storage. It should be in-app
@ -1838,31 +1831,26 @@ class App extends React.Component<AppProps, AppState> {
this.triggerRender();
};
private getTextFromElements(elements: readonly ExcalidrawElement[]) {
const text = elements
.reduce((acc: string[], element) => {
if (isTextElement(element)) {
acc.push(element.text);
}
return acc;
}, [])
.join("\n\n");
return text;
public plugins: {
diagramToCode?: {
generate: GenerateDiagramToCode;
};
} = {};
public setPlugins(plugins: Partial<App["plugins"]>) {
Object.assign(this.plugins, plugins);
}
private async onMagicFrameGenerate(
magicFrame: ExcalidrawMagicFrameElement,
source: "button" | "upstream",
) {
if (!this.OPENAI_KEY) {
const generateDiagramToCode = this.plugins.diagramToCode?.generate;
if (!generateDiagramToCode) {
this.setState({
openDialog: {
name: "settings",
tab: "diagram-to-code",
source: "generation",
},
errorMessage: "No diagram to code plugin found",
});
trackEvent("ai", "generate (missing key)", "d2c");
return;
}
@ -1901,68 +1889,50 @@ class App extends React.Component<AppProps, AppState> {
selectedElementIds: { [frameElement.id]: true },
});
const blob = await exportToBlob({
elements: this.scene.getNonDeletedElements(),
appState: {
...this.state,
exportBackground: true,
viewBackgroundColor: this.state.viewBackgroundColor,
},
exportingFrame: magicFrame,
files: this.files,
});
const dataURL = await getDataURL(blob);
const textFromFrameChildren = this.getTextFromElements(magicFrameChildren);
trackEvent("ai", "generate (start)", "d2c");
try {
const { html } = await generateDiagramToCode({
frame: magicFrame,
children: magicFrameChildren,
});
const result = await diagramToHTML({
image: dataURL,
apiKey: this.OPENAI_KEY,
text: textFromFrameChildren,
theme: this.state.theme,
});
trackEvent("ai", "generate (success)", "d2c");
if (!result.ok) {
if (!html.trim()) {
this.updateMagicGeneration({
frameElement,
data: {
status: "error",
code: "ERR_OAI",
message: "Nothing genereated :(",
},
});
return;
}
const parsedHtml =
html.includes("<!DOCTYPE html>") && html.includes("</html>")
? html.slice(
html.indexOf("<!DOCTYPE html>"),
html.indexOf("</html>") + "</html>".length,
)
: html;
this.updateMagicGeneration({
frameElement,
data: { status: "done", html: parsedHtml },
});
} catch (error: any) {
trackEvent("ai", "generate (failed)", "d2c");
console.error(result.error);
this.updateMagicGeneration({
frameElement,
data: {
status: "error",
code: "ERR_OAI",
message: result.error?.message || "Unknown error during generation",
message: error.message || "Unknown error during generation",
},
});
return;
}
trackEvent("ai", "generate (success)", "d2c");
if (result.choices[0].message.content == null) {
this.updateMagicGeneration({
frameElement,
data: {
status: "error",
code: "ERR_OAI",
message: "Nothing genereated :(",
},
});
return;
}
const message = result.choices[0].message.content;
const html = message.slice(
message.indexOf("<!DOCTYPE html>"),
message.indexOf("</html>") + "</html>".length,
);
this.updateMagicGeneration({
frameElement,
data: { status: "done", html },
});
}
private onIframeSrcCopy(element: ExcalidrawIframeElement) {
@ -1976,70 +1946,7 @@ class App extends React.Component<AppProps, AppState> {
}
}
private OPENAI_KEY: string | null = EditorLocalStorage.get(
EDITOR_LS_KEYS.OAI_API_KEY,
);
private OPENAI_KEY_IS_PERSISTED: boolean =
EditorLocalStorage.has(EDITOR_LS_KEYS.OAI_API_KEY) || false;
private onOpenAIKeyChange = (
openAIKey: string | null,
shouldPersist: boolean,
) => {
this.OPENAI_KEY = openAIKey || null;
if (shouldPersist) {
const didPersist = EditorLocalStorage.set(
EDITOR_LS_KEYS.OAI_API_KEY,
openAIKey,
);
this.OPENAI_KEY_IS_PERSISTED = didPersist;
} else {
this.OPENAI_KEY_IS_PERSISTED = false;
}
};
private onMagicSettingsConfirm = (
apiKey: string,
shouldPersist: boolean,
source: "tool" | "generation" | "settings",
) => {
this.OPENAI_KEY = apiKey || null;
this.onOpenAIKeyChange(this.OPENAI_KEY, shouldPersist);
if (source === "settings") {
return;
}
const selectedElements = this.scene.getSelectedElements({
selectedElementIds: this.state.selectedElementIds,
});
if (apiKey) {
if (selectedElements.length) {
this.onMagicframeToolSelect();
} else {
this.setActiveTool({ type: "magicframe" });
}
} else if (!isMagicFrameElement(selectedElements[0])) {
// even if user didn't end up setting api key, let's pick the tool
// so they can draw up a frame and move forward
this.setActiveTool({ type: "magicframe" });
}
};
public onMagicframeToolSelect = () => {
if (!this.OPENAI_KEY) {
this.setState({
openDialog: {
name: "settings",
tab: "diagram-to-code",
source: "tool",
},
});
trackEvent("ai", "tool-select (missing key)", "d2c");
return;
}
const selectedElements = this.scene.getSelectedElements({
selectedElementIds: this.state.selectedElementIds,
});

View file

@ -0,0 +1,17 @@
import { useLayoutEffect } from "react";
import { useApp } from "../App";
import type { GenerateDiagramToCode } from "../../types";
export const DiagramToCodePlugin = (props: {
generate: GenerateDiagramToCode;
}) => {
const app = useApp();
useLayoutEffect(() => {
app.setPlugins({
diagramToCode: { generate: props.generate },
});
}, [app, props.generate]);
return null;
};

View file

@ -60,7 +60,6 @@ import { mutateElement } from "../element/mutateElement";
import { ShapeCache } from "../scene/ShapeCache";
import Scene from "../scene/Scene";
import { LaserPointerButton } from "./LaserPointerButton";
import { MagicSettings } from "./MagicSettings";
import { TTDDialog } from "./TTDDialog/TTDDialog";
import { Stats } from "./Stats";
import { actionToggleStats } from "../actions";
@ -85,14 +84,6 @@ interface LayerUIProps {
children?: React.ReactNode;
app: AppClassProperties;
isCollaborating: boolean;
openAIKey: string | null;
isOpenAIKeyPersisted: boolean;
onOpenAIAPIKeyChange: (apiKey: string, shouldPersist: boolean) => void;
onMagicSettingsConfirm: (
apiKey: string,
shouldPersist: boolean,
source: "tool" | "generation" | "settings",
) => void;
}
const DefaultMainMenu: React.FC<{
@ -149,10 +140,6 @@ const LayerUI = ({
children,
app,
isCollaborating,
openAIKey,
isOpenAIKeyPersisted,
onOpenAIAPIKeyChange,
onMagicSettingsConfirm,
}: LayerUIProps) => {
const device = useDevice();
const tunnels = useInitializeTunnels();
@ -482,25 +469,6 @@ const LayerUI = ({
}}
/>
)}
{appState.openDialog?.name === "settings" && (
<MagicSettings
openAIKey={openAIKey}
isPersisted={isOpenAIKeyPersisted}
onChange={onOpenAIAPIKeyChange}
onConfirm={(apiKey, shouldPersist) => {
const source =
appState.openDialog?.name === "settings"
? appState.openDialog?.source
: "settings";
setAppState({ openDialog: null }, () => {
onMagicSettingsConfirm(apiKey, shouldPersist, source);
});
}}
onClose={() => {
setAppState({ openDialog: null });
}}
/>
)}
<ActiveConfirmDialog />
<tunnels.OverwriteConfirmDialogTunnel.Out />
{renderImageExportDialog()}

View file

@ -1,18 +0,0 @@
.excalidraw {
.MagicSettings {
.Island {
height: 100%;
display: flex;
flex-direction: column;
}
}
.MagicSettings-confirm {
padding: 0.5rem 1rem;
}
.MagicSettings__confirm {
margin-top: 2rem;
margin-right: auto;
}
}

View file

@ -1,160 +0,0 @@
import { useState } from "react";
import { Dialog } from "./Dialog";
import { TextField } from "./TextField";
import { MagicIcon, OpenAIIcon } from "./icons";
import { FilledButton } from "./FilledButton";
import { CheckboxItem } from "./CheckboxItem";
import { KEYS } from "../keys";
import { useUIAppState } from "../context/ui-appState";
import { InlineIcon } from "./InlineIcon";
import { Paragraph } from "./Paragraph";
import "./MagicSettings.scss";
import TTDDialogTabs from "./TTDDialog/TTDDialogTabs";
import { TTDDialogTab } from "./TTDDialog/TTDDialogTab";
export const MagicSettings = (props: {
openAIKey: string | null;
isPersisted: boolean;
onChange: (key: string, shouldPersist: boolean) => void;
onConfirm: (key: string, shouldPersist: boolean) => void;
onClose: () => void;
}) => {
const [keyInputValue, setKeyInputValue] = useState(props.openAIKey || "");
const [shouldPersist, setShouldPersist] = useState<boolean>(
props.isPersisted,
);
const appState = useUIAppState();
const onConfirm = () => {
props.onConfirm(keyInputValue.trim(), shouldPersist);
};
if (appState.openDialog?.name !== "settings") {
return null;
}
return (
<Dialog
onCloseRequest={() => {
props.onClose();
props.onConfirm(keyInputValue.trim(), shouldPersist);
}}
title={
<div style={{ display: "flex" }}>
Wireframe to Code (AI){" "}
<div
style={{
display: "flex",
alignItems: "center",
justifyContent: "center",
padding: "0.1rem 0.5rem",
marginLeft: "1rem",
fontSize: 14,
borderRadius: "12px",
background: "var(--color-promo)",
color: "var(--color-surface-lowest)",
}}
>
Experimental
</div>
</div>
}
className="MagicSettings"
autofocus={false}
>
{/* <h2
style={{
margin: 0,
fontSize: "1.25rem",
paddingLeft: "2.5rem",
}}
>
AI Settings
</h2> */}
<TTDDialogTabs dialog="settings" tab={appState.openDialog.tab}>
{/* <TTDDialogTabTriggers>
<TTDDialogTabTrigger tab="text-to-diagram">
<InlineIcon icon={brainIcon} /> Text to diagram
</TTDDialogTabTrigger>
<TTDDialogTabTrigger tab="diagram-to-code">
<InlineIcon icon={MagicIcon} /> Wireframe to code
</TTDDialogTabTrigger>
</TTDDialogTabTriggers> */}
{/* <TTDDialogTab className="ttd-dialog-content" tab="text-to-diagram">
TODO
</TTDDialogTab> */}
<TTDDialogTab
// className="ttd-dialog-content"
tab="diagram-to-code"
>
<Paragraph>
For the diagram-to-code feature we use{" "}
<InlineIcon icon={OpenAIIcon} />
OpenAI.
</Paragraph>
<Paragraph>
While the OpenAI API is in beta, its use is strictly limited as
such we require you use your own API key. You can create an{" "}
<a
href="https://platform.openai.com/login?launch"
rel="noopener noreferrer"
target="_blank"
>
OpenAI account
</a>
, add a small credit (5 USD minimum), and{" "}
<a
href="https://platform.openai.com/api-keys"
rel="noopener noreferrer"
target="_blank"
>
generate your own API key
</a>
.
</Paragraph>
<Paragraph>
Your OpenAI key does not leave the browser, and you can also set
your own limit in your OpenAI account dashboard if needed.
</Paragraph>
<TextField
isRedacted
value={keyInputValue}
placeholder="Paste your API key here"
label="OpenAI API key"
onChange={(value) => {
setKeyInputValue(value);
props.onChange(value.trim(), shouldPersist);
}}
selectOnRender
onKeyDown={(event) => event.key === KEYS.ENTER && onConfirm()}
/>
<Paragraph>
By default, your API token is not persisted anywhere so you'll need
to insert it again after reload. But, you can persist locally in
your browser below.
</Paragraph>
<CheckboxItem checked={shouldPersist} onChange={setShouldPersist}>
Persist API key in browser storage
</CheckboxItem>
<Paragraph>
Once API key is set, you can use the <InlineIcon icon={MagicIcon} />{" "}
tool to wrap your elements in a frame that will then allow you to
turn it into code. This dialog can be accessed using the{" "}
<b>AI Settings</b> <InlineIcon icon={OpenAIIcon} />.
</Paragraph>
<FilledButton
className="MagicSettings__confirm"
size="large"
label="Confirm"
onClick={onConfirm}
/>
</TTDDialogTab>
</TTDDialogTabs>
</Dialog>
);
};

View file

@ -7,10 +7,7 @@ import { isMemberOf } from "../../utils";
const TTDDialogTabs = (
props: {
children: ReactNode;
} & (
| { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" }
| { dialog: "settings"; tab: "text-to-diagram" | "diagram-to-code" }
),
} & { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" },
) => {
const setAppState = useExcalidrawSetAppState();
@ -39,13 +36,6 @@ const TTDDialogTabs = (
}
}
if (
props.dialog === "settings" &&
isMemberOf(["text-to-diagram", "diagram-to-code"], tab)
) {
setAppState({
openDialog: { name: props.dialog, tab, source: "settings" },
});
} else if (
props.dialog === "ttd" &&
isMemberOf(["text-to-diagram", "mermaid"], tab)
) {

View file

@ -13,6 +13,7 @@ const DropdownMenuItemLink = ({
onSelect,
className = "",
selected,
rel = "noreferrer",
...rest
}: {
href: string;
@ -22,6 +23,7 @@ const DropdownMenuItemLink = ({
className?: string;
selected?: boolean;
onSelect?: (event: Event) => void;
rel?: string;
} & React.AnchorHTMLAttributes<HTMLAnchorElement>) => {
const handleClick = useHandleDropdownMenuItemClick(rest.onClick, onSelect);