feat: wireframe-to-code (#7334)

This commit is contained in:
David Luzar 2023-11-23 23:07:53 +01:00 committed by GitHub
parent d1e4421823
commit c7ee46e7f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
63 changed files with 2106 additions and 444 deletions

View file

@ -0,0 +1,51 @@
import { EDITOR_LS_KEYS } from "../constants";
import { JSONValue } from "../types";
export class EditorLocalStorage {
static has(key: typeof EDITOR_LS_KEYS[keyof typeof EDITOR_LS_KEYS]) {
try {
return !!window.localStorage.getItem(key);
} catch (error: any) {
console.warn(`localStorage.getItem error: ${error.message}`);
return false;
}
}
static get<T extends JSONValue>(
key: typeof EDITOR_LS_KEYS[keyof typeof EDITOR_LS_KEYS],
) {
try {
const value = window.localStorage.getItem(key);
if (value) {
return JSON.parse(value) as T;
}
return null;
} catch (error: any) {
console.warn(`localStorage.getItem error: ${error.message}`);
return null;
}
}
static set = (
key: typeof EDITOR_LS_KEYS[keyof typeof EDITOR_LS_KEYS],
value: JSONValue,
) => {
try {
window.localStorage.setItem(key, JSON.stringify(value));
return true;
} catch (error: any) {
console.warn(`localStorage.setItem error: ${error.message}`);
return false;
}
};
static delete = (
name: typeof EDITOR_LS_KEYS[keyof typeof EDITOR_LS_KEYS],
) => {
try {
window.localStorage.removeItem(name);
} catch (error: any) {
console.warn(`localStorage.removeItem error: ${error.message}`);
}
};
}

300
src/data/ai/types.ts Normal file
View file

@ -0,0 +1,300 @@
export namespace OpenAIInput {
type ChatCompletionContentPart =
| ChatCompletionContentPartText
| ChatCompletionContentPartImage;
interface ChatCompletionContentPartImage {
image_url: ChatCompletionContentPartImage.ImageURL;
/**
* The type of the content part.
*/
type: "image_url";
}
namespace ChatCompletionContentPartImage {
export interface ImageURL {
/**
* Either a URL of the image or the base64 encoded image data.
*/
url: string;
/**
* Specifies the detail level of the image.
*/
detail?: "auto" | "low" | "high";
}
}
interface ChatCompletionContentPartText {
/**
* The text content.
*/
text: string;
/**
* The type of the content part.
*/
type: "text";
}
interface ChatCompletionUserMessageParam {
/**
* The contents of the user message.
*/
content: string | Array<ChatCompletionContentPart> | null;
/**
* The role of the messages author, in this case `user`.
*/
role: "user";
}
interface ChatCompletionSystemMessageParam {
/**
* The contents of the system message.
*/
content: string | null;
/**
* The role of the messages author, in this case `system`.
*/
role: "system";
}
export interface ChatCompletionCreateParamsBase {
/**
* A list of messages comprising the conversation so far.
* [Example Python code](https://cookbook.openai.com/examples/how_to_format_inputs_to_chatgpt_models).
*/
messages: Array<
ChatCompletionUserMessageParam | ChatCompletionSystemMessageParam
>;
/**
* ID of the model to use. See the
* [model endpoint compatibility](https://platform.openai.com/docs/models/model-endpoint-compatibility)
* table for details on which models work with the Chat API.
*/
model:
| (string & {})
| "gpt-4-1106-preview"
| "gpt-4-vision-preview"
| "gpt-4"
| "gpt-4-0314"
| "gpt-4-0613"
| "gpt-4-32k"
| "gpt-4-32k-0314"
| "gpt-4-32k-0613"
| "gpt-3.5-turbo"
| "gpt-3.5-turbo-16k"
| "gpt-3.5-turbo-0301"
| "gpt-3.5-turbo-0613"
| "gpt-3.5-turbo-16k-0613";
/**
* Number between -2.0 and 2.0. Positive values penalize new tokens based on their
* existing frequency in the text so far, decreasing the model's likelihood to
* repeat the same line verbatim.
*
* [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/gpt/parameter-details)
*/
frequency_penalty?: number | null;
/**
* Modify the likelihood of specified tokens appearing in the completion.
*
* Accepts a JSON object that maps tokens (specified by their token ID in the
* tokenizer) to an associated bias value from -100 to 100. Mathematically, the
* bias is added to the logits generated by the model prior to sampling. The exact
* effect will vary per model, but values between -1 and 1 should decrease or
* increase likelihood of selection; values like -100 or 100 should result in a ban
* or exclusive selection of the relevant token.
*/
logit_bias?: Record<string, number> | null;
/**
* The maximum number of [tokens](/tokenizer) to generate in the chat completion.
*
* The total length of input tokens and generated tokens is limited by the model's
* context length.
* [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken)
* for counting tokens.
*/
max_tokens?: number | null;
/**
* How many chat completion choices to generate for each input message.
*/
n?: number | null;
/**
* Number between -2.0 and 2.0. Positive values penalize new tokens based on
* whether they appear in the text so far, increasing the model's likelihood to
* talk about new topics.
*
* [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/gpt/parameter-details)
*/
presence_penalty?: number | null;
/**
* This feature is in Beta. If specified, our system will make a best effort to
* sample deterministically, such that repeated requests with the same `seed` and
* parameters should return the same result. Determinism is not guaranteed, and you
* should refer to the `system_fingerprint` response parameter to monitor changes
* in the backend.
*/
seed?: number | null;
/**
* Up to 4 sequences where the API will stop generating further tokens.
*/
stop?: string | null | Array<string>;
/**
* If set, partial message deltas will be sent, like in ChatGPT. Tokens will be
* sent as data-only
* [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format)
* as they become available, with the stream terminated by a `data: [DONE]`
* message.
* [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions).
*/
stream?: boolean | null;
/**
* What sampling temperature to use, between 0 and 2. Higher values like 0.8 will
* make the output more random, while lower values like 0.2 will make it more
* focused and deterministic.
*
* We generally recommend altering this or `top_p` but not both.
*/
temperature?: number | null;
/**
* An alternative to sampling with temperature, called nucleus sampling, where the
* model considers the results of the tokens with top_p probability mass. So 0.1
* means only the tokens comprising the top 10% probability mass are considered.
*
* We generally recommend altering this or `temperature` but not both.
*/
top_p?: number | null;
/**
* A unique identifier representing your end-user, which can help OpenAI to monitor
* and detect abuse.
* [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).
*/
user?: string;
}
}
export namespace OpenAIOutput {
export interface ChatCompletion {
/**
* A unique identifier for the chat completion.
*/
id: string;
/**
* A list of chat completion choices. Can be more than one if `n` is greater
* than 1.
*/
choices: Array<Choice>;
/**
* The Unix timestamp (in seconds) of when the chat completion was created.
*/
created: number;
/**
* The model used for the chat completion.
*/
model: string;
/**
* The object type, which is always `chat.completion`.
*/
object: "chat.completion";
/**
* This fingerprint represents the backend configuration that the model runs with.
*
* Can be used in conjunction with the `seed` request parameter to understand when
* backend changes have been made that might impact determinism.
*/
system_fingerprint?: string;
/**
* Usage statistics for the completion request.
*/
usage?: CompletionUsage;
}
export interface Choice {
/**
* The reason the model stopped generating tokens. This will be `stop` if the model
* hit a natural stop point or a provided stop sequence, `length` if the maximum
* number of tokens specified in the request was reached, `content_filter` if
* content was omitted due to a flag from our content filters, `tool_calls` if the
* model called a tool, or `function_call` (deprecated) if the model called a
* function.
*/
finish_reason:
| "stop"
| "length"
| "tool_calls"
| "content_filter"
| "function_call";
/**
* The index of the choice in the list of choices.
*/
index: number;
/**
* A chat completion message generated by the model.
*/
message: ChatCompletionMessage;
}
interface ChatCompletionMessage {
/**
* The contents of the message.
*/
content: string | null;
/**
* The role of the author of this message.
*/
role: "assistant";
}
/**
* Usage statistics for the completion request.
*/
interface CompletionUsage {
/**
* Number of tokens in the generated completion.
*/
completion_tokens: number;
/**
* Number of tokens in the prompt.
*/
prompt_tokens: number;
/**
* Total number of tokens used in the request (prompt + completion).
*/
total_tokens: number;
}
export interface APIError {
readonly status: 400 | 401 | 403 | 404 | 409 | 422 | 429 | 500 | undefined;
readonly headers: Headers | undefined;
readonly error: { message: string } | undefined;
readonly code: string | null | undefined;
readonly param: string | null | undefined;
readonly type: string | undefined;
}
}

View file

@ -3,10 +3,11 @@ import {
copyTextToSystemClipboard,
} from "../clipboard";
import { DEFAULT_EXPORT_PADDING, isFirefox, MIME_TYPES } from "../constants";
import { getNonDeletedElements, isFrameElement } from "../element";
import { getNonDeletedElements } from "../element";
import { isFrameLikeElement } from "../element/typeChecks";
import {
ExcalidrawElement,
ExcalidrawFrameElement,
ExcalidrawFrameLikeElement,
NonDeletedExcalidrawElement,
} from "../element/types";
import { t } from "../i18n";
@ -38,7 +39,7 @@ export const prepareElementsForExport = (
exportSelectionOnly &&
isSomeElementSelected(elements, { selectedElementIds });
let exportingFrame: ExcalidrawFrameElement | null = null;
let exportingFrame: ExcalidrawFrameLikeElement | null = null;
let exportedElements = isExportingSelection
? getSelectedElements(
elements,
@ -50,7 +51,10 @@ export const prepareElementsForExport = (
: elements;
if (isExportingSelection) {
if (exportedElements.length === 1 && isFrameElement(exportedElements[0])) {
if (
exportedElements.length === 1 &&
isFrameLikeElement(exportedElements[0])
) {
exportingFrame = exportedElements[0];
exportedElements = elementsOverlappingBBox({
elements,
@ -93,7 +97,7 @@ export const exportCanvas = async (
viewBackgroundColor: string;
name: string;
fileHandle?: FileSystemHandle | null;
exportingFrame: ExcalidrawFrameElement | null;
exportingFrame: ExcalidrawFrameLikeElement | null;
},
) => {
if (elements.length === 0) {

104
src/data/magic.ts Normal file
View file

@ -0,0 +1,104 @@
import { Theme } from "../element/types";
import { DataURL } from "../types";
import { OpenAIInput, OpenAIOutput } from "./ai/types";
export type MagicCacheData =
| {
status: "pending";
}
| { status: "done"; html: string }
| {
status: "error";
message?: string;
code: "ERR_GENERATION_INTERRUPTED" | string;
};
const SYSTEM_PROMPT = `You are a skilled front-end developer who builds interactive prototypes from wireframes, and is an expert at CSS Grid and Flex design.
Your role is to transform low-fidelity wireframes into working front-end HTML code.
YOU MUST FOLLOW FOLLOWING RULES:
- Use HTML, CSS, JavaScript to build a responsive, accessible, polished prototype
- Leverage Tailwind for styling and layout (import as script <script src="https://cdn.tailwindcss.com"></script>)
- Inline JavaScript when needed
- Fetch dependencies from CDNs when needed (using unpkg or skypack)
- Source images from Unsplash or create applicable placeholders
- Interpret annotations as intended vs literal UI
- Fill gaps using your expertise in UX and business logic
- generate primarily for desktop UI, but make it responsive.
- Use grid and flexbox wherever applicable.
- Convert the wireframe in its entirety, don't omit elements if possible.
If the wireframes, diagrams, or text is unclear or unreadable, refer to provided text for clarification.
Your goal is a production-ready prototype that brings the wireframes to life.
Please output JUST THE HTML file containing your best attempt at implementing the provided wireframes.`;
export async function diagramToHTML({
image,
apiKey,
text,
theme = "light",
}: {
image: DataURL;
apiKey: string;
text: string;
theme?: Theme;
}) {
const body: OpenAIInput.ChatCompletionCreateParamsBase = {
model: "gpt-4-vision-preview",
// 4096 are max output tokens allowed for `gpt-4-vision-preview` currently
max_tokens: 4096,
temperature: 0.1,
messages: [
{
role: "system",
content: SYSTEM_PROMPT,
},
{
role: "user",
content: [
{
type: "image_url",
image_url: {
url: image,
detail: "high",
},
},
{
type: "text",
text: `Above is the reference wireframe. Please make a new website based on these and return just the HTML file. Also, please make it for the ${theme} theme. What follows are the wireframe's text annotations (if any)...`,
},
{
type: "text",
text,
},
],
},
],
};
let result:
| ({ ok: true } & OpenAIOutput.ChatCompletion)
| ({ ok: false } & OpenAIOutput.APIError);
const resp = await fetch("https://api.openai.com/v1/chat/completions", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
},
body: JSON.stringify(body),
});
if (resp.ok) {
const json: OpenAIOutput.ChatCompletion = await resp.json();
result = { ...json, ok: true };
} else {
const json: OpenAIOutput.APIError = await resp.json();
result = { ...json, ok: false };
}
return result;
}

View file

@ -1,5 +1,6 @@
import {
ExcalidrawElement,
ExcalidrawElementType,
ExcalidrawSelectionElement,
ExcalidrawTextElement,
FontFamilyValues,
@ -68,6 +69,7 @@ export const AllowedExcalidrawActiveTools: Record<
embeddable: true,
hand: true,
laser: false,
magicframe: false,
};
export type RestoredDataState = {
@ -111,7 +113,7 @@ const restoreElementWithProperties = <
// @ts-ignore TS complains here but type checks the call sites fine.
keyof K
> &
Partial<Pick<ExcalidrawElement, "type" | "x" | "y">>,
Partial<Pick<ExcalidrawElement, "type" | "x" | "y" | "customData">>,
): T => {
const base: Pick<T, keyof ExcalidrawElement> & {
[PRECEDING_ELEMENT_KEY]?: string;
@ -159,8 +161,9 @@ const restoreElementWithProperties = <
locked: element.locked ?? false,
};
if ("customData" in element) {
base.customData = element.customData;
if ("customData" in element || "customData" in extra) {
base.customData =
"customData" in extra ? extra.customData : element.customData;
}
if (PRECEDING_ELEMENT_KEY in element) {
@ -273,7 +276,7 @@ const restoreElement = (
return restoreElementWithProperties(element, {
type:
(element.type as ExcalidrawElement["type"] | "draw") === "draw"
(element.type as ExcalidrawElementType | "draw") === "draw"
? "line"
: element.type,
startBinding: repairBinding(element.startBinding),
@ -289,15 +292,15 @@ const restoreElement = (
// generic elements
case "ellipse":
return restoreElementWithProperties(element, {});
case "rectangle":
return restoreElementWithProperties(element, {});
case "diamond":
case "iframe":
return restoreElementWithProperties(element, {});
case "embeddable":
return restoreElementWithProperties(element, {
validated: null,
});
case "magicframe":
case "frame":
return restoreElementWithProperties(element, {
name: element.name ?? null,

View file

@ -15,6 +15,7 @@ import {
ElementConstructorOpts,
newFrameElement,
newImageElement,
newMagicFrameElement,
newTextElement,
} from "../element/newElement";
import {
@ -26,12 +27,13 @@ import {
ExcalidrawArrowElement,
ExcalidrawBindableElement,
ExcalidrawElement,
ExcalidrawEmbeddableElement,
ExcalidrawFrameElement,
ExcalidrawFreeDrawElement,
ExcalidrawGenericElement,
ExcalidrawIframeLikeElement,
ExcalidrawImageElement,
ExcalidrawLinearElement,
ExcalidrawMagicFrameElement,
ExcalidrawSelectionElement,
ExcalidrawTextElement,
FileId,
@ -61,7 +63,12 @@ export type ValidLinearElement = {
| {
type: Exclude<
ExcalidrawBindableElement["type"],
"image" | "text" | "frame" | "embeddable"
| "image"
| "text"
| "frame"
| "magicframe"
| "embeddable"
| "iframe"
>;
id?: ExcalidrawGenericElement["id"];
}
@ -69,7 +76,12 @@ export type ValidLinearElement = {
id: ExcalidrawGenericElement["id"];
type?: Exclude<
ExcalidrawBindableElement["type"],
"image" | "text" | "frame" | "embeddable"
| "image"
| "text"
| "frame"
| "magicframe"
| "embeddable"
| "iframe"
>;
}
)
@ -93,7 +105,12 @@ export type ValidLinearElement = {
| {
type: Exclude<
ExcalidrawBindableElement["type"],
"image" | "text" | "frame" | "embeddable"
| "image"
| "text"
| "frame"
| "magicframe"
| "embeddable"
| "iframe"
>;
id?: ExcalidrawGenericElement["id"];
}
@ -101,7 +118,12 @@ export type ValidLinearElement = {
id: ExcalidrawGenericElement["id"];
type?: Exclude<
ExcalidrawBindableElement["type"],
"image" | "text" | "frame" | "embeddable"
| "image"
| "text"
| "frame"
| "magicframe"
| "embeddable"
| "iframe"
>;
}
)
@ -137,7 +159,7 @@ export type ValidContainer =
export type ExcalidrawElementSkeleton =
| Extract<
Exclude<ExcalidrawElement, ExcalidrawSelectionElement>,
ExcalidrawEmbeddableElement | ExcalidrawFreeDrawElement
ExcalidrawIframeLikeElement | ExcalidrawFreeDrawElement
>
| ({
type: Extract<ExcalidrawLinearElement["type"], "line">;
@ -163,7 +185,12 @@ export type ExcalidrawElementSkeleton =
type: "frame";
children: readonly ExcalidrawElement["id"][];
name?: string;
} & Partial<ExcalidrawFrameElement>);
} & Partial<ExcalidrawFrameElement>)
| ({
type: "magicframe";
children: readonly ExcalidrawElement["id"][];
name?: string;
} & Partial<ExcalidrawMagicFrameElement>);
const DEFAULT_LINEAR_ELEMENT_PROPS = {
width: 100,
@ -547,7 +574,16 @@ export const convertToExcalidrawElements = (
});
break;
}
case "magicframe": {
excalidrawElement = newMagicFrameElement({
x: 0,
y: 0,
...element,
});
break;
}
case "freedraw":
case "iframe":
case "embeddable": {
excalidrawElement = element;
break;
@ -656,7 +692,7 @@ export const convertToExcalidrawElements = (
// need to calculate coordinates and dimensions of frame which is possibe after all
// frame children are processed.
for (const [id, element] of elementsWithIds) {
if (element.type !== "frame") {
if (element.type !== "frame" && element.type !== "magicframe") {
continue;
}
const frame = elementStore.getElement(id);