Excalidraw on exe.dev: https://exe.dev/deploy/excalidraw Keeps a self-hosted Excalidraw on your VM. Applies to excalidraw/excalidraw at c0a0ca348faa273c768de839a22df113f997c095 with `git apply excalidraw-exe-dev.patch`. - .env.production: live collaboration on this VM (/socket.io/); share links and AI on VM paths that don't exist, so they fail instead of sending anything to excalidraw.com; no Firebase project. - excalidraw-app/data/firebase.ts: same functions, but rooms and their images are stored, encrypted in the browser as before, through nginx on this VM (/rooms/) instead of Firebase. - excalidraw-app/index.html: no Simple Analytics script. - scripts/woff2/woff2-vite-plugins.js: fonts from this VM, not Excalidraw's CDN (the UI font bundled with the app). diff --git a/.env.production b/.env.production index 72dd24d6..8b6fbcb9 100644 --- a/.env.production +++ b/.env.production @@ -1,7 +1,7 @@ MODE="production" -VITE_APP_BACKEND_V2_GET_URL=https://json.excalidraw.com/api/v2/ -VITE_APP_BACKEND_V2_POST_URL=https://json.excalidraw.com/api/v2/post/ +VITE_APP_BACKEND_V2_GET_URL=/share-links/ +VITE_APP_BACKEND_V2_POST_URL=/share-links/ VITE_APP_LIBRARY_URL=https://libraries.excalidraw.com VITE_APP_LIBRARY_BACKEND=https://us-central1-excalidraw-room-persistence.cloudfunctions.net/libraries @@ -9,12 +9,12 @@ VITE_APP_LIBRARY_BACKEND=https://us-central1-excalidraw-room-persistence.cloudfu VITE_APP_PLUS_LP=https://plus.excalidraw.com VITE_APP_PLUS_APP=https://app.excalidraw.com -VITE_APP_AI_BACKEND=https://oss-ai.excalidraw.com +VITE_APP_AI_BACKEND=/ai # socket server URL used for collaboration -VITE_APP_WS_SERVER_URL=https://oss-collab.excalidraw.com +VITE_APP_WS_SERVER_URL=/ -VITE_APP_FIREBASE_CONFIG='{"apiKey":"AIzaSyAd15pYlMci_xIp9ko6wkEsDzAAA0Dn0RU","authDomain":"excalidraw-room-persistence.firebaseapp.com","databaseURL":"https://excalidraw-room-persistence.firebaseio.com","projectId":"excalidraw-room-persistence","storageBucket":"excalidraw-room-persistence.appspot.com","messagingSenderId":"654800341332","appId":"1:654800341332:web:4a692de832b55bd57ce0c1"}' +VITE_APP_FIREBASE_CONFIG='{}' VITE_APP_ENABLE_TRACKING=false diff --git a/excalidraw-app/data/firebase.ts b/excalidraw-app/data/firebase.ts index e534f2ca..26dfbf23 100644 --- a/excalidraw-app/data/firebase.ts +++ b/excalidraw-app/data/firebase.ts @@ -1,9 +1,18 @@ +// Excalidraw on exe.dev: https://exe.dev/deploy/excalidraw +// +// Replaces excalidraw-app/data/firebase.ts (same exports) so that collaboration +// rooms and their images are stored on your VM instead of Excalidraw's +// Firebase project. Everything is encrypted in the browser with the key in +// the room link, exactly as before; nginx on the VM stores the encrypted +// bytes under /rooms/ (~/excalidraw/rooms on the VM's disk). + import { reconcileElements } from "@excalidraw/excalidraw"; import { MIME_TYPES, toBrandedType } from "@excalidraw/common"; import { decompressData } from "@excalidraw/excalidraw/data/encode"; import { encryptData, decryptData, + IV_LENGTH_BYTES, } from "@excalidraw/excalidraw/data/encryption"; import { restoreElements } from "@excalidraw/excalidraw/data/restore"; import { getSceneVersion } from "@excalidraw/element"; @@ -21,123 +30,94 @@ import type { DataURL, } from "@excalidraw/excalidraw/types"; -import { FILE_CACHE_MAX_AGE_SEC } from "../app_constants"; - import { getSyncableElements } from "."; import type { SyncableExcalidrawElement } from "."; import type Portal from "../collab/Portal"; import type { Socket } from "socket.io-client"; -import type { Bytes } from "firebase/firestore"; -import type { UploadMetadata } from "firebase/storage"; -// private +// storage on the VM // ----------------------------------------------------------------------------- -let FIREBASE_CONFIG: Record; -try { - FIREBASE_CONFIG = JSON.parse(import.meta.env.VITE_APP_FIREBASE_CONFIG); -} catch (error: any) { - console.warn( - `Error JSON parsing firebase config. Supplied value: ${ - import.meta.env.VITE_APP_FIREBASE_CONFIG - }`, - ); - FIREBASE_CONFIG = {}; -} +const storeURL = (path: string) => `/rooms/${path.replace(/^\/+/, "")}`; -const lazy = (load: () => Promise) => { - let promise: Promise | null = null; - return () => - (promise ??= load().catch((error) => { - promise = null; - throw error; - })); +const putBytes = async (path: string, body: Uint8Array | Blob) => { + const response = await fetch(storeURL(path), { + method: "PUT", + body: body as BodyInit, + headers: { "Content-Type": "application/octet-stream" }, + }); + if (!response.ok) { + throw new Error(`Couldn't save to the VM (HTTP ${response.status})`); + } }; -const _initializeFirebase = lazy(async () => { - const sdk = await import("./firebaseSDK"); - return { sdk, app: sdk.initializeApp(FIREBASE_CONFIG) }; -}); - -const _loadFirestore = lazy(async () => { - const { sdk, app } = await _initializeFirebase(); - const { getFirestore, doc, getDoc, runTransaction, Bytes } = sdk; - return { firestore: getFirestore(app), doc, getDoc, runTransaction, Bytes }; -}); - -const _loadStorage = lazy(async () => { - const { sdk, app } = await _initializeFirebase(); - const { getStorage, ref, uploadBytes } = sdk; - return { storage: getStorage(app), ref, uploadBytes }; -}); - -// ----------------------------------------------------------------------------- - -export const uploadBytesToFirebase = async ( - path: string, - data: Blob | Uint8Array, - metadata?: UploadMetadata, -) => { - const { storage, ref, uploadBytes } = await _loadStorage(); - return uploadBytes(ref(storage, path), data, metadata); +const getBytes = async (path: string): Promise => { + const response = await fetch(storeURL(path), { cache: "no-store" }); + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`Couldn't load from the VM (HTTP ${response.status})`); + } + return new Uint8Array(await response.arrayBuffer()); }; -type FirebaseStoredScene = { - sceneVersion: number; - iv: Bytes; - ciphertext: Bytes; -}; +const scenePath = (roomId: string) => `scenes/${roomId}`; +// A stored scene is the IV followed by the encrypted JSON of its elements. const encryptElements = async ( key: string, elements: readonly ExcalidrawElement[], -): Promise<{ ciphertext: ArrayBuffer; iv: Uint8Array }> => { - const json = JSON.stringify(elements); - const encoded = new TextEncoder().encode(json); +): Promise => { + const encoded = new TextEncoder().encode(JSON.stringify(elements)); const { encryptedBuffer, iv } = await encryptData(key, encoded); - - return { ciphertext: encryptedBuffer, iv }; + const stored = new Uint8Array(iv.byteLength + encryptedBuffer.byteLength); + stored.set(iv, 0); + stored.set(new Uint8Array(encryptedBuffer), iv.byteLength); + return stored; }; const decryptElements = async ( - data: FirebaseStoredScene, + stored: Uint8Array, roomKey: string, ): Promise => { - const ciphertext = data.ciphertext.toUint8Array() as Uint8Array; - const iv = data.iv.toUint8Array() as Uint8Array; - + const iv = stored.slice(0, IV_LENGTH_BYTES); + const ciphertext = stored.slice(IV_LENGTH_BYTES); const decrypted = await decryptData(iv, ciphertext, roomKey); - const decodedData = new TextDecoder("utf-8").decode( - new Uint8Array(decrypted), - ); - return JSON.parse(decodedData); + return JSON.parse(new TextDecoder("utf-8").decode(new Uint8Array(decrypted))); }; -class FirebaseSceneVersionCache { +class SceneVersionCache { private static cache = new WeakMap(); - static get = (socket: Socket) => { - return FirebaseSceneVersionCache.cache.get(socket); - }; + static get = (socket: Socket) => SceneVersionCache.cache.get(socket); static set = ( socket: Socket, elements: readonly SyncableExcalidrawElement[], ) => { - FirebaseSceneVersionCache.cache.set(socket, getSceneVersion(elements)); + SceneVersionCache.cache.set(socket, getSceneVersion(elements)); }; } +// the API Excalidraw expects (named after Firebase upstream) +// ----------------------------------------------------------------------------- + +export const uploadBytesToFirebase = async ( + path: string, + data: Blob | Uint8Array, + _metadata?: unknown, +) => { + await putBytes(path, data); +}; + export const isSavedToFirebase = ( portal: Portal, elements: readonly ExcalidrawElement[], ): boolean => { if (portal.socket && portal.roomId && portal.roomKey) { - const sceneVersion = getSceneVersion(elements); - - return FirebaseSceneVersionCache.get(portal.socket) === sceneVersion; + return SceneVersionCache.get(portal.socket) === getSceneVersion(elements); } - // if no room exists, consider the room saved so that we don't unnecessarily - // prevent unload (there's nothing we could do at that point anyway) + // no room: nothing to save return true; }; @@ -148,21 +128,13 @@ export const saveFilesToFirebase = async ({ prefix: string; files: { id: FileId; buffer: Uint8Array }[]; }) => { - // failing to load the SDK must fail the whole save rather than be reported - // per file as `erroredFiles` (which share links ignore, and collab never - // retries) - const { storage, ref, uploadBytes } = await _loadStorage(); - const erroredFiles: FileId[] = []; const savedFiles: FileId[] = []; await Promise.all( files.map(async ({ id, buffer }) => { try { - const storageRef = ref(storage, `${prefix}/${id}`); - await uploadBytes(storageRef, buffer, { - cacheControl: `public, max-age=${FILE_CACHE_MAX_AGE_SEC}`, - }); + await putBytes(`${prefix}/${id}`, buffer); savedFiles.push(id); } catch (error: any) { erroredFiles.push(id); @@ -173,20 +145,6 @@ export const saveFilesToFirebase = async ({ return { savedFiles, erroredFiles }; }; -const createFirebaseSceneDocument = async ( - elements: readonly SyncableExcalidrawElement[], - roomKey: string, -) => { - const sceneVersion = getSceneVersion(elements); - const { ciphertext, iv } = await encryptElements(roomKey, elements); - const { Bytes } = await _loadFirestore(); - return { - sceneVersion, - ciphertext: Bytes.fromUint8Array(new Uint8Array(ciphertext)), - iv: Bytes.fromUint8Array(iv), - } as FirebaseStoredScene; -}; - export const saveToFirebase = async ( portal: Portal, elements: readonly SyncableExcalidrawElement[], @@ -194,7 +152,6 @@ export const saveToFirebase = async ( ) => { const { roomId, roomKey, socket } = portal; if ( - // bail if no room exists as there's nothing we can do at this point !roomId || !roomKey || !socket || @@ -203,49 +160,30 @@ export const saveToFirebase = async ( return null; } - const { firestore, doc, runTransaction } = await _loadFirestore(); - const docRef = doc(firestore, "scenes", roomId); - - const storedScene = await runTransaction(firestore, async (transaction) => { - const snapshot = await transaction.get(docRef); - - if (!snapshot.exists()) { - const storedScene = await createFirebaseSceneDocument(elements, roomKey); - - transaction.set(docRef, storedScene); - - return storedScene; - } - - const prevStoredScene = snapshot.data() as FirebaseStoredScene; - const prevStoredElements = getSyncableElements( - restoreElements(await decryptElements(prevStoredScene, roomKey), null), + // Merge with what's stored, as Firebase's transaction does upstream. + let toStore: readonly SyncableExcalidrawElement[] = elements; + const prevStored = await getBytes(scenePath(roomId)); + if (prevStored) { + const prevElements = getSyncableElements( + restoreElements(await decryptElements(prevStored, roomKey), null), ); - const reconciledElements = getSyncableElements( + toStore = getSyncableElements( reconcileElements( elements, - prevStoredElements as OrderedExcalidrawElement[] as RemoteExcalidrawElement[], + prevElements as OrderedExcalidrawElement[] as RemoteExcalidrawElement[], appState, ), ); + } - const storedScene = await createFirebaseSceneDocument( - reconciledElements, - roomKey, - ); - - transaction.update(docRef, storedScene); - - // Return the stored elements as the in memory `reconciledElements` could have mutated in the meantime - return storedScene; - }); + const stored = await encryptElements(roomKey, toStore); + await putBytes(scenePath(roomId), stored); + // Return a copy of what was stored, as upstream does. const storedElements = getSyncableElements( - restoreElements(await decryptElements(storedScene, roomKey), null), + restoreElements(await decryptElements(stored, roomKey), null), ); - - FirebaseSceneVersionCache.set(socket, storedElements); - + SceneVersionCache.set(socket, storedElements); return toBrandedType(storedElements); }; @@ -254,23 +192,18 @@ export const loadFromFirebase = async ( roomKey: string, socket: Socket | null, ): Promise => { - const { firestore, doc, getDoc } = await _loadFirestore(); - const docRef = doc(firestore, "scenes", roomId); - const docSnap = await getDoc(docRef); - if (!docSnap.exists()) { + const stored = await getBytes(scenePath(roomId)); + if (!stored) { return null; } - const storedScene = docSnap.data() as FirebaseStoredScene; const elements = getSyncableElements( - restoreElements(await decryptElements(storedScene, roomKey), null, { + restoreElements(await decryptElements(stored, roomKey), null, { deleteInvisibleElements: true, }), ); - if (socket) { - FirebaseSceneVersionCache.set(socket, elements); + SceneVersionCache.set(socket, elements); } - return elements; }; @@ -285,32 +218,22 @@ export const loadFilesFromFirebase = async ( await Promise.all( [...new Set(filesIds)].map(async (id) => { try { - const url = `https://firebasestorage.googleapis.com/v0/b/${ - FIREBASE_CONFIG.storageBucket - }/o/${encodeURIComponent(prefix.replace(/^\//, ""))}%2F${id}`; - const response = await fetch(`${url}?alt=media`); - if (response.status < 400) { - const arrayBuffer = await response.arrayBuffer(); - - const { data, metadata } = await decompressData( - new Uint8Array(arrayBuffer), - { - decryptionKey, - }, - ); - - const dataURL = new TextDecoder().decode(data) as DataURL; - - loadedFiles.push({ - mimeType: metadata.mimeType || MIME_TYPES.binary, - id, - dataURL, - created: metadata?.created || Date.now(), - lastRetrieved: metadata?.created || Date.now(), - }); - } else { + const bytes = await getBytes(`${prefix}/${id}`); + if (!bytes) { erroredFiles.set(id, true); + return; } + const { data, metadata } = await decompressData( + bytes as Uint8Array, + { decryptionKey }, + ); + loadedFiles.push({ + mimeType: metadata.mimeType || MIME_TYPES.binary, + id, + dataURL: new TextDecoder().decode(data) as DataURL, + created: metadata?.created || Date.now(), + lastRetrieved: metadata?.created || Date.now(), + }); } catch (error: any) { erroredFiles.set(id, true); console.error(error); diff --git a/excalidraw-app/index.html b/excalidraw-app/index.html index 1c29d722..7e6ec571 100644 --- a/excalidraw-app/index.html +++ b/excalidraw-app/index.html @@ -215,38 +215,6 @@
<% if (typeof PROD != 'undefined' && PROD == true) { %> - - - <% } %> diff --git a/scripts/woff2/woff2-vite-plugins.js b/scripts/woff2/woff2-vite-plugins.js index 3870bbcd..9e6739f6 100644 --- a/scripts/woff2/woff2-vite-plugins.js +++ b/scripts/woff2/woff2-vite-plugins.js @@ -1,5 +1,5 @@ // define `EXCALIDRAW_ASSET_PATH` as a SSOT -const OSS_FONTS_CDN = "https://excalidraw.nyc3.cdn.digitaloceanspaces.com/oss/"; +const OSS_FONTS_CDN = "/"; const OSS_FONTS_FALLBACK = "/"; /** @@ -24,9 +24,7 @@ module.exports.woff2BrowserPlugin = () => { @font-face { font-family: "Assistant"; - src: url(${OSS_FONTS_CDN}fonts/Assistant/Assistant-Regular.woff2) - format("woff2"), - url(./Assistant-Regular.woff2) format("woff2"); + src: url(./Assistant/Assistant-Regular.woff2) format("woff2"); font-weight: 400; style: normal; display: swap; @@ -34,9 +32,7 @@ module.exports.woff2BrowserPlugin = () => { @font-face { font-family: "Assistant"; - src: url(${OSS_FONTS_CDN}fonts/Assistant/Assistant-Medium.woff2) - format("woff2"), - url(./Assistant-Medium.woff2) format("woff2"); + src: url(./Assistant/Assistant-Medium.woff2) format("woff2"); font-weight: 500; style: normal; display: swap; @@ -44,9 +40,7 @@ module.exports.woff2BrowserPlugin = () => { @font-face { font-family: "Assistant"; - src: url(${OSS_FONTS_CDN}fonts/Assistant/Assistant-SemiBold.woff2) - format("woff2"), - url(./Assistant-SemiBold.woff2) format("woff2"); + src: url(./Assistant/Assistant-SemiBold.woff2) format("woff2"); font-weight: 600; style: normal; display: swap; @@ -54,9 +48,7 @@ module.exports.woff2BrowserPlugin = () => { @font-face { font-family: "Assistant"; - src: url(${OSS_FONTS_CDN}fonts/Assistant/Assistant-Bold.woff2) - format("woff2"), - url(./Assistant-Bold.woff2) format("woff2"); + src: url(./Assistant/Assistant-Bold.woff2) format("woff2"); font-weight: 700; style: normal; display: swap;