diff --git a/sites/backend/example.env b/sites/backend/example.env index c9ae1fa21c2..549f5125689 100644 --- a/sites/backend/example.env +++ b/sites/backend/example.env @@ -38,13 +38,16 @@ BACKEND_WEBSITE_SCHEME=https ##################################################################### # For users -#BACKEND_AVATAR_USER=https://freesewing.org/avatar.svg +#BACKEND_AVATAR_USER=default-avatar # For measurement sets -#BACKEND_AVATAR_SET=https://freesewing.org/avatar.svg +#BACKEND_AVATAR_SET=default-avatar + +# For curated measurement sets +#BACKEND_AVATAR_CSET=default-avatar # For patterns -#BACKEND_AVATAR_PATTERN=https://freesewing.org/avatar.svg +#BACKEND_AVATAR_PATTERN=default-avatar ##################################################################### @@ -110,38 +113,23 @@ BACKEND_AWS_SES_BCC='["FreeSewing records "]' ##################################################################### -# Sanity # +# Clouldflare images # # # -# We use Sanity to store the avatars of our users # +# We use Cloudflare's images service to store images # ##################################################################### -# Set this to no to disable Sanity altogther +# Set this to no to disable Cloudflare images altogther # This will cause uploading images to not work -BACKEND_ENABLE_SANITY=no +BACKEND_ENABLE_CLOUDFLARE_IMAGES=yes -# Sanity project ID -#SANITY_PROJECT +# Cloudflare account ID +BACKEND_CLOUDFLARE_ACCOUNT=yourAccountIdHere -# Sanity dataset -#SANITY_DATASET +# Cloudflare Image Delivery URL +BACKEND_CLOUDFLARE_IMAGE_URL=https://imagedelivery.net/ouSuR9yY1bHt-fuAokSA5Q -# Sanity access token -#SANITY_TOKEN - -# Sanity API version -#SANITY_VERSION=v2022-10-31 - -##################################################################### -# Payments via Stripe # -# # -# We use Stripe as payments processor for patron contributions # -##################################################################### - -# Set this to no to disable Stripe payments altogther -BACKEND_ENABLE_PAYMENTS=no - -# Stripe API key with permissions to create a payment intent -#BACKEND_STRIPE_CREATE_INTENT_KEY=yourKeyHere +# Cloudflare API token +BACKEND_CLOUDFLARE_IMAGES_TOKEN=yourTokenHere ##################################################################### # Github integration # @@ -206,8 +194,8 @@ BACKEND_TESTS_DOMAIN=freesewing.dev # Only relevant if BACKEND_TEST_ALLOW is true #BACKEND_ENABLE_TESTS_EMAIL=yes -# Whether toinclude the (slow) tests that involve reaching out to Sanity -#BACKEND_ENABLE_TESTS_SANITY=yes +# Whether toinclude the (slow) tests that involve reaching out to cloudflare +#BACKEND_ENABLE_TESTS_CLOUDFLARE_IMAGES=yes ##################################################################### diff --git a/sites/backend/src/config.mjs b/sites/backend/src/config.mjs index 3532c0d567f..7b3d503854e 100644 --- a/sites/backend/src/config.mjs +++ b/sites/backend/src/config.mjs @@ -42,12 +42,12 @@ const baseConfig = { github: envToBool(process.env.BACKEND_ENABLE_OAUTH_GITHUB), google: envToBool(process.env.BACKEND_ENABLE_OAUTH_GOOGLE), }, - sanity: envToBool(process.env.BACKEND_ENABLE_SANITY), + cloudflareImages: envToBool(process.env.BACKEND_ENABLE_CLOUDFLARE_IMAGES), ses: envToBool(process.env.BACKEND_ENABLE_AWS_SES), tests: { base: envToBool(process.env.BACKEND_ENABLE_TESTS), email: envToBool(process.env.BACKEND_ENABLE_TESTS_EMAIL), - sanity: envToBool(process.env.BACKEND_ENABLE_TESTS_SANITY), + cloudflareImages: envToBool(process.env.BACKEND_ENABLE_TESTS_CLOUDFLARE_IMAGES), }, import: envToBool(process.env.BACKEND_ENABLE_IMPORT), }, @@ -58,9 +58,10 @@ const baseConfig = { expiryMaxSeconds: 365 * 24 * 3600, }, avatars: { - user: process.env.BACKEND_AVATAR_USER || 'https://freesewing.org/avatar.svg', - set: process.env.BACKEND_AVATAR_SET || 'https://freesewing.org/avatar.svg', - pattern: process.env.BACKEND_AVATAR_PATTERN || 'https://freesewing.org/avatar.svg', + user: process.env.BACKEND_AVATAR_USER || 'default-avatar', + set: process.env.BACKEND_AVATAR_SET || 'default-avatar', + cset: process.env.BACKEND_AVATAR_CSET || 'default-avatar', + pattern: process.env.BACKEND_AVATAR_PATTERN || 'default-avatar', }, db: { url: process.env.BACKEND_DB_URL || './db.sqlite', @@ -141,17 +142,15 @@ if (baseConfig.use.github) }, } -// Sanity config -if (baseConfig.use.sanity) - baseConfig.sanity = { - project: process.env.SANITY_PROJECT, - dataset: process.env.SANITY_DATASET || 'site-content', - token: process.env.SANITY_TOKEN || 'fixmeSetSanityToken', - version: process.env.SANITY_VERSION || 'v2022-10-31', - api: `https://${process.env.SANITY_PROJECT || 'missing-project-id'}.api.sanity.io/${ - process.env.SANITY_VERSION || 'v2022-10-31' - }`, +// Cloudflare Images config +if (baseConfig.use.cloudflareImages) { + const account = process.env.BACKEND_CLOUDFLARE_ACCOUNT || 'fixmeSetCloudflareAccountId' + baseConfig.cloudflareImages = { + account, + api: `https://api.cloudflare.com/client/v4/accounts/${account}/images/v1`, + token: process.env.BACKEND_CLOUDFLARE_IMAGES_TOKEN || 'fixmeSetCloudflareToken', } +} // AWS SES config (for sending out emails) if (baseConfig.use.ses) @@ -193,7 +192,7 @@ if (baseConfig.use.oauth?.google) const config = postConfig(baseConfig) // Exporting this stand-alone config -export const sanity = config.sanity || {} +export const cloudflareImages = config.cloudflareImages || {} export const website = config.website const vars = { @@ -206,7 +205,7 @@ const vars = { BACKEND_JWT_EXPIRY: 'optional', // Feature flags BACKEND_ENABLE_AWS_SES: 'optional', - BACKEND_ENABLE_SANITY: 'optional', + BACKEND_ENABLE_CLOUDFLARE_IMAGES: 'optional', BACKEND_ENABLE_GITHUB: 'optional', BACKEND_ENABLE_OAUTH_GITHUB: 'optional', BACKEND_ENABLE_OAUTH_GOOGLE: 'optional', @@ -227,12 +226,10 @@ if (envToBool(process.env.BACKEND_ENABLE_AWS_SES)) { vars.BACKEND_AWS_SES_CC = 'optional' vars.BACKEND_AWS_SES_BCC = 'optional' } -// Vars for Sanity integration -if (envToBool(process.env.BACKEND_USE_SANITY)) { - vars.SANITY_PROJECT = 'required' - vars.SANITY_TOKEN = 'requiredSecret' - vars.SANITY_VERSION = 'optional' - vars.BACKEND_TEST_SANITY = 'optional' +// Vars for Cloudflare Images integration +if (envToBool(process.env.BACKEND_USE_CLOUDFLARE_IMAGES)) { + vars.BACKEND_CLOUDFLARE_IMAGES_TOKEN = 'requiredSecret' + vars.BACKEND_TEST_CLOUDFLARE_IMAGES = 'optional' } // Vars for Github integration if (envToBool(process.env.BACKEND_ENABLE_GITHUB)) { @@ -327,10 +324,13 @@ export function verifyConfig(silent = false) { config.jwt.secretOrKey.slice(0, 4) + '**redacted**' + config.jwt.secretOrKey.slice(-4), }, } - if (config.sanity) - dump.sanity = { - ...config.sanity, - token: config.sanity.token.slice(0, 4) + '**redacted**' + config.sanity.token.slice(-4), + if (config.cloudflareImages) + dump.cloudflareImages = { + ...config.cloudflareImages, + token: + config.cloudflareImages.token.slice(0, 4) + + '**redacted**' + + config.cloudflareImages.token.slice(-4), } console.log(chalk.cyan.bold('Dumping configuration:\n'), asJson(dump, null, 2)) } diff --git a/sites/backend/src/models/curated-set.mjs b/sites/backend/src/models/curated-set.mjs index 40310ce0308..5d4cf46c37e 100644 --- a/sites/backend/src/models/curated-set.mjs +++ b/sites/backend/src/models/curated-set.mjs @@ -1,6 +1,6 @@ import { capitalize } from '../utils/index.mjs' import { log } from '../utils/log.mjs' -import { setSetAvatar } from '../utils/sanity.mjs' +import { storeImage } from '../utils/cloudflare-images.mjs' import yaml from 'js-yaml' export function CuratedSetModel(tools) { @@ -28,7 +28,7 @@ CuratedSetModel.prototype.guardedCreate = async function ({ body, user }) { } if (body.measies) data.measies = this.sanitizeMeasurements(body.measies) else data.measies = {} - // Set this one initially as we need the ID to create a custom img via Sanity + // Set this one initially as we need the ID to store an image on cloudflare data.img = this.config.avatars.set // Create record @@ -36,10 +36,14 @@ CuratedSetModel.prototype.guardedCreate = async function ({ body, user }) { // Update img? (now that we have the ID) const img = - this.config.use.sanity && + this.config.use.cloudflareImages && typeof body.img === 'string' && - (!body.test || (body.test && this.config.use.tests?.sanity)) - ? await setSetAvatar(this.record.id, body.img) + (!body.test || (body.test && this.config.use.tests?.cloudflareImages)) + ? await storeImage({ + id: `cset-${this.record.id}`, + metadata: { user: user.uid }, + b64: body.img, + }) : false if (img) await this.unguardedUpdate({ img: img.url }) @@ -235,7 +239,11 @@ CuratedSetModel.prototype.guardedUpdate = async function ({ params, body, user } // Image (img) if (typeof body.img === 'string') { - const img = await setSetAvatar(params.id, body.img) + const img = await storeImage({ + id: `cset-${this.record.id}`, + metadata: { user: this.user.uid }, + b64: body.img, + }) data.img = img.url } diff --git a/sites/backend/src/models/pattern.mjs b/sites/backend/src/models/pattern.mjs index c37c7f8ae8b..2940998e436 100644 --- a/sites/backend/src/models/pattern.mjs +++ b/sites/backend/src/models/pattern.mjs @@ -1,6 +1,6 @@ import { log } from '../utils/log.mjs' import { capitalize } from '../utils/index.mjs' -import { setPatternAvatar } from '../utils/sanity.mjs' +import { storeImage } from '../utils/cloudflare-images.mjs' import yaml from 'js-yaml' import { SetModel } from './set.mjs' @@ -75,7 +75,7 @@ PatternModel.prototype.guardedCreate = async function ({ body, user }) { // Public if (body.public === true) data.public = true data.userId = user.uid - // Set this one initially as we need the ID to create a custom img via Sanity + // Set this one initially as we need the ID to store an image on cloudflare data.img = this.config.avatars.pattern // Create record @@ -83,10 +83,14 @@ PatternModel.prototype.guardedCreate = async function ({ body, user }) { // Update img? (now that we have the ID) const img = - this.config.use.sanity && + this.config.use.cloudflareImages && typeof body.img === 'string' && - (!body.test || (body.test && this.config.use.tests?.sanity)) - ? await setPatternAvatar(this.record.id, body.img) + (!body.test || (body.test && this.config.use.tests?.cloudflareImages)) + ? await storeImage({ + id: `pattern-${this.record.id}`, + metadata: { user: user.uid }, + b64: body.img, + }) : false if (img) await this.unguardedUpdate(this.cloak({ img: img.url })) @@ -300,7 +304,11 @@ PatternModel.prototype.guardedUpdate = async function ({ params, body, user }) { if (typeof body.settings === 'object') data.settings = body.settings // Image (img) if (typeof body.img === 'string') { - const img = await setPatternAvatar(params.id, body.img) + const img = await storeImage({ + id: `pattern-${this.record.id}`, + metadata: { user: this.user.uid }, + b64: body.img, + }) data.img = img.url } diff --git a/sites/backend/src/models/set.mjs b/sites/backend/src/models/set.mjs index 77f1f30e133..afcae49a85c 100644 --- a/sites/backend/src/models/set.mjs +++ b/sites/backend/src/models/set.mjs @@ -1,5 +1,5 @@ import { log } from '../utils/log.mjs' -import { setSetAvatar, downloadImage } from '../utils/sanity.mjs' +import { storeImage } from '../utils/cloudflare-images.mjs' import yaml from 'js-yaml' export function SetModel(tools) { @@ -32,7 +32,7 @@ SetModel.prototype.guardedCreate = async function ({ body, user }) { else data.measies = {} data.imperial = body.imperial === true ? true : false data.userId = user.uid - // Set this one initially as we need the ID to create a custom img via Sanity + // Set this one initially as we need the ID to store the image on cloudflare data.img = this.config.avatars.set // Create record @@ -40,10 +40,18 @@ SetModel.prototype.guardedCreate = async function ({ body, user }) { // Update img? (now that we have the ID) const img = - this.config.use.sanity && + this.config.use.cloudflareImages && typeof body.img === 'string' && - (!body.test || (body.test && this.config.use.tests?.sanity)) - ? await setSetAvatar(this.record.id, body.img, this.clear.name) + (!body.test || (body.test && this.config.use.tests?.cloudflareImages)) + ? await storeImage({ + id: `set-${this.record.id}`, + metadata: { + user: user.uid, + name: this.clear.name, + }, + b64: body.img, + requireSignedURLs: true, + }) : false if (img) await this.unguardedUpdate(this.cloak({ img: img.url })) @@ -264,7 +272,15 @@ SetModel.prototype.guardedUpdate = async function ({ params, body, user }) { // Image (img) if (typeof body.img === 'string') { - const img = await setSetAvatar(params.id, body.img, data.name || this.clear.name) + const img = await replaceImage({ + id: `set-${this.record.id}`, + metadata: { + user: user.uid, + name: this.clear.name, + }, + b64: body.img, + notPublic: true, + }) data.img = img.url } diff --git a/sites/backend/src/models/subscriber.mjs b/sites/backend/src/models/subscriber.mjs index 4add0f7d193..d65667dfc04 100644 --- a/sites/backend/src/models/subscriber.mjs +++ b/sites/backend/src/models/subscriber.mjs @@ -1,5 +1,4 @@ import { hash, randomString } from '../utils/crypto.mjs' -import { setUserAvatar } from '../utils/sanity.mjs' import { log } from '../utils/log.mjs' import { clean, i18nUrl } from '../utils/index.mjs' diff --git a/sites/backend/src/models/user.mjs b/sites/backend/src/models/user.mjs index bf0f02ddfac..b16e27ce0ae 100644 --- a/sites/backend/src/models/user.mjs +++ b/sites/backend/src/models/user.mjs @@ -1,7 +1,7 @@ import jwt from 'jsonwebtoken' import { log } from '../utils/log.mjs' import { hash, hashPassword, randomString, verifyPassword } from '../utils/crypto.mjs' -import { setUserAvatar, downloadImage } from '../utils/sanity.mjs' +import { storeImage, replaceImage } from '../utils/cloudflare-images.mjs' import { clean, asJson, i18nUrl } from '../utils/index.mjs' import { ConfirmationModel } from './confirmation.mjs' import { SetModel } from './set.mjs' @@ -250,7 +250,7 @@ UserModel.prototype.guardedCreate = async function ({ body }) { password: asJson(hashPassword(randomString())), // We'll change this later github: this.encrypt(''), bio: this.encrypt(''), - // Set this one initially as we need the ID to create a custom img via Sanity + // Set this one initially as we need the ID to store an image on Cloudflare img: this.encrypt(this.config.avatars.user), } // During tests, users can set their own permission level so you can test admin stuff @@ -544,10 +544,15 @@ UserModel.prototype.guardedUpdate = async function ({ body, user }) { } } // Image (img) - if (typeof body.img === 'string') { - const img = await setUserAvatar(this.record.ihash, body.img, data.username) - data.img = img.url - } + if (typeof body.img === 'string') + data.img = await replaceImage({ + id: `user-${this.record.ihash}`, + metadata: { + user: user.uid, + ihash: this.record.ihash, + }, + b64: body.img, + }) // Now update the record await this.unguardedUpdate(this.cloak(data)) diff --git a/sites/backend/src/utils/cloudflare-images.mjs b/sites/backend/src/utils/cloudflare-images.mjs new file mode 100644 index 00000000000..6abff757691 --- /dev/null +++ b/sites/backend/src/utils/cloudflare-images.mjs @@ -0,0 +1,84 @@ +import axios from 'axios' +import { cloudflareImages as config } from '../config.mjs' + +// We'll use this a bunch +const headers = { Authorization: `Bearer ${config.token}` } + +/* + * Method that does the actual image upload to cloudflare + * Use this for a new image that does not yet exist + */ +export async function storeImage(props) { + const form = getFormData(props) + let result + try { + result = await axios.post(config.api, form, { headers }) + } catch (err) { + if (err.response.status == 409) { + /* + * Image already exists. + * Cloudflare does not support udating the image, + * so we need to delete it and upload again + */ + try { + log.warn(props.id, 'Called storeImage when replaceImage should have been used') + await axios.delete(`${config.api}/${props.id}`) + result = await axios.post(config.api, form, { headers }) + } catch (err) { + console.log('Failed to replace image on cloudflare', err) + } + } else { + console.log('Failed to upload image to cloudflare', err) + } + } + + return result.data?.result?.id ? result.data.result.id : false +} + +/* + * Method that does the actual image upload to cloudflare + * Use this to replace an existing image + */ +export async function replaceImage(props) { + const form = getFormData(props) + // Ignore errors on delete, probably means the image does not exist + try { + await axios.delete(`${config.api}/${props.id}`) + } catch (err) {} + let result + try { + result = await axios.post(config.api, form, { headers }) + } catch (err) { + console.log('Failed to replace image on cloudflare', err) + } + + return result.data?.result?.id ? result.data.result.id : false +} + +/* + * Helper method to construct the form data for cloudflare + */ +function getFormData({ id, metadata, url = false, b64 = false, blob = false, notPublic = false }) { + const form = new FormData() + form.append('id', id) + form.append('metadata', JSON.stringify(metadata)) + // Handle base-64 encoded data + if (b64) form.append('file', b64ToBlob(b64), id) + // Handle binary data + else if (blob) form.append('file', blob) + // Handle URL-based upload + else if (url) form.append('url', url) + // Handle requireSignedURLs + if (notPublic) form.append('requireSignedURLs', true) + + return form +} + +/* + * Helper method to turn a data-uri into binary data + */ +function b64ToBlob(dataUri) { + const [start, data] = dataUri.split(';base64,') + + return new Blob([new Buffer.from(data, 'base64')]) +} diff --git a/sites/backend/src/utils/sanity.mjs b/sites/backend/src/utils/sanity.mjs deleted file mode 100644 index d522aa59381..00000000000 --- a/sites/backend/src/utils/sanity.mjs +++ /dev/null @@ -1,98 +0,0 @@ -import axios from 'axios' -import { sanity as config } from '../config.mjs' -import { createClient } from '@sanity/client' - -const sanity = createClient({ - projectId: config.project, - dataset: config.dataset, - useCdn: true, // Set to false to bypass cache - apiVersion: config.version, - token: config.token, -}) - -// We'll use this a bunch -const headers = { Authorization: `Bearer ${config.token}` } - -/* - * Turns a sanity asset _ref into an image URL - */ -function imageUrl(ref) { - return `https://cdn.sanity.io/images/${config.project}/${config.dataset}/${ref.slice(6)}` -} - -/* - * Retrieval of avatar images from the Sanity API - */ -export const getUserAvatar = async (id) => getAvatar('user', id) -export const getSetAvatar = async (id) => getAvatar('set', id) -async function getAvatar(type, id) { - const url = - `${config.api}/data/query/${config.dataset}?query=` + - encodeURIComponent(`*[_type=='${type}img' && recordid==${id}]{ img }`) - const result = await axios.get(url, { headers }) - if (result.data?.result && Array.isArray(result.data.result) && result.data.result.length === 1) { - return imageUrl(result.data.result[0].img.asset._ref) - } - - return false -} - -/* - * Uploads an image to sanity - */ -export const setUserAvatar = async (id, data, title) => setAvatar('user', id, data, title) -export const setSetAvatar = async (id, data, title) => setAvatar('set', id, data, title) -export const setPatternAvatar = async (id, data, title) => setAvatar('pattern', id, data, title) -export async function setAvatar(type, id, data, title) { - // Step 1: Upload the image as asset - const [contentType, binary] = Array.isArray(data) ? data : b64ToBinaryWithType(data) - if (!contentType) return '' - - const imgDocument = await sanity.assets.upload('image', binary, { - contentType, - filename: `${type}.${contentType.split('/').pop()}`, - }) - const document = await sanity.createOrReplace({ - _id: `${type}-${id}`, - _type: `${type}img`, - title: title, - recordid: id, - img: { - asset: { - _ref: imgDocument._id, - _type: 'reference', - }, - }, - }) - - return document._id -} - -/* - * Helper method to turn a data-uri into binary data + content type - */ -function b64ToBinaryWithType(dataUri) { - let type = false - const [start, data] = dataUri.split(';base64,') - if (start.includes('image/png')) type = 'image/png' - else if (start.includes('image/jpg')) type = 'image/jpeg' - else if (start.includes('image/jpeg')) type = 'image/jpeg' - - return [type, new Buffer.from(data, 'base64')] -} - -/* - * Helper method to download an image - * Used in import only, thus ok for removal post v3 release - */ -export const downloadImage = async (url) => { - let result - try { - result = await axios.get(url, { responseType: 'arraybuffer' }) - } catch (err) { - console.log(`Could not download from ${url}`, err, result) - } - - // Returning [contentType, data] - return [result.headers['content-type'], Buffer.from(result.data, 'binary')] -} diff --git a/sites/backend/tests/account.mjs b/sites/backend/tests/account.mjs index 6300e8f0588..a5335275da0 100644 --- a/sites/backend/tests/account.mjs +++ b/sites/backend/tests/account.mjs @@ -188,8 +188,11 @@ export const accountTests = async (chai, config, expect, store) => { done() }) }) - - if (store.config.tests.includeSanity) { + /* + * Running this twice immeadiatly (jwt and key) will break because cloudflare api + * will not be ready yet + */ + if (store.config.use.tests.cloudflareImages && auth === 'jwt') { it(`${store.icon('user', auth)} Should update the account img (${auth})`, (done) => { chai .request(config.api) diff --git a/sites/backend/tests/shared.mjs b/sites/backend/tests/shared.mjs index c16c3ee9569..b3e08756675 100644 --- a/sites/backend/tests/shared.mjs +++ b/sites/backend/tests/shared.mjs @@ -124,6 +124,4 @@ export const setup = async () => { return { chai, config, expect, store } } -export const teardown = async function (/*store*/) { - //console.log(store) -} +export const teardown = async function (/*store*/) {}