diff --git a/sites/backend/scripts/rmdb.mjs b/sites/backend/scripts/rmdb.mjs index 5935a6d833f..0845fc6ccc0 100644 --- a/sites/backend/scripts/rmdb.mjs +++ b/sites/backend/scripts/rmdb.mjs @@ -6,11 +6,18 @@ import dotenv from 'dotenv' dotenv.config() const rmdb = async () => { + // Figure out what file would be removed so we can include it in the warning + let db = process.env.BACKEND_DB_URL + // Deal with prisma considering its own folder as the root + if (db.slice(0, 7) === 'file:./') db = './prisma/' + db.slice(7) + else db = db.slice(6) + console.log({ db }) + // Say hi console.log(banner + '\n') console.log(` - 🚨 This will ${chalk.yellow('remove your database')} + 🚨 This will ${chalk.yellow('remove your database')} : ${chalk.bold.red(db)} ⚠️ There is ${chalk.bold('no way back')} from this - proceed with caution `) @@ -26,7 +33,6 @@ const rmdb = async () => { if (answer.confirms) { console.log() // Nuke it from orbit - const db = process.env.BACKEND_DB_URL.slice(6) fs.access(db, fs.constants.W_OK, (err) => { if (err) console.log(` ⛔ Cannot remove ${chalk.green(db)} 🤔`) else { diff --git a/sites/backend/src/config.mjs b/sites/backend/src/config.mjs index 7b3d503854e..7667b0193a0 100644 --- a/sites/backend/src/config.mjs +++ b/sites/backend/src/config.mjs @@ -149,6 +149,7 @@ if (baseConfig.use.cloudflareImages) { account, api: `https://api.cloudflare.com/client/v4/accounts/${account}/images/v1`, token: process.env.BACKEND_CLOUDFLARE_IMAGES_TOKEN || 'fixmeSetCloudflareToken', + import: envToBool(process.env.BACKEND_IMPORT_CLOUDFLARE_IMAGES), } } diff --git a/sites/backend/src/models/pattern.mjs b/sites/backend/src/models/pattern.mjs index 2940998e436..78291cfcda2 100644 --- a/sites/backend/src/models/pattern.mjs +++ b/sites/backend/src/models/pattern.mjs @@ -431,3 +431,164 @@ PatternModel.prototype.asPublicPattern = function () { return data } + +const migratePattern = (v2, userId) => ({ + createdAt: new Date(v2.created ? v2.created : v2.createdAt), + data: { version: v2.data.version, notes: ['Migrated from version 2'] }, + design: v2.design || v2.data.design, + name: v2.name || '--', + notes: v2.notes ? v2.notes + '\n\nMigrated from v2' : 'Migrated from v2', + settings: v2.data.settings, + userId, +}) + +const v2lut = { + 'size 28, with breasts': 1, + 'size 30, with breasts': 2, + 'size 32, with breasts': 3, + 'size 34, with breasts': 4, + 'size 36, with breasts': 5, + 'size 38, with breasts': 6, + 'size 40, with breasts': 7, + 'size 42, with breasts': 8, + 'size 44, with breasts': 9, + 'size 46, with breasts': 10, + 'size-28-b': 1, + 'size-30-b': 2, + 'size-32-b': 3, + 'size-34-b': 4, + 'size-36-b': 5, + 'size-38-b': 6, + 'size-40-b': 7, + 'size-42-b': 8, + 'size-44-b': 9, + 'size-46-b': 10, + 'size-28-with-breasts': 1, + 'size-30-with-breasts': 2, + 'size-32-with-breasts': 3, + 'size-34-with-breasts': 4, + 'size-36-with-breasts': 5, + 'size-38-with-breasts': 6, + 'size-40-with-breasts': 7, + 'size-42-with-breasts': 8, + 'size-44-with-breasts': 9, + 'size-46-with-breasts': 10, + 'größe 28, mit brüsten': 1, + 'größe 30, mit brüsten': 2, + 'größe 32, mit brüsten': 3, + 'größe 34, mit brüsten': 4, + 'größe 36, mit brüsten': 5, + 'größe 38, mit brüsten': 6, + 'größe 40, mit brüsten': 7, + 'größe 42, mit brüsten': 8, + 'größe 44, mit brüsten': 9, + 'größe 46, mit brüsten': 10, + 'taille 28, avec des seins': 1, + 'taille 30, avec des seins': 2, + 'taille 32, avec des seins': 3, + 'taille 34, avec des seins': 4, + 'taille 36, avec des seins': 5, + 'taille 38, avec des seins': 6, + 'taille 40, avec des seins': 7, + 'taille 42, avec des seins': 8, + 'taille 44, avec des seins': 9, + 'taille 46, avec des seins': 10, + 'tamaño 28, con pechos': 1, + 'tamaño 30, con pechos': 2, + 'tamaño 32, con pechos': 3, + 'tamaño 34, con pechos': 4, + 'tamaño 36, con pechos': 5, + 'tamaño 38, con pechos': 6, + 'tamaño 40, con pechos': 7, + 'tamaño 42, con pechos': 8, + 'tamaño 44, con pechos': 9, + 'tamaño 46, con pechos': 10, + + 'size 32, without breasts': 11, + 'size 34, without breasts': 12, + 'size 36, without breasts': 13, + 'size 38, without breasts': 14, + 'size 40, without breasts': 15, + 'size 42, without breasts': 16, + 'size 44, without breasts': 17, + 'size 46, without breasts': 18, + 'size 48, without breasts': 19, + 'size 50, without breasts': 20, + 'taille 32, sans seins': 11, + 'taille 34, sans seins': 12, + 'taille 36, sans seins': 13, + 'taille 38, sans seins': 14, + 'taille 40, sans seins': 15, + 'taille 42, sans seins': 16, + 'taille 44, sans seins': 17, + 'taille 46, sans seins': 18, + 'taille 48, sans seins': 19, + 'taille 50, sans seins': 20, + 'size-32-a': 11, + 'size-34-a': 12, + 'size-36-a': 13, + 'size-38-a': 14, + 'size-40-a': 15, + 'size-42-a': 16, + 'size-44-a': 17, + 'size-46-a': 18, + 'size-48-a': 19, + 'size-50-a': 20, + 'maat 32, zonder borsten': 11, + 'maat 34, zonder borsten': 12, + 'maat 36, zonder borsten': 13, + 'maat 38, zonder borsten': 14, + 'maat 40, zonder borsten': 15, + 'maat 42, zonder borsten': 16, + 'maat 44, zonder borsten': 17, + 'maat 46, zonder borsten': 18, + 'maat 48, zonder borsten': 19, + 'maat 50, zonder borsten': 20, + 'größe 32, ohne brüste': 11, + 'größe 34, ohne brüste': 12, + 'größe 36, ohne brüste': 13, + 'größe 38, ohne brüste': 14, + 'größe 40, ohne brüste': 15, + 'größe 42, ohne brüste': 16, + 'größe 44, ohne brüste': 17, + 'größe 46, ohne brüste': 18, + 'größe 48, ohne brüste': 19, + 'größe 50, ohne brüste': 20, + 'tamaño 32, sin pechos': 11, + 'tamaño 34, sin pechos': 12, + 'tamaño 36, sin pechos': 13, + 'tamaño 38, sin pechos': 14, + 'tamaño 40, sin pechos': 15, + 'tamaño 42, sin pechos': 16, + 'tamaño 44, sin pechos': 17, + 'tamaño 46, sin pechos': 18, + 'tamaño 48, sin pechos': 19, + 'tamaño 50, sin pechos': 20, +} +/* + * This is a special route not available for API users + */ +PatternModel.prototype.import = async function (v2user, lut, userId) { + for (const [handle, pattern] of Object.entries(v2user.patterns)) { + let skip = false + const data = { ...migratePattern(pattern, userId), userId } + if (lut[pattern.person]) data.setId = lut[pattern.person] + else if (v2lut[pattern.person]) data.csetId = v2lut[pattern.person] + else if (pattern.person.length !== 5 && !['any', 'original'].includes(pattern.person)) { + console.log(`Cannot find ${pattern.person}`, pattern, { lut, v2lut }) + process.exit() + } + if (!data.design || ['theo', 'ursula', 'unice'].includes(data.design)) skip = true + if (!skip) { + // V2 does not support images for patterns + data.img = 'default-avatar' + const cloaked = await this.cloak(data) + try { + this.record = await this.prisma.pattern.create({ data: cloaked }) + } catch (err) { + log.warn(err, 'Could not create pattern') + console.log(data) + } + } + } +} diff --git a/sites/backend/src/models/set.mjs b/sites/backend/src/models/set.mjs index 792cd09735c..f040e08ee96 100644 --- a/sites/backend/src/models/set.mjs +++ b/sites/backend/src/models/set.mjs @@ -1,5 +1,5 @@ import { log } from '../utils/log.mjs' -import { replaceImage, storeImage } from '../utils/cloudflare-images.mjs' +import { replaceImage, storeImage, ensureImage, importImage } from '../utils/cloudflare-images.mjs' import yaml from 'js-yaml' export function SetModel(tools) { @@ -450,7 +450,6 @@ SetModel.prototype.sanitizeMeasurements = function (input) { const migratePerson = (v2) => ({ createdAt: new Date(v2.created ? v2.created : v2.createdAt), - img: v2.picture, imperial: v2.units === 'imperial', name: v2.name || '--', // Encrypted, so always set _some_ value notes: v2.notes || '--', // Encrypted, so always set _some_ value @@ -462,11 +461,13 @@ const migratePerson = (v2) => ({ * This is a special route not available for API users */ SetModel.prototype.import = async function (v2user, userId) { - for (const person of v2user.people) { + const lut = {} // lookup tabel for v2 handle to v3 id + for (const [handle, person] of Object.entries(v2user.people)) { const data = { ...migratePerson(person), userId } await this.unguardedCreate(data) // Now that we have an ID, we can handle the image - if (data.img) { + if (person.picture && person.picture.slice(-4) !== '.svg') { + const imgId = `set-${this.record.id}` const imgUrl = 'https://static.freesewing.org/users/' + encodeURIComponent(v2user.handle.slice(0, 1)) + @@ -475,14 +476,26 @@ SetModel.prototype.import = async function (v2user, userId) { '/people/' + encodeURIComponent(person.handle) + '/' + - encodeURIComponent(data.img) - console.log('Grabbing', imgUrl) - //const [contentType, imgData] = await downloadImage(imgUrl) - //// Do not import the default SVG avatar - //if (contentType !== 'image/svg+xml') { - // const img = await setSetAvatar(this.record.id, [contentType, imgData], data.name) - // data.img = img - //} + encodeURIComponent(person.picture) + data.img = await importImage({ + id: imgId, + metadata: { + user: userId, + v2PersonHandle: handle, + }, + url: imgUrl, + }) + data.img = imgId + } else data.img = 'default-avatar' + const cloaked = await this.cloak(data) + try { + this.record = await this.prisma.set.create({ data: cloaked }) + lut[handle] = this.record.id + } catch (err) { + log.warn(err, 'Could not create set') + console.log(person) } } + + return lut } diff --git a/sites/backend/src/models/user.mjs b/sites/backend/src/models/user.mjs index 16e82d6cf9a..81aaec43ea5 100644 --- a/sites/backend/src/models/user.mjs +++ b/sites/backend/src/models/user.mjs @@ -1,10 +1,11 @@ import jwt from 'jsonwebtoken' import { log } from '../utils/log.mjs' import { hash, hashPassword, randomString, verifyPassword } from '../utils/crypto.mjs' -import { replaceImage, ensureImage } from '../utils/cloudflare-images.mjs' +import { replaceImage, ensureImage, importImage } from '../utils/cloudflare-images.mjs' import { clean, asJson, i18nUrl } from '../utils/index.mjs' import { ConfirmationModel } from './confirmation.mjs' import { SetModel } from './set.mjs' +import { PatternModel } from './pattern.mjs' export function UserModel(tools) { this.config = tools.config @@ -19,6 +20,7 @@ export function UserModel(tools) { this.clear = {} // For holding decrypted data // Only used for import, can be removed after v3 is released this.Set = new SetModel(tools) + this.Pattern = new PatternModel(tools) return this } @@ -910,10 +912,10 @@ UserModel.prototype.import = async function (list) { const data = migrateUser(sub) await this.read({ ehash: data.ehash }) if (!this.record) { - /* - * Grab the image from the FreeSewing server and upload it to Sanity - */ if (data.img) { + /* + * Figure out what image to grab from the FreeSewing v2 backend server + */ const imgId = `user-${data.ihash}` const imgUrl = 'https://static.freesewing.org/users/' + @@ -922,7 +924,7 @@ UserModel.prototype.import = async function (list) { encodeURIComponent(sub.handle) + '/' + encodeURIComponent(data.img) - data.img = await ensureImage({ + data.img = await importImage({ id: imgId, metadata: { user: `v2-${sub.handle}`, @@ -957,7 +959,9 @@ UserModel.prototype.import = async function (list) { } } else skipped.push(sub.email) // That's the user, now load their people as sets - //if (sub.people) await this.Set.import(sub, this.record.id) + let lut = false + if (sub.people) lut = await this.Set.import(sub, this.record.id) + if (sub.patterns) await this.Pattern.import(sub, lut, this.record.id) } else skipped.push(sub.email) } diff --git a/sites/backend/src/utils/cloudflare-images.mjs b/sites/backend/src/utils/cloudflare-images.mjs index 81e82043fb6..d62e8d0aefc 100644 --- a/sites/backend/src/utils/cloudflare-images.mjs +++ b/sites/backend/src/utils/cloudflare-images.mjs @@ -71,11 +71,25 @@ export async function ensureImage(props) { result = await axios.post(config.api, form, { headers }) } catch (err) { // It's fine + console.log(err) } return props.id } +/* + * Method that imports and image from URL and does not bother waiting for the answer + */ +export async function importImage(props) { + // Bypass slow ass upload when testing import + if (!config.import) return `default-avatar` + + const form = getFormData(props) + await axios.post(config.api, form, { headers }) + + return props.id +} + /* * Helper method to construct the form data for cloudflare */ diff --git a/sites/backend/v2-v3/import.mjs b/sites/backend/v2-v3/import.mjs index c588c8f396a..95fc8b1cff5 100644 --- a/sites/backend/v2-v3/import.mjs +++ b/sites/backend/v2-v3/import.mjs @@ -1,12 +1,16 @@ +import dotenv from 'dotenv' //import subscribers from './v2-newsletters.json' assert { type: 'json' } import users from '../dump/v2-users.json' assert { type: 'json' } import people from '../dump/v2-people.json' assert { type: 'json' } import patterns from '../dump/v2-patterns.json' assert { type: 'json' } +dotenv.config() + +const batchSize = 100 /* * Only this token allows exporting data */ -const import_token = 'TOKEN_HERE' +const import_token = process.env.IMPORT_TOKEN /* * Where to connect to? @@ -28,7 +32,7 @@ const importSubscribers = async () => { console.log('Importing subscribers') const count = subscribers.length let total = 0 - const batches = splitArray(subscribers, 50) + const batches = splitArray(subscribers, batchSize) for (const batch of batches) { const result = await fetch(`${BACKEND}/import/subscribers`, { method: 'POST', @@ -83,7 +87,7 @@ const importUsers = async () => { console.log('Importing users') const count = todo.length let total = 0 - const batches = splitArray(todo, 50) + const batches = splitArray(todo, batchSize) for (const batch of batches) { const result = await fetch(`${BACKEND}/import/users`, { method: 'POST', @@ -96,9 +100,8 @@ const importUsers = async () => { }), }) const data = await result.json() - total += data.imported - console.log(`${total}/${count} (${data.skipped} skipped)`) - console.log(data) + total += batchSize + console.log(`${total}/${count}`) } } diff --git a/sites/backend/v2-v3/remove-set-images.mjs b/sites/backend/v2-v3/remove-set-images.mjs new file mode 100644 index 00000000000..16f5b04167e --- /dev/null +++ b/sites/backend/v2-v3/remove-set-images.mjs @@ -0,0 +1,20 @@ +import { cloudflareImages as config } from '../src/config.mjs' +import axios from 'axios' + +const headers = { Authorization: `Bearer ${config.token}` } + +const toRemove = [] + +const result = await axios.get(`${config.api}?page=1&per_page=10000`, { headers }) + +const images = result.data.result.images.map((i) => i.id).filter((id) => id.slice(0, 4) === 'set-') +const total = images.length +if (total > 0) { + console.log(`${total} images to remove`) + let i = 1 + for (const id of images) { + console.log(`${i}/${total} : Removing ${id}`) + await axios.delete(`${config.api}/${id}`, { headers }) + i++ + } +} else console.log('No images to remove')