From 550830310ce9826f253ac6e7cdbbe06079391575 Mon Sep 17 00:00:00 2001 From: joostdecock Date: Sun, 6 Aug 2023 18:27:36 +0200 Subject: [PATCH] wip(backend): Working on migration of user data --- scripts/email-lib.mjs | 56 ++++++++ scripts/send-real-email.mjs | 96 ++++++++++++++ scripts/send-test-email.mjs | 68 ++++++++++ sites/backend/src/controllers/imports.mjs | 42 ++++-- sites/backend/src/models/set.mjs | 45 ++++++- sites/backend/src/models/subscriber.mjs | 33 +++++ sites/backend/src/models/user.mjs | 123 +++++++++++++++++- .../src/routes/{import.mjs => imports.mjs} | 8 +- sites/backend/src/routes/index.mjs | 2 + .../src/templates/email/shared/blocks.mjs | 32 +++-- sites/backend/src/utils/crypto.mjs | 1 + sites/backend/src/utils/sanity.mjs | 92 ++++++------- sites/backend/v2-v3/import.mjs | 100 ++++++++++++++ sites/backend/v2-v3/inactive.mjs | 22 ++++ sites/backend/v2-v3/migrate-export.mjs | 38 ++++++ sites/sanity/sanity.config.js | 2 +- sites/sanity/schemas/avatar.js | 49 +++++++ sites/sanity/schemas/index.js | 4 +- 18 files changed, 734 insertions(+), 79 deletions(-) create mode 100644 scripts/email-lib.mjs create mode 100644 scripts/send-real-email.mjs create mode 100644 scripts/send-test-email.mjs rename sites/backend/src/routes/{import.mjs => imports.mjs} (63%) create mode 100644 sites/backend/v2-v3/import.mjs create mode 100644 sites/backend/v2-v3/inactive.mjs create mode 100644 sites/backend/v2-v3/migrate-export.mjs diff --git a/scripts/email-lib.mjs b/scripts/email-lib.mjs new file mode 100644 index 00000000000..9057c56a76f --- /dev/null +++ b/scripts/email-lib.mjs @@ -0,0 +1,56 @@ +import path from 'path' +import { fileURLToPath } from 'url' +import { SESv2Client, SendEmailCommand } from '@aws-sdk/client-sesv2' + +// Current working directory +const cwd = path.dirname(fileURLToPath(import.meta.url)) + +export const send = async ({ + to = ['joost@joost.at'], + bcc = [], + subject = false, + html = false, + text = false, +}) => { + if (!subject || !html || !text) return console.log('No subject, html, or text provided') + + const us = 'Joost from FreeSewing ' + + // Oh AWS your APIs are such a clusterfuck + const client = new SESv2Client({ region: 'us-east-1' }) + + // Via API + const command = new SendEmailCommand({ + ConfigurationSetName: 'backend', + Content: { + Simple: { + Body: { + Text: { + Charset: 'utf-8', + Data: text, + }, + Html: { + Charset: 'utf-8', + Data: html, + }, + }, + Subject: { + Charset: 'utf-8', + Data: subject, + }, + }, + }, + Destination: { + ToAddresses: to, + BccAddresses: to, + }, + FromEmailAddress: us, + }) + try { + const result = await client.send(command) + console.log(result) + } catch (err) { + console.log(err) + return false + } +} diff --git a/scripts/send-real-email.mjs b/scripts/send-real-email.mjs new file mode 100644 index 00000000000..6dd94a1d050 --- /dev/null +++ b/scripts/send-real-email.mjs @@ -0,0 +1,96 @@ +import 'dotenv/config' +import mustache from 'mustache' +import { send } from './email-lib.mjs' +import { + buttonRow, + newsletterClosingRow, + headingRow, + lead1Row, + wrap, +} from '../sites/backend/src/templates/email/shared/blocks.mjs' +import users from '../sites/backend/dump/inactive.json' assert { type: 'json' } + +const splitArray = (split, batchSize) => + split.reduce((result, item, index) => { + const batchIndex = Math.floor(index / batchSize) + if (!result[batchIndex]) result[batchIndex] = [] + result[batchIndex].push(item) + + return result + }, []) + +const text = ` +Dear FreeSewing user, + +Your account on FreeSewing.org has been inactive an scheduled for removal. + +If you want to keep your account, please login on FreeSewing.org within the next 2 weeks. + +If your account stays dormant, it will at one point become unavailable. +That does not have to be a disaster, you can always create a new one, but you will loose all data stored in your account. + +This will be the only email I sent you about this. +If you have questions, you can reply to this message. + +love, +joost +` +const html = mustache.render( + wrap.html(` + ${headingRow.html} + ${lead1Row.html} + ${buttonRow.html} + ${newsletterClosingRow.html} + `), + { + intro: 'Your account is inactive. Login now to revive it.', + heading: 'Your FreeSewing account is inactive and marked for removal', + lead: ` + Due to inactivity, your account on FreeSewing.org has been marked for removal. +

+ If you want to keep your account, please login on FreeSewing.org within the next 2 weeks.`, + button: 'Go to FreeSewing.org', + actionUrl: 'https://freesewing.org/login', + closing: ` + If your account stays dormant, it will at one point become unavailable. +

+ That does not have to be a disaster, you can always create a new one, but you will loose all data stored in your account. +

+ This will be the only email I sent you about this. +
+ If you have questions, you can reply to this message.`, + + greeting: `love,`, + website: 'FreeSewing.org', + seeWhy: 'You received this email because removing inactive accounts is in line with our', + urlWhy: 'https://freesewing.org/docs/various/privacy/account/', + whyDidIGetThis: 'Privacy Notice', + notMarketing: 'This is not marketing, but a transactional email about your FreeSewing account.', + } +) + +/* + * AWS limit is 50 recipients. + * So rather than send a bunch of individual emails, + * let's send it to ourselves * and put 45 people in BCC + */ +const batches = splitArray(users, 45) + +let i = 0 +const total = batches.length +batches.forEach((batch, i) => { + /* + * Maximum email rate is 14 messages per second + * So we can't go too fast + */ + setTimeout(() => { + console.log(`Sending ${i}/${total}`) + send({ + to: ['info@freesewing.org'], + bcc: batch, + subject: '[FreeSewing] Your account is inactive and marked for removal', + html, + text, + }) + }, i * 100) +}) diff --git a/scripts/send-test-email.mjs b/scripts/send-test-email.mjs new file mode 100644 index 00000000000..5f617d522cb --- /dev/null +++ b/scripts/send-test-email.mjs @@ -0,0 +1,68 @@ +import 'dotenv/config' +import mustache from 'mustache' +import { send } from './email-lib.mjs' +import { + buttonRow, + newsletterClosingRow, + headingRow, + lead1Row, + wrap, +} from '../sites/backend/src/templates/email/shared/blocks.mjs' + +const text = ` +Dear FreeSewing user, + +Your account on FreeSewing.org has been inactive an scheduled for removal. + +If you want to keep your account, please login on FreeSewing.org within the next 2 weeks. + +If your account stays dormant, it will at one point become unavailable. +That does not have to be a disaster, you can always create a new one, but you will loose all data stored in your account. + +This will be the only email I sent you about this. +If you have questions, you can reply to this message. + +love, +joost +` +const html = mustache.render( + wrap.html(` + ${headingRow.html} + ${lead1Row.html} + ${buttonRow.html} + ${newsletterClosingRow.html} + `), + { + intro: 'Your account is inactive. Login now to revive it.', + heading: 'Your FreeSewing account is inactive and marked for removal', + lead: ` + Due to inactivity, your account on FreeSewing.org has been marked for removal. +

+ If you want to keep your account, please login on FreeSewing.org within the next 2 weeks.`, + button: 'Go to FreeSewing.org', + actionUrl: 'https://freesewing.org/login', + closing: ` + If your account stays dormant, it will at one point become unavailable. +

+ That does not have to be a disaster, you can always create a new one, but you will loose all data stored in your account. +

+ This will be the only email I sent you about this. +
+ If you have questions, you can reply to this message.`, + + greeting: `love,`, + website: 'FreeSewing.org', + seeWhy: 'You received this email because removing inactive accounts is in line with our', + urlWhy: 'https://freesewing.org/docs/various/privacy/account/', + whyDidIGetThis: 'Privacy Notice', + notMarketing: 'This is not marketing, but a transactional email about your FreeSewing account.', + } +) + +send({ + to: ['nidhubhs@gmail.com'], + bcc: ['joost@joost.at', 'joost@decock.org'], + subject: '[FreeSewing] Your account is inactive and marked for removal', + html, + text, +}) diff --git a/sites/backend/src/controllers/imports.mjs b/sites/backend/src/controllers/imports.mjs index 958e2cdea4b..553c833f839 100644 --- a/sites/backend/src/controllers/imports.mjs +++ b/sites/backend/src/controllers/imports.mjs @@ -1,18 +1,44 @@ -import { ImportModel } from '../models/import.mjs' +import { SubscriberModel } from '../models/subscriber.mjs' +import { UserModel } from '../models/user.mjs' export function ImportsController() {} +/* + * This is a special route that uses hard-coded credentials + */ +const runChecks = (req) => { + if (req.body.import_token !== process.env.IMPORT_TOKEN) { + return [401, { result: 'error', error: 'accessDenied' }] + } + if (req.body.import_token !== process.env.IMPORT_TOKEN) { + return [400, { result: 'error', error: 'listMissing' }] + } + + return true +} + /* * Imports newsletters subscribers in v2 format */ ImportsController.prototype.subscribers = async (req, res, tools) => { - /* - * This is a special route that uses hard-coded credentials - */ - console.log(req.body) + const check = runChecks(req) + if (check !== true) return res.status(check[0]).send(check[1]) - const Import = new ImportModel(tools) - //await Flow.sendTranslatorInvite(req) + const Subscriber = new SubscriberModel(tools) + await Subscriber.import(req.body.list) - return Import.sendResponse(res) + return Subscriber.sendResponse(res) +} + +/* + * Imports users in v2 format + */ +ImportsController.prototype.users = async (req, res, tools) => { + const check = runChecks(req) + if (check !== true) return res.status(check[0]).send(check[1]) + + const User = new UserModel(tools) + await User.import(req.body.list) + + return User.sendResponse(res) } diff --git a/sites/backend/src/models/set.mjs b/sites/backend/src/models/set.mjs index 07479b198d6..77f1f30e133 100644 --- a/sites/backend/src/models/set.mjs +++ b/sites/backend/src/models/set.mjs @@ -1,5 +1,5 @@ import { log } from '../utils/log.mjs' -import { setSetAvatar } from '../utils/sanity.mjs' +import { setSetAvatar, downloadImage } from '../utils/sanity.mjs' import yaml from 'js-yaml' export function SetModel(tools) { @@ -43,7 +43,7 @@ SetModel.prototype.guardedCreate = async function ({ body, user }) { this.config.use.sanity && typeof body.img === 'string' && (!body.test || (body.test && this.config.use.tests?.sanity)) - ? await setSetAvatar(this.record.id, body.img) + ? await setSetAvatar(this.record.id, body.img, this.clear.name) : false if (img) await this.unguardedUpdate(this.cloak({ img: img.url })) @@ -264,7 +264,7 @@ SetModel.prototype.guardedUpdate = async function ({ params, body, user }) { // Image (img) if (typeof body.img === 'string') { - const img = await setSetAvatar(params.id, body.img) + const img = await setSetAvatar(params.id, body.img, data.name || this.clear.name) data.img = img.url } @@ -431,3 +431,42 @@ SetModel.prototype.sanitizeMeasurements = function (input) { return measies } + +const migratePerson = (v2) => ({ + createdAt: new Date(v2.created ? v2.created : v2.createdAt), + img: v2.picture, + imperial: v2.units === 'imperial', + name: v2.name || '--', // Encrypted, so always set _some_ value + notes: v2.notes || '--', // Encrypted, so always set _some_ value + measies: v2.measurements || {}, // Encrypted, so always set _some_ value + updatedAt: new Date(v2.updatedAt), +}) + +/* + * This is a special route not available for API users + */ +SetModel.prototype.import = async function (v2user, userId) { + for (const person of v2user.people) { + const data = { ...migratePerson(person), userId } + await this.unguardedCreate(data) + // Now that we have an ID, we can handle the image + if (data.img) { + const imgUrl = + 'https://static.freesewing.org/users/' + + encodeURIComponent(v2user.handle.slice(0, 1)) + + '/' + + encodeURIComponent(v2user.handle) + + '/people/' + + encodeURIComponent(person.handle) + + '/' + + encodeURIComponent(data.img) + console.log('Grabbing', imgUrl) + const [contentType, imgData] = await downloadImage(imgUrl) + // Do not import the default SVG avatar + if (contentType !== 'image/svg+xml') { + const img = await setSetAvatar(this.record.id, [contentType, imgData], data.name) + data.img = img + } + } + } +} diff --git a/sites/backend/src/models/subscriber.mjs b/sites/backend/src/models/subscriber.mjs index 1600c694596..4add0f7d193 100644 --- a/sites/backend/src/models/subscriber.mjs +++ b/sites/backend/src/models/subscriber.mjs @@ -295,3 +295,36 @@ SubscriberModel.prototype.isTest = function (body) { return true } + +/* + * This is a special route not available for API users + */ +SubscriberModel.prototype.import = async function (list) { + let created = 0 + for (const sub of list) { + const email = clean(sub) + const ehash = hash(email) + await this.read({ ehash }) + + if (!this.record) { + const data = await this.cloak({ + ehash, + email, + language: 'en', + active: true, + }) + try { + this.record = await this.prisma.subscriber.create({ data }) + created++ + } catch (err) { + log.warn(err, 'Could not create subscriber record') + return this.setResponse(500, 'createSubscriberFailed') + } + } + } + + return this.setResponse(200, 'success', { + total: list.length, + imported: created, + }) +} diff --git a/sites/backend/src/models/user.mjs b/sites/backend/src/models/user.mjs index 3bc8f969c2b..bf0f02ddfac 100644 --- a/sites/backend/src/models/user.mjs +++ b/sites/backend/src/models/user.mjs @@ -1,9 +1,10 @@ import jwt from 'jsonwebtoken' import { log } from '../utils/log.mjs' import { hash, hashPassword, randomString, verifyPassword } from '../utils/crypto.mjs' -import { setUserAvatar } from '../utils/sanity.mjs' +import { setUserAvatar, downloadImage } from '../utils/sanity.mjs' import { clean, asJson, i18nUrl } from '../utils/index.mjs' import { ConfirmationModel } from './confirmation.mjs' +import { SetModel } from './set.mjs' export function UserModel(tools) { this.config = tools.config @@ -16,6 +17,8 @@ export function UserModel(tools) { this.Confirmation = new ConfirmationModel(tools) this.encryptedFields = ['bio', 'github', 'email', 'initial', 'img', 'mfaSecret'] this.clear = {} // For holding decrypted data + // Only used for import, can be removed after v3 is released + this.Set = new SetModel(tools) return this } @@ -542,7 +545,7 @@ UserModel.prototype.guardedUpdate = async function ({ body, user }) { } // Image (img) if (typeof body.img === 'string') { - const img = await setUserAvatar(this.record.id, body.img) + const img = await setUserAvatar(this.record.ihash, body.img, data.username) data.img = img.url } @@ -840,3 +843,119 @@ UserModel.prototype.isLusernameAvailable = async function (lusername) { return true } + +const migrateUser = (v2) => { + const email = clean(v2.email) + const initial = clean(v2.initial) + const data = { + bio: v2.bio, + consent: 0, + createdAt: v2.time?.created ? new Date(v2.time.created) : new Date(), + email, + ehash: hash(email), + github: v2.social?.github, + ihash: hash(initial), + img: + v2.picture.slice(-4).toLowerCase() === '.svg' // Don't bother with default avatars + ? '' + : v2.picture, + initial, + imperial: v2.units === 'imperial', + language: v2.settings.language, + lastSignIn: v2.time.login, + lusername: v2.username.toLowerCase(), + mfaEnabled: false, + newsletter: false, + password: JSON.stringify({ + type: 'v2', + data: v2.password, + }), + patron: v2.patron, + role: v2._id === '5d62aa44ce141a3b816a3dd9' ? 'admin' : 'user', + status: v2.status === 'active' ? 1 : 0, + username: v2.username, + } + if (data.consent.profile) data.consent++ + if (data.consent.measurements) data.consent++ + if (data.consent.openData) data.consent++ + + return data +} + +const lastLoginInDays = (user) => { + const now = new Date() + if (!user.time) console.log(user) + const then = new Date(user.time.login) + + const delta = Math.floor((now - then) / (1000 * 60 * 60 * 24)) + + return delta +} + +/* + * This is a special route not available for API users + */ +UserModel.prototype.import = async function (list) { + let created = 0 + const skipped = [] + for (const sub of list) { + if (sub.status === 'active') { + const days = lastLoginInDays(sub) + if (days < 370) { + const data = migrateUser(sub) + await this.read({ ehash: data.ehash }) + if (!this.record) { + /* + * Grab the image from the FreeSewing server and upload it to Sanity + */ + if (data.img) { + const imgUrl = + 'https://static.freesewing.org/users/' + + encodeURIComponent(sub.handle.slice(0, 1)) + + '/' + + encodeURIComponent(sub.handle) + + '/' + + encodeURIComponent(data.img) + console.log('Grabbing', imgUrl) + const [contentType, imgData] = await downloadImage(imgUrl) + // Do not import the default SVG avatar + if (contentType !== 'image/svg+xml') { + const img = await setUserAvatar(data.ihash, [contentType, imgData], data.username) + data.img = img + } + } + let cloaked = await this.cloak(data) + try { + this.record = await this.prisma.user.create({ data: cloaked }) + created++ + } catch (err) { + if ( + err.toString().indexOf('Unique constraint failed on the fields: (`lusername`)') !== -1 + ) { + // Just add a '+' to the username + data.username += '+' + data.lusername += '+' + cloaked = await this.cloak(data) + try { + this.record = await this.prisma.user.create({ data: cloaked }) + created++ + } catch (err) { + log.warn(err, 'Could not create user record') + console.log(sub) + return this.setResponse(500, 'createUserFailed') + } + } + } + } + } else skipped.push(sub.email) + // That's the user, not load their people as sets + if (sub.people) await this.Set.import(sub, this.record.id) + } else skipped.push(sub.email) + } + + return this.setResponse(200, 'success', { + skipped, + total: list.length, + imported: created, + }) +} diff --git a/sites/backend/src/routes/import.mjs b/sites/backend/src/routes/imports.mjs similarity index 63% rename from sites/backend/src/routes/import.mjs rename to sites/backend/src/routes/imports.mjs index ea8a23d9980..1103c4c9c11 100644 --- a/sites/backend/src/routes/import.mjs +++ b/sites/backend/src/routes/imports.mjs @@ -1,6 +1,6 @@ -import { ImportController } from '../controllers/import.mjs' +import { ImportsController } from '../controllers/imports.mjs' -const Import = new ImportController() +const Import = new ImportsController() export function importsRoutes(tools) { const { app } = tools @@ -9,6 +9,10 @@ export function importsRoutes(tools) { * All these routes use hard-coded credentials because they should never be used * outside the v2-v3 migration which is handled by joost */ + // Import newsletter subscriptions app.post('/import/subscribers', (req, res) => Import.subscribers(req, res, tools)) + + // Import users + app.post('/import/users', (req, res) => Import.users(req, res, tools)) } diff --git a/sites/backend/src/routes/index.mjs b/sites/backend/src/routes/index.mjs index 83d6ebe8df9..e5032a2cd6e 100644 --- a/sites/backend/src/routes/index.mjs +++ b/sites/backend/src/routes/index.mjs @@ -7,6 +7,7 @@ import { curatedSetsRoutes } from './curated-sets.mjs' import { issuesRoutes } from './issues.mjs' import { subscribersRoutes } from './subscribers.mjs' import { flowsRoutes } from './flows.mjs' +import { importsRoutes } from './imports.mjs' export const routes = { apikeysRoutes, @@ -18,4 +19,5 @@ export const routes = { issuesRoutes, subscribersRoutes, flowsRoutes, + importsRoutes, } diff --git a/sites/backend/src/templates/email/shared/blocks.mjs b/sites/backend/src/templates/email/shared/blocks.mjs index 68ade9ac4f1..13649cf5504 100644 --- a/sites/backend/src/templates/email/shared/blocks.mjs +++ b/sites/backend/src/templates/email/shared/blocks.mjs @@ -19,16 +19,16 @@ export const closingRow = {

- {{ closing }} + {{{ closing }}}

- {{ greeting }} + {{{ greeting }}}
joost

- PS: {{ ps-pre-link}} + PS: {{{ ps-pre-link}}} - {{ ps-link}} - {{ ps-post-link }} + {{{ ps-link}}} + {{{ ps-post-link }}}

`, @@ -46,9 +46,9 @@ export const newsletterClosingRow = {

- {{ closing }} + {{{ closing }}}

- {{ greeting }} + {{{ greeting }}}
joost

@@ -67,8 +67,8 @@ export const headingRow = {

- - {{ heading }} + + {{{ heading }}}

@@ -84,7 +84,7 @@ export const lead1Row = {

- {{ lead }} + {{{ lead }}}

@@ -99,10 +99,10 @@ export const preLeadRow = {

- {{ preLead }} + {{{ preLead }}}

- {{ lead }} + {{{ lead }}}

@@ -182,7 +182,7 @@ export const wrap = { {{ intro }} ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ ͏ -
+