wip(backend): Working on migration of user data

scripts/email-lib.mjs

@@ -0,0 +1,56 @@
+import path from 'path'
+import { fileURLToPath } from 'url'
+import { SESv2Client, SendEmailCommand } from '@aws-sdk/client-sesv2'
+
+// Current working directory
+const cwd = path.dirname(fileURLToPath(import.meta.url))
+
+export const send = async ({
+  to = ['joost@joost.at'],
+  bcc = [],
+  subject = false,
+  html = false,
+  text = false,
+}) => {
+  if (!subject || !html || !text) return console.log('No subject, html, or text provided')
+
+  const us = 'Joost from FreeSewing <info@freesewing.org>'
+
+  // Oh AWS your APIs are such a clusterfuck
+  const client = new SESv2Client({ region: 'us-east-1' })
+
+  // Via API
+  const command = new SendEmailCommand({
+    ConfigurationSetName: 'backend',
+    Content: {
+      Simple: {
+        Body: {
+          Text: {
+            Charset: 'utf-8',
+            Data: text,
+          },
+          Html: {
+            Charset: 'utf-8',
+            Data: html,
+          },
+        },
+        Subject: {
+          Charset: 'utf-8',
+          Data: subject,
+        },
+      },
+    },
+    Destination: {
+      ToAddresses: to,
+      BccAddresses: to,
+    },
+    FromEmailAddress: us,
+  })
+  try {
+    const result = await client.send(command)
+    console.log(result)
+  } catch (err) {
+    console.log(err)
+    return false
+  }
+}

scripts/send-real-email.mjs

@@ -0,0 +1,96 @@
+import 'dotenv/config'
+import mustache from 'mustache'
+import { send } from './email-lib.mjs'
+import {
+  buttonRow,
+  newsletterClosingRow,
+  headingRow,
+  lead1Row,
+  wrap,
+} from '../sites/backend/src/templates/email/shared/blocks.mjs'
+import users from '../sites/backend/dump/inactive.json' assert { type: 'json' }
+
+const splitArray = (split, batchSize) =>
+  split.reduce((result, item, index) => {
+    const batchIndex = Math.floor(index / batchSize)
+    if (!result[batchIndex]) result[batchIndex] = []
+    result[batchIndex].push(item)
+
+    return result
+  }, [])
+
+const text = `
+Dear FreeSewing user,
+
+Your account on FreeSewing.org has been inactive an scheduled for removal.
+
+If you want to keep your account, please login on FreeSewing.org within the next 2 weeks.
+
+If your account stays dormant, it will at one point become unavailable.
+That does not have to be a disaster, you can always create a new one, but you will loose all data stored in your account.
+
+This will be the only email I sent you about this.
+If you have questions, you can reply to this message.
+
+love,
+joost
+`
+const html = mustache.render(
+  wrap.html(`
+    ${headingRow.html}
+    ${lead1Row.html}
+    ${buttonRow.html}
+    ${newsletterClosingRow.html}
+  `),
+  {
+    intro: 'Your account is inactive. Login now to revive it.',
+    heading: 'Your FreeSewing account is inactive and marked for removal',
+    lead: `
+      Due to inactivity, your account on FreeSewing.org has been marked for removal.
+      <br><br>
+      If you want to keep your account, please login on FreeSewing.org within the next 2 weeks.`,
+    button: 'Go to FreeSewing.org',
+    actionUrl: 'https://freesewing.org/login',
+    closing: `
+      If your account stays dormant, it will at one point become unavailable.
+      <br><br>
+      That does not have to be a disaster, you can always create a new one, but you will loose all data stored in your account.
+      <br><br>
+      This will be the only email I sent you about this.
+      <br>
+      If you have questions, you can reply to this message.`,
+
+    greeting: `love,`,
+    website: 'FreeSewing.org',
+    seeWhy: 'You received this email because removing inactive accounts is in line with our',
+    urlWhy: 'https://freesewing.org/docs/various/privacy/account/',
+    whyDidIGetThis: 'Privacy Notice',
+    notMarketing: 'This is not marketing, but a transactional email about your FreeSewing account.',
+  }
+)
+
+/*
+ * AWS limit is 50 recipients.
+ * So rather than send a bunch of individual emails,
+ * let's send it to ourselves * and put 45 people in BCC
+ */
+const batches = splitArray(users, 45)
+
+let i = 0
+const total = batches.length
+batches.forEach((batch, i) => {
+  /*
+   * Maximum email rate is 14 messages per second
+   * So we can't go too fast
+   */
+  setTimeout(() => {
+    console.log(`Sending ${i}/${total}`)
+    send({
+      to: ['info@freesewing.org'],
+      bcc: batch,
+      subject: '[FreeSewing] Your account is inactive and marked for removal',
+      html,
+      text,
+    })
+  }, i * 100)
+})

scripts/send-test-email.mjs

@@ -0,0 +1,68 @@
+import 'dotenv/config'
+import mustache from 'mustache'
+import { send } from './email-lib.mjs'
+import {
+  buttonRow,
+  newsletterClosingRow,
+  headingRow,
+  lead1Row,
+  wrap,
+} from '../sites/backend/src/templates/email/shared/blocks.mjs'
+
+const text = `
+Dear FreeSewing user,
+
+Your account on FreeSewing.org has been inactive an scheduled for removal.
+
+If you want to keep your account, please login on FreeSewing.org within the next 2 weeks.
+
+If your account stays dormant, it will at one point become unavailable.
+That does not have to be a disaster, you can always create a new one, but you will loose all data stored in your account.
+
+This will be the only email I sent you about this.
+If you have questions, you can reply to this message.
+
+love,
+joost
+`
+const html = mustache.render(
+  wrap.html(`
+    ${headingRow.html}
+    ${lead1Row.html}
+    ${buttonRow.html}
+    ${newsletterClosingRow.html}
+  `),
+  {
+    intro: 'Your account is inactive. Login now to revive it.',
+    heading: 'Your FreeSewing account is inactive and marked for removal',
+    lead: `
+      Due to inactivity, your account on FreeSewing.org has been marked for removal.
+      <br><br>
+      If you want to keep your account, please login on FreeSewing.org within the next 2 weeks.`,
+    button: 'Go to FreeSewing.org',
+    actionUrl: 'https://freesewing.org/login',
+    closing: `
+      If your account stays dormant, it will at one point become unavailable.
+      <br><br>
+      That does not have to be a disaster, you can always create a new one, but you will loose all data stored in your account.
+      <br><br>
+      This will be the only email I sent you about this.
+      <br>
+      If you have questions, you can reply to this message.`,
+
+    greeting: `love,`,
+    website: 'FreeSewing.org',
+    seeWhy: 'You received this email because removing inactive accounts is in line with our',
+    urlWhy: 'https://freesewing.org/docs/various/privacy/account/',
+    whyDidIGetThis: 'Privacy Notice',
+    notMarketing: 'This is not marketing, but a transactional email about your FreeSewing account.',
+  }
+)
+
+send({
+  to: ['nidhubhs@gmail.com'],
+  bcc: ['joost@joost.at', 'joost@decock.org'],
+  subject: '[FreeSewing] Your account is inactive and marked for removal',
+  html,
+  text,
+})

sites/backend/src/controllers/imports.mjs

@@ -1,18 +1,44 @@
-import { ImportModel } from '../models/import.mjs'
+import { SubscriberModel } from '../models/subscriber.mjs'
+import { UserModel } from '../models/user.mjs'
 
 export function ImportsController() {}
 
+/*
+ * This is a special route that uses hard-coded credentials
+ */
+const runChecks = (req) => {
+  if (req.body.import_token !== process.env.IMPORT_TOKEN) {
+    return [401, { result: 'error', error: 'accessDenied' }]
+  }
+  if (req.body.import_token !== process.env.IMPORT_TOKEN) {
+    return [400, { result: 'error', error: 'listMissing' }]
+  }
+
+  return true
+}
+
 /*
  * Imports newsletters subscribers in v2 format
  */
 ImportsController.prototype.subscribers = async (req, res, tools) => {
-  /*
+  const check = runChecks(req)
-   * This is a special route that uses hard-coded credentials
+  if (check !== true) return res.status(check[0]).send(check[1])
-   */
-  console.log(req.body)
 
-  const Import = new ImportModel(tools)
+  const Subscriber = new SubscriberModel(tools)
-  //await Flow.sendTranslatorInvite(req)
+  await Subscriber.import(req.body.list)
 
-  return Import.sendResponse(res)
+  return Subscriber.sendResponse(res)
+}
+
+/*
+ * Imports users in v2 format
+ */
+ImportsController.prototype.users = async (req, res, tools) => {
+  const check = runChecks(req)
+  if (check !== true) return res.status(check[0]).send(check[1])
+
+  const User = new UserModel(tools)
+  await User.import(req.body.list)
+
+  return User.sendResponse(res)
 }

sites/backend/src/models/set.mjs

@@ -1,5 +1,5 @@
 import { log } from '../utils/log.mjs'
-import { setSetAvatar } from '../utils/sanity.mjs'
+import { setSetAvatar, downloadImage } from '../utils/sanity.mjs'
 import yaml from 'js-yaml'
 
 export function SetModel(tools) {
@@ -43,7 +43,7 @@ SetModel.prototype.guardedCreate = async function ({ body, user }) {
     this.config.use.sanity &&
     typeof body.img === 'string' &&
     (!body.test || (body.test && this.config.use.tests?.sanity))
-      ? await setSetAvatar(this.record.id, body.img)
+      ? await setSetAvatar(this.record.id, body.img, this.clear.name)
       : false
 
   if (img) await this.unguardedUpdate(this.cloak({ img: img.url }))
@@ -264,7 +264,7 @@ SetModel.prototype.guardedUpdate = async function ({ params, body, user }) {
 
   // Image (img)
   if (typeof body.img === 'string') {
-    const img = await setSetAvatar(params.id, body.img)
+    const img = await setSetAvatar(params.id, body.img, data.name || this.clear.name)
     data.img = img.url
   }
 
@@ -431,3 +431,42 @@ SetModel.prototype.sanitizeMeasurements = function (input) {
 
   return measies
 }
+
+const migratePerson = (v2) => ({
+  createdAt: new Date(v2.created ? v2.created : v2.createdAt),
+  img: v2.picture,
+  imperial: v2.units === 'imperial',
+  name: v2.name || '--', // Encrypted, so always set _some_ value
+  notes: v2.notes || '--', // Encrypted, so always set _some_ value
+  measies: v2.measurements || {}, // Encrypted, so always set _some_ value
+  updatedAt: new Date(v2.updatedAt),
+})
+
+/*
+ * This is a special route not available for API users
+ */
+SetModel.prototype.import = async function (v2user, userId) {
+  for (const person of v2user.people) {
+    const data = { ...migratePerson(person), userId }
+    await this.unguardedCreate(data)
+    // Now that we have an ID, we can handle the image
+    if (data.img) {
+      const imgUrl =
+        'https://static.freesewing.org/users/' +
+        encodeURIComponent(v2user.handle.slice(0, 1)) +
+        '/' +
+        encodeURIComponent(v2user.handle) +
+        '/people/' +
+        encodeURIComponent(person.handle) +
+        '/' +
+        encodeURIComponent(data.img)
+      console.log('Grabbing', imgUrl)
+      const [contentType, imgData] = await downloadImage(imgUrl)
+      // Do not import the default SVG avatar
+      if (contentType !== 'image/svg+xml') {
+        const img = await setSetAvatar(this.record.id, [contentType, imgData], data.name)
+        data.img = img
+      }
+    }
+  }
+}

sites/backend/src/models/subscriber.mjs

@@ -295,3 +295,36 @@ SubscriberModel.prototype.isTest = function (body) {
 
   return true
 }
+
+/*
+ * This is a special route not available for API users
+ */
+SubscriberModel.prototype.import = async function (list) {
+  let created = 0
+  for (const sub of list) {
+    const email = clean(sub)
+    const ehash = hash(email)
+    await this.read({ ehash })
+
+    if (!this.record) {
+      const data = await this.cloak({
+        ehash,
+        email,
+        language: 'en',
+        active: true,
+      })
+      try {
+        this.record = await this.prisma.subscriber.create({ data })
+        created++
+      } catch (err) {
+        log.warn(err, 'Could not create subscriber record')
+        return this.setResponse(500, 'createSubscriberFailed')
+      }
+    }
+  }
+
+  return this.setResponse(200, 'success', {
+    total: list.length,
+    imported: created,
+  })
+}

sites/backend/src/models/user.mjs

@@ -1,9 +1,10 @@
 import jwt from 'jsonwebtoken'
 import { log } from '../utils/log.mjs'
 import { hash, hashPassword, randomString, verifyPassword } from '../utils/crypto.mjs'
-import { setUserAvatar } from '../utils/sanity.mjs'
+import { setUserAvatar, downloadImage } from '../utils/sanity.mjs'
 import { clean, asJson, i18nUrl } from '../utils/index.mjs'
 import { ConfirmationModel } from './confirmation.mjs'
+import { SetModel } from './set.mjs'
 
 export function UserModel(tools) {
   this.config = tools.config
@@ -16,6 +17,8 @@ export function UserModel(tools) {
   this.Confirmation = new ConfirmationModel(tools)
   this.encryptedFields = ['bio', 'github', 'email', 'initial', 'img', 'mfaSecret']
   this.clear = {} // For holding decrypted data
+  // Only used for import, can be removed after v3 is released
+  this.Set = new SetModel(tools)
 
   return this
 }
@@ -542,7 +545,7 @@ UserModel.prototype.guardedUpdate = async function ({ body, user }) {
   }
   // Image (img)
   if (typeof body.img === 'string') {
-    const img = await setUserAvatar(this.record.id, body.img)
+    const img = await setUserAvatar(this.record.ihash, body.img, data.username)
     data.img = img.url
   }
 
@@ -840,3 +843,119 @@ UserModel.prototype.isLusernameAvailable = async function (lusername) {
 
   return true
 }
+
+const migrateUser = (v2) => {
+  const email = clean(v2.email)
+  const initial = clean(v2.initial)
+  const data = {
+    bio: v2.bio,
+    consent: 0,
+    createdAt: v2.time?.created ? new Date(v2.time.created) : new Date(),
+    email,
+    ehash: hash(email),
+    github: v2.social?.github,
+    ihash: hash(initial),
+    img:
+      v2.picture.slice(-4).toLowerCase() === '.svg' // Don't bother with default avatars
+        ? ''
+        : v2.picture,
+    initial,
+    imperial: v2.units === 'imperial',
+    language: v2.settings.language,
+    lastSignIn: v2.time.login,
+    lusername: v2.username.toLowerCase(),
+    mfaEnabled: false,
+    newsletter: false,
+    password: JSON.stringify({
+      type: 'v2',
+      data: v2.password,
+    }),
+    patron: v2.patron,
+    role: v2._id === '5d62aa44ce141a3b816a3dd9' ? 'admin' : 'user',
+    status: v2.status === 'active' ? 1 : 0,
+    username: v2.username,
+  }
+  if (data.consent.profile) data.consent++
+  if (data.consent.measurements) data.consent++
+  if (data.consent.openData) data.consent++
+
+  return data
+}
+
+const lastLoginInDays = (user) => {
+  const now = new Date()
+  if (!user.time) console.log(user)
+  const then = new Date(user.time.login)
+
+  const delta = Math.floor((now - then) / (1000 * 60 * 60 * 24))
+
+  return delta
+}
+
+/*
+ * This is a special route not available for API users
+ */
+UserModel.prototype.import = async function (list) {
+  let created = 0
+  const skipped = []
+  for (const sub of list) {
+    if (sub.status === 'active') {
+      const days = lastLoginInDays(sub)
+      if (days < 370) {
+        const data = migrateUser(sub)
+        await this.read({ ehash: data.ehash })
+        if (!this.record) {
+          /*
+           * Grab the image from the FreeSewing server and upload it to Sanity
+           */
+          if (data.img) {
+            const imgUrl =
+              'https://static.freesewing.org/users/' +
+              encodeURIComponent(sub.handle.slice(0, 1)) +
+              '/' +
+              encodeURIComponent(sub.handle) +
+              '/' +
+              encodeURIComponent(data.img)
+            console.log('Grabbing', imgUrl)
+            const [contentType, imgData] = await downloadImage(imgUrl)
+            // Do not import the default SVG avatar
+            if (contentType !== 'image/svg+xml') {
+              const img = await setUserAvatar(data.ihash, [contentType, imgData], data.username)
+              data.img = img
+            }
+          }
+          let cloaked = await this.cloak(data)
+          try {
+            this.record = await this.prisma.user.create({ data: cloaked })
+            created++
+          } catch (err) {
+            if (
+              err.toString().indexOf('Unique constraint failed on the fields: (`lusername`)') !== -1
+            ) {
+              // Just add a '+' to the username
+              data.username += '+'
+              data.lusername += '+'
+              cloaked = await this.cloak(data)
+              try {
+                this.record = await this.prisma.user.create({ data: cloaked })
+                created++
+              } catch (err) {
+                log.warn(err, 'Could not create user record')
+                console.log(sub)
+                return this.setResponse(500, 'createUserFailed')
+              }
+            }
+          }
+        }
+      } else skipped.push(sub.email)
+      // That's the user, not load their people as sets
+      if (sub.people) await this.Set.import(sub, this.record.id)
+    } else skipped.push(sub.email)
+  }
+
+  return this.setResponse(200, 'success', {
+    skipped,
+    total: list.length,
+    imported: created,
+  })
+}

sites/backend/src/routes/imports.mjs

@@ -1,6 +1,6 @@
-import { ImportController } from '../controllers/import.mjs'
+import { ImportsController } from '../controllers/imports.mjs'
 
-const Import = new ImportController()
+const Import = new ImportsController()
 
 export function importsRoutes(tools) {
   const { app } = tools
@@ -9,6 +9,10 @@ export function importsRoutes(tools) {
    * All these routes use hard-coded credentials because they should never be used
    * outside the v2-v3 migration which is handled by joost
    */
+
   // Import newsletter subscriptions
   app.post('/import/subscribers', (req, res) => Import.subscribers(req, res, tools))
+
+  // Import users
+  app.post('/import/users', (req, res) => Import.users(req, res, tools))
 }

sites/backend/src/routes/index.mjs

@@ -7,6 +7,7 @@ import { curatedSetsRoutes } from './curated-sets.mjs'
 import { issuesRoutes } from './issues.mjs'
 import { subscribersRoutes } from './subscribers.mjs'
 import { flowsRoutes } from './flows.mjs'
+import { importsRoutes } from './imports.mjs'
 
 export const routes = {
   apikeysRoutes,
@@ -18,4 +19,5 @@ export const routes = {
   issuesRoutes,
   subscribersRoutes,
   flowsRoutes,
+  importsRoutes,
 }

sites/backend/src/templates/email/shared/blocks.mjs

@@ -19,16 +19,16 @@ export const closingRow = {
   <tr>
     <td align="left" class="sm-p-15px" style="padding-top: 30px">
       <p style="margin: 0; font-size: 16px; line-height: 25px; color: #262626">
-        {{ closing }}
+        {{{ closing }}}
         <br><br>
-        {{ greeting }}
+        {{{ greeting }}}
         <br>
         joost
         <br><br>
-        PS: {{ ps-pre-link}}
+        PS: {{{ ps-pre-link}}}
         <a href="{{ supportUrl }}" target="_blank" style="text-decoration: underline; color: #262626">
-          <b>{{ ps-link}}</b>
+          <b>{{{ ps-link}}}</b>
-        </a> {{ ps-post-link }}
+        </a> {{{ ps-post-link }}}
       </p>
     </td>
   </tr>`,
@@ -46,9 +46,9 @@ export const newsletterClosingRow = {
   <tr>
     <td align="left" class="sm-p-15px" style="padding-top: 30px">
       <p style="margin: 0; font-size: 16px; line-height: 25px; color: #262626">
-        {{ closing }}
+        {{{ closing }}}
         <br><br>
-        {{ greeting }}
+        {{{ greeting }}}
         <br>
         joost
       </p>
@@ -67,8 +67,8 @@ export const headingRow = {
   <tr>
     <td align="left" class="sm-p-15px" style="padding-top: 30px">
       <h2 style="margin: 0; font-size: 30px; color: #525252">
-        <a href="{{ actionUrl }}" target="_blank" style="text-decoration: none; color: #525252">
+        <a href="{{{ actionUrl }}}" target="_blank" style="text-decoration: none; color: #525252">
-          {{ heading }}
+          {{{ heading }}}
         </a>
       </h2>
     </td>
@@ -84,7 +84,7 @@ export const lead1Row = {
     <td align="left" class="sm-p-15px" style="padding-top: 15px">
       <p style="margin: 0; font-size: 16px; line-height: 25px; color: #262626">
         <a href="{{ actionUrl }}" target="_blank" style="text-decoration: none; color: #262626">
-          <b>{{ lead }}</b>
+          <b>{{{ lead }}}</b>
         </a>
       </p>
     </td>
@@ -99,10 +99,10 @@ export const preLeadRow = {
   <tr>
     <td align="left" class="sm-p-15px" style="padding-top: 15px">
       <p style="margin: 0; font-size: 16px; line-height: 25px; color: #262626">
-        {{ preLead }}
+        {{{ preLead }}}
         <br><br>
         <a href="{{ actionUrl }}" target="_blank" style="text-decoration: none; color: #262626">
-          <b>{{ lead }}</b>
+          <b>{{{ lead }}}</b>
         </a>
       </p>
     </td>
@@ -182,7 +182,7 @@ export const wrap = {
     {{ intro }}
     &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847; &#847;
   </div>
-  <div role="article" aria-roledescription="email" aria-label="Please confirm your new email address" lang="en">
+  <div role="article" aria-roledescription="email" aria-label="{{ intro }}" lang="en">
     <table class="all-font-sans" style="width: 100%" border="0" cellpadding="0" cellspacing="0" role="presentation">
       <tr>
         <td align="center" class="sm-py-10px" style="background-color: #fff; padding: 10px 15px">
@@ -226,9 +226,9 @@ export const wrap = {
       <p style="margin: 0; font-size: 14px; line-height: 24px; color: #868e96; text-align: center;">
         <a href="https://{{ website }}" target="_blank" style="text-decoration: underline; color: #868e96"><b>{{ website }}</b></a>
         <span style="font-size: 13px; color: #737373">&nbsp;&nbsp;|&nbsp;&nbsp;</span>
-        <a href="https://github.com/fresewing/freesewing" target="_blank" style="text-decoration: underline; color: #868e96"><b>Github</b></a>
+        <a href="https://freesewing.dev" target="_blank" style="text-decoration: underline; color: #868e96"><b>FreeSewing.dev</b></a>
         <span style="font-size: 13px; color: #737373">&nbsp;&nbsp;|&nbsp;&nbsp;</span>
-        <a href="https://discord.freesewing.org/" target="_blank" style="text-decoration: underline; color: #868e96"><b>Discord</b></a>
+        <a href="https://github.com/fresewing/freesewing" target="_blank" style="text-decoration: underline; color: #868e96"><b>GitHub.com/freesewing</b></a>
       </p>
       <p style="margin: 0; font-size: 12px; margin-top: 12px; line-height: 18px; color: #868e96; text-align: center;">
         {{ notMarketing }}
@@ -270,8 +270,6 @@ Belgium
 
 {{ website }} : {{{ urlWebsite }}}
 Github : https://github.com/freesewing/freesewing
-Discord : https://discord.freesewing.org/
-Twitter : https://twitter.com/freesewing_org
 {{ whyDidIGetThis }} : {{{ whyUrl }}}
 `,
 }

sites/backend/src/utils/crypto.mjs

@@ -79,6 +79,7 @@ export function encryption(stringKey, salt = 'FreeSewing') {
       })
     },
     decrypt: (data) => {
+      if (data === null || data === '') return ''
       /*
        * Don't blindly assume this data is properly formatted ciphertext
        */

sites/backend/src/utils/sanity.mjs

@@ -1,5 +1,14 @@
 import axios from 'axios'
 import { sanity as config } from '../config.mjs'
+import { createClient } from '@sanity/client'
+
+const sanity = createClient({
+  projectId: config.project,
+  dataset: config.dataset,
+  useCdn: true, // Set to false to bypass cache
+  apiVersion: config.version,
+  token: config.token,
+})
 
 // We'll use this a bunch
 const headers = { Authorization: `Bearer ${config.token}` }
@@ -31,55 +40,32 @@ async function getAvatar(type, id) {
 /*
  * Uploads an image to sanity
  */
-export const setUserAvatar = async (id, data) => setAvatar('user', id, data)
+export const setUserAvatar = async (id, data, title) => setAvatar('user', id, data, title)
-export const setSetAvatar = async (id, data) => setAvatar('set', id, data)
+export const setSetAvatar = async (id, data, title) => setAvatar('set', id, data, title)
-export const setPatternAvatar = async (id, data) => setAvatar('pattern', id, data)
+export const setPatternAvatar = async (id, data, title) => setAvatar('pattern', id, data, title)
-export async function setAvatar(type, id, data) {
+export async function setAvatar(type, id, data, title) {
   // Step 1: Upload the image as asset
-  const [contentType, binary] = b64ToBinaryWithType(data)
+  const [contentType, binary] = Array.isArray(data) ? data : b64ToBinaryWithType(data)
-  if (contentType) {
+  if (!contentType) return ''
-    const img = await axios.post(`${config.api}/assets/images/${config.dataset}`, binary, {
-      headers: {
-        ...headers,
-        'Content-Type': 'contentType',
-      },
-    })
-    if (!img.data?.document?._id) return false // Upload failed
 
-    // Step 2, update the document
+  const imgDocument = await sanity.assets.upload('image', binary, {
-    await axios.post(
+    contentType,
-      `${config.api}/data/mutate/${config.dataset}`,
+    filename: `${type}.${contentType.split('/').pop()}`,
-      {
+  })
-        mutations: [
+  const document = await sanity.createOrReplace({
-          {
+    _id: `${type}-${id}`,
-            createOrReplace: {
+    _type: `${type}img`,
-              _id: `${type}-${id}`,
+    title: title,
-              _type: `${type}img`,
+    recordid: id,
-              recordid: id,
+    img: {
-            },
+      asset: {
-          },
+        _ref: imgDocument._id,
-          {
+        _type: 'reference',
-            patch: {
-              id: `${type}-${id}`,
-              set: {
-                'img.asset': {
-                  _ref: img.data.document.id,
-                  _type: 'reference',
-                },
-              },
-            },
-          },
-        ],
       },
-      { headers }
+    },
-    )
+  })
 
-    return {
+  return document._id
-      id: img.data.document._id,
-      url: img.data.document.url,
-    }
-  }
-  return false
 }
 
 /*
@@ -94,3 +80,19 @@ function b64ToBinaryWithType(dataUri) {
 
   return [type, new Buffer.from(data, 'base64')]
 }
+
+/*
+ * Helper method to download an image
+ * Used in import only, thus ok for removal post v3 release
+ */
+export const downloadImage = async (url) => {
+  let result
+  try {
+    result = await axios.get(url, { responseType: 'arraybuffer' })
+  } catch (err) {
+    console.log(`Could not download from ${url}`, err, result)
+  }
+
+  // Returning [contentType, data]
+  return [result.headers['content-type'], Buffer.from(result.data, 'binary')]
+}

sites/backend/v2-v3/import.mjs

@@ -0,0 +1,100 @@
+import subscribers from './v2-newsletters.json' assert { type: 'json' }
+import users from './v2-users.json' assert { type: 'json' }
+import people from './v2-people.json' assert { type: 'json' }
+
+/*
+ * Only this token allows exporting data
+ */
+const import_token = 'TOKEN_HERE'
+
+/*
+ * Where to connect to?
+ */
+const BACKEND = 'http://localhost:3000'
+
+const splitArray = (split, batchSize) =>
+  split.reduce((result, item, index) => {
+    const batchIndex = Math.floor(index / batchSize)
+    if (!result[batchIndex]) result[batchIndex] = []
+    result[batchIndex].push(item)
+
+    return result
+  }, [])
+
+const importSubscribers = async () => {
+  console.log('Importing subscribers')
+  const count = subscribers.length
+  let total = 0
+  const batches = splitArray(subscribers, 50)
+  for (const batch of batches) {
+    const result = await fetch(`${BACKEND}/import/subscribers`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        import_token: import_token,
+        list: batch,
+      }),
+    })
+    const data = await result.json()
+    total += data.imported
+    console.log(`${total}/${count}`)
+  }
+}
+
+const lastLoginInDays = (user) => {
+  const now = new Date()
+  const then = new Date(user.time.login)
+
+  return Math.floor((now - then) / (1000 * 60 * 60 * 24))
+}
+
+const usersToImport = () =>
+  users.filter((user) => user.status === 'active' && lastLoginInDays(user) < 370)
+const usersToNotImport = () =>
+  users.filter((user) => user.status !== 'active' && lastLoginInDays(user) >= 370)
+
+const importUsers = async () => {
+  console.log('Processing users')
+  const todo = usersToImport()
+  // Put users in an object with their handle as key
+  const allUsers = {}
+  for (const user of todo) allUsers[user.handle] = user
+  // Find all people belonging to these users
+  for (const person of people) {
+    if (typeof allUsers[person.user] !== 'undefined') {
+      if (typeof allUsers[person.user].people === 'undefined') allUsers[person.user].people = []
+      allUsers[person.user].people.push(person)
+    }
+  }
+  console.log('Importing users')
+  console.log(JSON.stringify(allUsers.joost, null, 2))
+  process.exit()
+  const count = todo.length
+  let total = 0
+  const batches = splitArray(todo, 50)
+  for (const batch of batches) {
+    const result = await fetch(`${BACKEND}/import/users`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        import_token: import_token,
+        list: batch,
+      }),
+    })
+    const data = await result.json()
+    total += data.imported
+    console.log(`${total}/${count} (${data.skipped} skipped)`)
+    console.log(data)
+  }
+}
+
+const importAll = async () => {
+  //await importSubscribers()
+  await importUsers()
+}
+
+importAll()

sites/backend/v2-v3/inactive.mjs

@@ -0,0 +1,22 @@
+import users from './v2-users.json' assert { type: 'json' }
+
+const lastLoginInDays = (user) => {
+  if (!user.time?.login) return 1000
+  const now = new Date()
+  const then = new Date(user.time.login)
+
+  return Math.floor((now - then) / (1000 * 60 * 60 * 24))
+}
+
+const usersToNotImport = () =>
+  users.filter((user) => user.status !== 'active' || lastLoginInDays(user) >= 370)
+const usersToImport = () =>
+  users.filter((user) => user.status === 'active' && lastLoginInDays(user) < 370)
+
+console.log(
+  JSON.stringify(
+    usersToNotImport().map((user) => user.email),
+    null,
+    2
+  )
+)

sites/backend/v2-v3/migrate-export.mjs

@@ -0,0 +1,38 @@
+import path from 'path'
+import fs from 'fs'
+
+/*
+ * Only this token allows exporting data
+ */
+const export_token = 'TOKEN_HERE'
+
+/*
+ * Helper method to export a given collection
+ * from mongo via the v2 backend
+ */
+const exportCollection = async (name) => {
+  const result = await fetch(`https://backend.freesewing.org/admin/export/${name}`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ export_token: export_token }),
+  })
+  const data = await result.json()
+
+  return data
+}
+
+/*
+ * Load data from v2 backend
+ */
+const loadV2Data = async () => {
+  for (const collection of ['newsletters', 'people', 'patterns', 'users']) {
+    console.log(`Exporting: ${collection.toUpperCase()}`)
+    const data = await exportCollection(collection)
+    console.log(`  - ${data.length} records exported, writing to disk as v2-${collection}.json`)
+    fs.writeFileSync(`./v2-${collection}.json`, JSON.stringify(data, null, 2), 'utf-8')
+  }
+}
+
+await loadV2Data()

sites/sanity/sanity.config.js

@@ -6,7 +6,7 @@ import { schemaTypes } from './schemas'
 import { capitalize } from '../shared/utils.mjs'
 
 export default defineConfig(
-  ['site-content', 'user-content'].map((dataset) => ({
+  ['site-content'].map((dataset) => ({
     name: dataset,
     title: `FreeSewing ${dataset
       .split('-')

sites/sanity/schemas/avatar.js

@@ -8,6 +8,55 @@ export const userimg = {
       type: 'string',
       title: 'Image ID',
     },
+    {
+      name: 'title',
+      type: 'string',
+      title: 'Title',
+    },
+    {
+      name: 'img',
+      type: 'image',
+      title: 'Image',
+    },
+  ],
+}
+export const setimg = {
+  name: `setimg`,
+  type: 'document',
+  title: `Set Image`,
+  fields: [
+    {
+      name: 'recordid',
+      type: 'string',
+      title: 'Image ID',
+    },
+    {
+      name: 'title',
+      type: 'string',
+      title: 'Title',
+    },
+    {
+      name: 'img',
+      type: 'image',
+      title: 'Image',
+    },
+  ],
+}
+export const patternimg = {
+  name: `patternimg`,
+  type: 'document',
+  title: `Pattern Image`,
+  fields: [
+    {
+      name: 'recordid',
+      type: 'string',
+      title: 'Image ID',
+    },
+    {
+      name: 'title',
+      type: 'string',
+      title: 'Title',
+    },
     {
       name: 'img',
       type: 'image',

sites/sanity/schemas/index.js

@@ -1,5 +1,5 @@
 import { contentimg } from './img.js'
-import { userimg } from './avatar.js'
+import { userimg, setimg, patternimg } from './avatar.js'
 import { blogSchemaBuilder } from './blog.js'
 import { showcaseSchemaBuilder } from './showcase.js'
 import { newsletter } from './newsletter.js'
@@ -9,6 +9,8 @@ const languages = ['en', 'es', 'fr', 'nl', 'de']
 export const schemaTypes = [
   contentimg,
   userimg,
+  setimg,
+  patternimg,
   newsletter,
   ...languages.map((lang) => blogSchemaBuilder(lang)),
   ...languages.map((lang) => showcaseSchemaBuilder(lang)),