fix(backend): import fixes

.gitignore

@@ -1,4 +1,5 @@
 # See https://help.github.com/ignore-files/ for more about ignoring files.
+dump
 
 # .env
 .env

sites/backend/scripts/import-sizing-table.mjs

@@ -158,8 +158,6 @@ sets.push(
   )
 )
 
-importSets(sets)
-
 async function createSet(data) {
   try {
     await prisma.curatedSet.create({ data })
@@ -174,3 +172,7 @@ async function importSets(sets) {
     await createSet(set)
   }
 }
+
+export async function importSizingTable() {
+  await importSets(sets)
+}

sites/backend/src/controllers/imports.mjs

@@ -10,9 +10,6 @@ const runChecks = (req) => {
   if (req.body.import_token !== process.env.IMPORT_TOKEN) {
     return [401, { result: 'error', error: 'accessDenied' }]
   }
-  if (!Array.isArray(req.body.list)) {
-    return [400, { result: 'error', error: 'listMissing' }]
-  }
 
   return true
 }
@@ -33,12 +30,12 @@ ImportsController.prototype.subscribers = async (req, res, tools) => {
 /*
  * Imports users in v2 format
  */
-ImportsController.prototype.users = async (req, res, tools) => {
+ImportsController.prototype.user = async (req, res, tools) => {
   const check = runChecks(req)
   if (check !== true) return res.status(check[0]).send(check[1])
 
   const User = new UserModel(tools)
-  await User.import(req.body.list)
+  await User.import(req.body.user)
 
   return User.sendResponse(res)
 }

sites/backend/src/models/pattern.mjs

@@ -486,6 +486,16 @@ const v2lut = {
   'größe 42, mit brüsten': 8,
   'größe 44, mit brüsten': 9,
   'größe 46, mit brüsten': 10,
+  'grösse 28, mit brüsten': 1,
+  'grösse 30, mit brüsten': 2,
+  'grösse 32, mit brüsten': 3,
+  'grösse 34, mit brüsten': 4,
+  'grösse 36, mit brüsten': 5,
+  'grösse 38, mit brüsten': 6,
+  'grösse 40, mit brüsten': 7,
+  'grösse 42, mit brüsten': 8,
+  'grösse 44, mit brüsten': 9,
+  'grösse 46, mit brüsten': 10,
   'taille 28, avec des seins': 1,
   'taille 30, avec des seins': 2,
   'taille 32, avec des seins': 3,
@@ -557,6 +567,16 @@ const v2lut = {
   'größe 46, ohne brüste': 18,
   'größe 48, ohne brüste': 19,
   'größe 50, ohne brüste': 20,
+  'grösse 32, ohne brüste': 11,
+  'grösse 34, ohne brüste': 12,
+  'grösse 36, ohne brüste': 13,
+  'grösse 38, ohne brüste': 14,
+  'grösse 40, ohne brüste': 15,
+  'grösse 42, ohne brüste': 16,
+  'grösse 44, ohne brüste': 17,
+  'grösse 46, ohne brüste': 18,
+  'grösse 48, ohne brüste': 19,
+  'grösse 50, ohne brüste': 20,
   'tamaño 32, sin pechos': 11,
   'tamaño 34, sin pechos': 12,
   'tamaño 36, sin pechos': 13,
@@ -585,9 +605,8 @@ PatternModel.prototype.import = async function (v2user, lut, userId) {
     if (!skip) {
       // V2 does not support images for patterns
       data.img = 'default-avatar'
-      const cloaked = await this.cloak(data)
       try {
-        this.record = await this.prisma.pattern.create({ data: cloaked })
+        this.createRecord(data)
       } catch (err) {
         log.warn(err, 'Could not create pattern')
         console.log(data)

sites/backend/src/models/set.mjs

@@ -441,9 +441,8 @@ SetModel.prototype.import = async function (v2user, userId) {
       })
       data.img = imgId
     } else data.img = 'default-avatar'
-    const cloaked = await this.cloak(data)
     try {
-      this.record = await this.prisma.set.create({ data: cloaked })
+      await this.createRecord(data)
       lut[handle] = this.record.id
     } catch (err) {
       log.warn(err, 'Could not create set')

sites/backend/src/models/user.mjs

@@ -513,7 +513,7 @@ UserModel.prototype.linkSignIn = async function (req) {
   /*
    * Looks like we're good, so attempt to read the user from the database
    */
-  await this.read({ id: this.Confirmation.record.user.id })
+  await this.read({ id: this.Confirmation.record.userId })
 
   /*
    * if anything went wrong, this.error will be set
@@ -683,12 +683,12 @@ UserModel.prototype.confirm = async function ({ body, params }) {
   /*
    * If the id does not match, return 404
    */
-  if (data.id !== this.Confirmation.record.user.id) return this.setResponse(404)
+  if (data.id !== this.Confirmation.record.userId) return this.setResponse(404)
 
   /*
    * Attempt to load the user from the database
    */
-  await this.read({ id: this.Confirmation.record.user.id })
+  await this.read({ id: this.Confirmation.record.useId })
 
   /*
    * If an error occured, it will be in this.error and we can return here
@@ -1265,7 +1265,7 @@ const migrateUser = (v2) => {
     initial,
     imperial: v2.units === 'imperial',
     language: v2.settings.language,
-    lastSeen: Date.now(),
+    lastSeen: new Date(),
     lusername: v2.username.toLowerCase(),
     mfaEnabled: false,
     newsletter: false,
@@ -1285,54 +1285,49 @@ const migrateUser = (v2) => {
   return data
 }
 
-const lastLoginInDays = (user) => {
-  const now = new Date()
-  if (!user.time) console.log(user)
-  const then = new Date(user.time.login)
-
-  const delta = Math.floor((now - then) / (1000 * 60 * 60 * 24))
-
-  return delta
-}
-
 /*
  * This is a special route not available for API users
  */
-UserModel.prototype.import = async function (list) {
+UserModel.prototype.import = async function (user) {
   let created = 0
   const skipped = []
-  for (const sub of list) {
+  if (user.status === 'active') {
-    if (sub.status === 'active') {
+    const data = migrateUser(user)
-      const days = lastLoginInDays(sub)
+    if (user.consent.profile) data.consent++
-      if (days < 370) {
+    if (user.consent.model || user.consent.measurements) {
-        const data = migrateUser(sub)
+      data.consent++
+      if (user.consent.openData) data.consent++
+    }
+
     await this.read({ ehash: data.ehash })
     if (!this.record) {
-          if (data.img) {
+      /*
+       * Skip images for now
+       */
+      if (false && data.img) {
         /*
          * Figure out what image to grab from the FreeSewing v2 backend server
          */
         const imgId = `user-${data.ihash}`
         const imgUrl =
           'https://static.freesewing.org/users/' +
-              encodeURIComponent(sub.handle.slice(0, 1)) +
+          encodeURIComponent(user.handle.slice(0, 1)) +
           '/' +
-              encodeURIComponent(sub.handle) +
+          encodeURIComponent(user.handle) +
           '/' +
           encodeURIComponent(data.img)
         data.img = await importImage({
           id: imgId,
           metadata: {
-                user: `v2-${sub.handle}`,
+            user: `v2-${user.handle}`,
             ihash: data.ihash,
           },
           url: imgUrl,
         })
         data.img = imgId
       } else data.img = 'default-avatar'
-          let cloaked = await this.cloak(data)
       try {
-            this.record = await this.prisma.user.create({ data: cloaked })
+        await this.createRecord(data)
         created++
       } catch (err) {
         if (
@@ -1341,29 +1336,22 @@ UserModel.prototype.import = async function (list) {
           // Just add a '+' to the username
           data.username += '+'
           data.lusername += '+'
-              cloaked = await this.cloak(data)
           try {
-                this.record = await this.prisma.user.create({ data: cloaked })
+            await this.createRecord(data)
             created++
           } catch (err) {
             log.warn(err, 'Could not create user record')
-                console.log(sub)
+            console.log(user)
             return this.setResponse(500, 'createUserFailed')
           }
         }
       }
-        }
-      } else skipped.push(sub.email)
       // That's the user, now load their people as sets
       let lut = false
-      if (sub.people) lut = await this.Set.import(sub, this.record.id)
+      if (user.people) lut = await this.Set.import(user, this.record.id)
-      if (sub.patterns) await this.Pattern.import(sub, lut, this.record.id)
+      if (user.patterns) await this.Pattern.import(user, lut, this.record.id)
-    } else skipped.push(sub.email)
+    }
   }
 
-  return this.setResponse200({
+  return this.setResponse200()
-    skipped,
-    total: list.length,
-    imported: created,
-  })
 }

sites/backend/src/routes/imports.mjs

@@ -14,5 +14,5 @@ export function importsRoutes(tools) {
   app.post('/import/subscribers', (req, res) => Import.subscribers(req, res, tools))
 
   // Import users
-  app.post('/import/users', (req, res) => Import.users(req, res, tools))
+  app.post('/import/user', (req, res) => Import.user(req, res, tools))
 }

sites/backend/src/utils/model-decorator.mjs

@@ -197,8 +197,8 @@ export function decorateModel(Model, tools, modelConfig) {
       /*
        * Some error occured. Log warning and return 500
        */
-      log.warn(err, 'Could not create set')
+      log.warn(err, `Could not create ${modelConfig.name}`)
-      return this.setResponse(500, 'createSetFailed')
+      return this.setResponse(500, `create${capitalize(modelConfig.name)}Failed`)
     }
 
     return this

sites/backend/v2-v3/import.mjs

@@ -1,112 +0,0 @@
-import dotenv from 'dotenv'
-//import subscribers from './v2-newsletters.json' assert { type: 'json' }
-import users from '../dump/v2-users.json' assert { type: 'json' }
-import people from '../dump/v2-people.json' assert { type: 'json' }
-import patterns from '../dump/v2-patterns.json' assert { type: 'json' }
-dotenv.config()
-
-const batchSize = 100
-
-/*
- * Only this token allows exporting data
- */
-const import_token = process.env.IMPORT_TOKEN
-
-/*
- * Where to connect to?
- */
-const BACKEND = 'http://localhost:3000'
-
-const splitArray = (split, batchSize) =>
-  split.reduce((result, item, index) => {
-    const batchIndex = Math.floor(index / batchSize)
-    if (!result[batchIndex]) result[batchIndex] = []
-    result[batchIndex].push(item)
-
-    return result
-  }, [])
-
-/*
- * Commented out because linter
-const importSubscribers = async () => {
-  console.log('Importing subscribers')
-  const count = subscribers.length
-  let total = 0
-  const batches = splitArray(subscribers, batchSize)
-  for (const batch of batches) {
-    const result = await fetch(`${BACKEND}/import/subscribers`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        import_token: import_token,
-        list: batch,
-      }),
-    })
-    const data = await result.json()
-    total += data.imported
-    console.log(`${total}/${count}`)
-  }
-}
-*/
-const lastLoginInDays = (user) => {
-  const now = new Date()
-  const then = new Date(user.time.login)
-
-  return Math.floor((now - then) / (1000 * 60 * 60 * 24))
-}
-
-const usersToImport = () =>
-  users.filter((user) => user.status === 'active' && lastLoginInDays(user) < 370)
-// Commented out for linter
-// const usersToNotImport = () =>
-//   users.filter((user) => user.status !== 'active' && lastLoginInDays(user) >= 370)
-
-const importUsers = async () => {
-  console.log('Processing users')
-  const todo = usersToImport()
-  // Put users in an object with their handle as key
-  const allUsers = {}
-  for (const user of todo) allUsers[user.handle] = user
-  // Find all people belonging to this user
-  for (const person of people) {
-    if (typeof allUsers[person.user] !== 'undefined') {
-      if (typeof allUsers[person.user].people === 'undefined') allUsers[person.user].people = {}
-      allUsers[person.user].people[person.handle] = person
-    }
-  }
-  // Find all patterns belonging to this user
-  for (const pattern of patterns) {
-    if (typeof allUsers[pattern.user] !== 'undefined') {
-      if (typeof allUsers[pattern.user].patterns === 'undefined')
-        allUsers[pattern.user].patterns = {}
-      allUsers[pattern.user].patterns[pattern.handle] = pattern
-    }
-  }
-  console.log('Importing users')
-  const count = todo.length
-  let total = 0
-  const batches = splitArray(todo, batchSize)
-  for (const batch of batches) {
-    await fetch(`${BACKEND}/import/users`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        import_token: import_token,
-        list: batch,
-      }),
-    })
-    total += batchSize
-    console.log(`${total}/${count}`)
-  }
-}
-
-const importAll = async () => {
-  //await importSubscribers()
-  await importUsers()
-}
-
-importAll()

sites/backend/v2-v3/inactive.mjs

@@ -1,14 +0,0 @@
-import users from '../dump/v2-users.json' assert { type: 'json' }
-
-const usersToNotImport = () => users.filter((user) => user.status !== 'active')
-// Commented out for linter
-//const usersToImport = () =>
-//  users.filter((user) => user.status === 'active')
-
-console.log(
-  JSON.stringify(
-    usersToNotImport().map((user) => user.email),
-    null,
-    2
-  )
-)

sites/backend/v2-v3/migrate-export.mjs

@@ -1,38 +0,0 @@
-//import path from 'path'
-import fs from 'fs'
-
-/*
- * Only this token allows exporting data
- */
-const export_token = 'TOKEN_HERE'
-
-/*
- * Helper method to export a given collection
- * from mongo via the v2 backend
- */
-const exportCollection = async (name) => {
-  const result = await fetch(`https://backend.freesewing.org/admin/export/${name}`, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify({ export_token: export_token }),
-  })
-  const data = await result.json()
-
-  return data
-}
-
-/*
- * Load data from v2 backend
- */
-const loadV2Data = async () => {
-  for (const collection of ['newsletters', 'people', 'patterns', 'users']) {
-    console.log(`Exporting: ${collection.toUpperCase()}`)
-    const data = await exportCollection(collection)
-    console.log(`  - ${data.length} records exported, writing to disk as v2-${collection}.json`)
-    fs.writeFileSync(`./v2-${collection}.json`, JSON.stringify(data, null, 2), 'utf-8')
-  }
-}
-
-await loadV2Data()

sites/backend/v2-v3/remove-set-images.mjs

@@ -1,18 +0,0 @@
-import { cloudflareImages as config } from '../src/config.mjs'
-import axios from 'axios'
-
-const headers = { Authorization: `Bearer ${config.token}` }
-
-const result = await axios.get(`${config.api}?page=1&per_page=10000`, { headers })
-
-const images = result.data.result.images.map((i) => i.id).filter((id) => id.slice(0, 4) === 'set-')
-const total = images.length
-if (total > 0) {
-  console.log(`${total} images to remove`)
-  let i = 1
-  for (const id of images) {
-    console.log(`${i}/${total} : Removing ${id}`)
-    await axios.delete(`${config.api}/${id}`, { headers })
-    i++
-  }
-} else console.log('No images to remove')