From b248d9892813bfd7105a514d68005c1fde4afde6 Mon Sep 17 00:00:00 2001 From: joostdecock Date: Sun, 13 Aug 2023 16:15:06 +0200 Subject: [PATCH] fix(backend): import fixes --- .gitignore | 1 + sites/backend/scripts/import-sizing-table.mjs | 6 +- sites/backend/src/controllers/imports.mjs | 7 +- sites/backend/src/models/pattern.mjs | 23 +++- sites/backend/src/models/set.mjs | 3 +- sites/backend/src/models/user.mjs | 130 ++++++++---------- sites/backend/src/routes/imports.mjs | 2 +- sites/backend/src/utils/model-decorator.mjs | 4 +- sites/backend/v2-v3/import.mjs | 112 --------------- sites/backend/v2-v3/inactive.mjs | 14 -- sites/backend/v2-v3/migrate-export.mjs | 38 ----- sites/backend/v2-v3/remove-set-images.mjs | 18 --- 12 files changed, 91 insertions(+), 267 deletions(-) delete mode 100644 sites/backend/v2-v3/import.mjs delete mode 100644 sites/backend/v2-v3/inactive.mjs delete mode 100644 sites/backend/v2-v3/migrate-export.mjs delete mode 100644 sites/backend/v2-v3/remove-set-images.mjs diff --git a/.gitignore b/.gitignore index 81beabdb614..8bbdcdd1cbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # See https://help.github.com/ignore-files/ for more about ignoring files. +dump # .env .env diff --git a/sites/backend/scripts/import-sizing-table.mjs b/sites/backend/scripts/import-sizing-table.mjs index 827e714ecc2..b27d6a399cb 100644 --- a/sites/backend/scripts/import-sizing-table.mjs +++ b/sites/backend/scripts/import-sizing-table.mjs @@ -158,8 +158,6 @@ sets.push( ) ) -importSets(sets) - async function createSet(data) { try { await prisma.curatedSet.create({ data }) @@ -174,3 +172,7 @@ async function importSets(sets) { await createSet(set) } } + +export async function importSizingTable() { + await importSets(sets) +} diff --git a/sites/backend/src/controllers/imports.mjs b/sites/backend/src/controllers/imports.mjs index a90705e5500..2b54985dfca 100644 --- a/sites/backend/src/controllers/imports.mjs +++ b/sites/backend/src/controllers/imports.mjs @@ -10,9 +10,6 @@ const runChecks = (req) => { if (req.body.import_token !== process.env.IMPORT_TOKEN) { return [401, { result: 'error', error: 'accessDenied' }] } - if (!Array.isArray(req.body.list)) { - return [400, { result: 'error', error: 'listMissing' }] - } return true } @@ -33,12 +30,12 @@ ImportsController.prototype.subscribers = async (req, res, tools) => { /* * Imports users in v2 format */ -ImportsController.prototype.users = async (req, res, tools) => { +ImportsController.prototype.user = async (req, res, tools) => { const check = runChecks(req) if (check !== true) return res.status(check[0]).send(check[1]) const User = new UserModel(tools) - await User.import(req.body.list) + await User.import(req.body.user) return User.sendResponse(res) } diff --git a/sites/backend/src/models/pattern.mjs b/sites/backend/src/models/pattern.mjs index d45d24c88c2..c28d003eb5b 100644 --- a/sites/backend/src/models/pattern.mjs +++ b/sites/backend/src/models/pattern.mjs @@ -486,6 +486,16 @@ const v2lut = { 'größe 42, mit brüsten': 8, 'größe 44, mit brüsten': 9, 'größe 46, mit brüsten': 10, + 'grösse 28, mit brüsten': 1, + 'grösse 30, mit brüsten': 2, + 'grösse 32, mit brüsten': 3, + 'grösse 34, mit brüsten': 4, + 'grösse 36, mit brüsten': 5, + 'grösse 38, mit brüsten': 6, + 'grösse 40, mit brüsten': 7, + 'grösse 42, mit brüsten': 8, + 'grösse 44, mit brüsten': 9, + 'grösse 46, mit brüsten': 10, 'taille 28, avec des seins': 1, 'taille 30, avec des seins': 2, 'taille 32, avec des seins': 3, @@ -557,6 +567,16 @@ const v2lut = { 'größe 46, ohne brüste': 18, 'größe 48, ohne brüste': 19, 'größe 50, ohne brüste': 20, + 'grösse 32, ohne brüste': 11, + 'grösse 34, ohne brüste': 12, + 'grösse 36, ohne brüste': 13, + 'grösse 38, ohne brüste': 14, + 'grösse 40, ohne brüste': 15, + 'grösse 42, ohne brüste': 16, + 'grösse 44, ohne brüste': 17, + 'grösse 46, ohne brüste': 18, + 'grösse 48, ohne brüste': 19, + 'grösse 50, ohne brüste': 20, 'tamaño 32, sin pechos': 11, 'tamaño 34, sin pechos': 12, 'tamaño 36, sin pechos': 13, @@ -585,9 +605,8 @@ PatternModel.prototype.import = async function (v2user, lut, userId) { if (!skip) { // V2 does not support images for patterns data.img = 'default-avatar' - const cloaked = await this.cloak(data) try { - this.record = await this.prisma.pattern.create({ data: cloaked }) + this.createRecord(data) } catch (err) { log.warn(err, 'Could not create pattern') console.log(data) diff --git a/sites/backend/src/models/set.mjs b/sites/backend/src/models/set.mjs index c1f92b18adc..3d2223751a7 100644 --- a/sites/backend/src/models/set.mjs +++ b/sites/backend/src/models/set.mjs @@ -441,9 +441,8 @@ SetModel.prototype.import = async function (v2user, userId) { }) data.img = imgId } else data.img = 'default-avatar' - const cloaked = await this.cloak(data) try { - this.record = await this.prisma.set.create({ data: cloaked }) + await this.createRecord(data) lut[handle] = this.record.id } catch (err) { log.warn(err, 'Could not create set') diff --git a/sites/backend/src/models/user.mjs b/sites/backend/src/models/user.mjs index b3e56941d0b..029f9629711 100644 --- a/sites/backend/src/models/user.mjs +++ b/sites/backend/src/models/user.mjs @@ -513,7 +513,7 @@ UserModel.prototype.linkSignIn = async function (req) { /* * Looks like we're good, so attempt to read the user from the database */ - await this.read({ id: this.Confirmation.record.user.id }) + await this.read({ id: this.Confirmation.record.userId }) /* * if anything went wrong, this.error will be set @@ -683,12 +683,12 @@ UserModel.prototype.confirm = async function ({ body, params }) { /* * If the id does not match, return 404 */ - if (data.id !== this.Confirmation.record.user.id) return this.setResponse(404) + if (data.id !== this.Confirmation.record.userId) return this.setResponse(404) /* * Attempt to load the user from the database */ - await this.read({ id: this.Confirmation.record.user.id }) + await this.read({ id: this.Confirmation.record.useId }) /* * If an error occured, it will be in this.error and we can return here @@ -1265,7 +1265,7 @@ const migrateUser = (v2) => { initial, imperial: v2.units === 'imperial', language: v2.settings.language, - lastSeen: Date.now(), + lastSeen: new Date(), lusername: v2.username.toLowerCase(), mfaEnabled: false, newsletter: false, @@ -1285,85 +1285,73 @@ const migrateUser = (v2) => { return data } -const lastLoginInDays = (user) => { - const now = new Date() - if (!user.time) console.log(user) - const then = new Date(user.time.login) - - const delta = Math.floor((now - then) / (1000 * 60 * 60 * 24)) - - return delta -} - /* * This is a special route not available for API users */ -UserModel.prototype.import = async function (list) { +UserModel.prototype.import = async function (user) { let created = 0 const skipped = [] - for (const sub of list) { - if (sub.status === 'active') { - const days = lastLoginInDays(sub) - if (days < 370) { - const data = migrateUser(sub) - await this.read({ ehash: data.ehash }) - if (!this.record) { - if (data.img) { - /* - * Figure out what image to grab from the FreeSewing v2 backend server - */ - const imgId = `user-${data.ihash}` - const imgUrl = - 'https://static.freesewing.org/users/' + - encodeURIComponent(sub.handle.slice(0, 1)) + - '/' + - encodeURIComponent(sub.handle) + - '/' + - encodeURIComponent(data.img) - data.img = await importImage({ - id: imgId, - metadata: { - user: `v2-${sub.handle}`, - ihash: data.ihash, - }, - url: imgUrl, - }) - data.img = imgId - } else data.img = 'default-avatar' - let cloaked = await this.cloak(data) + if (user.status === 'active') { + const data = migrateUser(user) + if (user.consent.profile) data.consent++ + if (user.consent.model || user.consent.measurements) { + data.consent++ + if (user.consent.openData) data.consent++ + } + + await this.read({ ehash: data.ehash }) + if (!this.record) { + /* + * Skip images for now + */ + if (false && data.img) { + /* + * Figure out what image to grab from the FreeSewing v2 backend server + */ + const imgId = `user-${data.ihash}` + const imgUrl = + 'https://static.freesewing.org/users/' + + encodeURIComponent(user.handle.slice(0, 1)) + + '/' + + encodeURIComponent(user.handle) + + '/' + + encodeURIComponent(data.img) + data.img = await importImage({ + id: imgId, + metadata: { + user: `v2-${user.handle}`, + ihash: data.ihash, + }, + url: imgUrl, + }) + data.img = imgId + } else data.img = 'default-avatar' + try { + await this.createRecord(data) + created++ + } catch (err) { + if ( + err.toString().indexOf('Unique constraint failed on the fields: (`lusername`)') !== -1 + ) { + // Just add a '+' to the username + data.username += '+' + data.lusername += '+' try { - this.record = await this.prisma.user.create({ data: cloaked }) + await this.createRecord(data) created++ } catch (err) { - if ( - err.toString().indexOf('Unique constraint failed on the fields: (`lusername`)') !== -1 - ) { - // Just add a '+' to the username - data.username += '+' - data.lusername += '+' - cloaked = await this.cloak(data) - try { - this.record = await this.prisma.user.create({ data: cloaked }) - created++ - } catch (err) { - log.warn(err, 'Could not create user record') - console.log(sub) - return this.setResponse(500, 'createUserFailed') - } - } + log.warn(err, 'Could not create user record') + console.log(user) + return this.setResponse(500, 'createUserFailed') } } - } else skipped.push(sub.email) + } // That's the user, now load their people as sets let lut = false - if (sub.people) lut = await this.Set.import(sub, this.record.id) - if (sub.patterns) await this.Pattern.import(sub, lut, this.record.id) - } else skipped.push(sub.email) + if (user.people) lut = await this.Set.import(user, this.record.id) + if (user.patterns) await this.Pattern.import(user, lut, this.record.id) + } } - return this.setResponse200({ - skipped, - total: list.length, - imported: created, - }) + return this.setResponse200() } diff --git a/sites/backend/src/routes/imports.mjs b/sites/backend/src/routes/imports.mjs index 1103c4c9c11..9e773ab8ad9 100644 --- a/sites/backend/src/routes/imports.mjs +++ b/sites/backend/src/routes/imports.mjs @@ -14,5 +14,5 @@ export function importsRoutes(tools) { app.post('/import/subscribers', (req, res) => Import.subscribers(req, res, tools)) // Import users - app.post('/import/users', (req, res) => Import.users(req, res, tools)) + app.post('/import/user', (req, res) => Import.user(req, res, tools)) } diff --git a/sites/backend/src/utils/model-decorator.mjs b/sites/backend/src/utils/model-decorator.mjs index 97264ab4865..6636efefdc6 100644 --- a/sites/backend/src/utils/model-decorator.mjs +++ b/sites/backend/src/utils/model-decorator.mjs @@ -197,8 +197,8 @@ export function decorateModel(Model, tools, modelConfig) { /* * Some error occured. Log warning and return 500 */ - log.warn(err, 'Could not create set') - return this.setResponse(500, 'createSetFailed') + log.warn(err, `Could not create ${modelConfig.name}`) + return this.setResponse(500, `create${capitalize(modelConfig.name)}Failed`) } return this diff --git a/sites/backend/v2-v3/import.mjs b/sites/backend/v2-v3/import.mjs deleted file mode 100644 index 0c4e1565ddd..00000000000 --- a/sites/backend/v2-v3/import.mjs +++ /dev/null @@ -1,112 +0,0 @@ -import dotenv from 'dotenv' -//import subscribers from './v2-newsletters.json' assert { type: 'json' } -import users from '../dump/v2-users.json' assert { type: 'json' } -import people from '../dump/v2-people.json' assert { type: 'json' } -import patterns from '../dump/v2-patterns.json' assert { type: 'json' } -dotenv.config() - -const batchSize = 100 - -/* - * Only this token allows exporting data - */ -const import_token = process.env.IMPORT_TOKEN - -/* - * Where to connect to? - */ -const BACKEND = 'http://localhost:3000' - -const splitArray = (split, batchSize) => - split.reduce((result, item, index) => { - const batchIndex = Math.floor(index / batchSize) - if (!result[batchIndex]) result[batchIndex] = [] - result[batchIndex].push(item) - - return result - }, []) - -/* - * Commented out because linter -const importSubscribers = async () => { - console.log('Importing subscribers') - const count = subscribers.length - let total = 0 - const batches = splitArray(subscribers, batchSize) - for (const batch of batches) { - const result = await fetch(`${BACKEND}/import/subscribers`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - import_token: import_token, - list: batch, - }), - }) - const data = await result.json() - total += data.imported - console.log(`${total}/${count}`) - } -} -*/ -const lastLoginInDays = (user) => { - const now = new Date() - const then = new Date(user.time.login) - - return Math.floor((now - then) / (1000 * 60 * 60 * 24)) -} - -const usersToImport = () => - users.filter((user) => user.status === 'active' && lastLoginInDays(user) < 370) -// Commented out for linter -// const usersToNotImport = () => -// users.filter((user) => user.status !== 'active' && lastLoginInDays(user) >= 370) - -const importUsers = async () => { - console.log('Processing users') - const todo = usersToImport() - // Put users in an object with their handle as key - const allUsers = {} - for (const user of todo) allUsers[user.handle] = user - // Find all people belonging to this user - for (const person of people) { - if (typeof allUsers[person.user] !== 'undefined') { - if (typeof allUsers[person.user].people === 'undefined') allUsers[person.user].people = {} - allUsers[person.user].people[person.handle] = person - } - } - // Find all patterns belonging to this user - for (const pattern of patterns) { - if (typeof allUsers[pattern.user] !== 'undefined') { - if (typeof allUsers[pattern.user].patterns === 'undefined') - allUsers[pattern.user].patterns = {} - allUsers[pattern.user].patterns[pattern.handle] = pattern - } - } - console.log('Importing users') - const count = todo.length - let total = 0 - const batches = splitArray(todo, batchSize) - for (const batch of batches) { - await fetch(`${BACKEND}/import/users`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - import_token: import_token, - list: batch, - }), - }) - total += batchSize - console.log(`${total}/${count}`) - } -} - -const importAll = async () => { - //await importSubscribers() - await importUsers() -} - -importAll() diff --git a/sites/backend/v2-v3/inactive.mjs b/sites/backend/v2-v3/inactive.mjs deleted file mode 100644 index 6d8461f1eca..00000000000 --- a/sites/backend/v2-v3/inactive.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import users from '../dump/v2-users.json' assert { type: 'json' } - -const usersToNotImport = () => users.filter((user) => user.status !== 'active') -// Commented out for linter -//const usersToImport = () => -// users.filter((user) => user.status === 'active') - -console.log( - JSON.stringify( - usersToNotImport().map((user) => user.email), - null, - 2 - ) -) diff --git a/sites/backend/v2-v3/migrate-export.mjs b/sites/backend/v2-v3/migrate-export.mjs deleted file mode 100644 index 87c35f5a75d..00000000000 --- a/sites/backend/v2-v3/migrate-export.mjs +++ /dev/null @@ -1,38 +0,0 @@ -//import path from 'path' -import fs from 'fs' - -/* - * Only this token allows exporting data - */ -const export_token = 'TOKEN_HERE' - -/* - * Helper method to export a given collection - * from mongo via the v2 backend - */ -const exportCollection = async (name) => { - const result = await fetch(`https://backend.freesewing.org/admin/export/${name}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ export_token: export_token }), - }) - const data = await result.json() - - return data -} - -/* - * Load data from v2 backend - */ -const loadV2Data = async () => { - for (const collection of ['newsletters', 'people', 'patterns', 'users']) { - console.log(`Exporting: ${collection.toUpperCase()}`) - const data = await exportCollection(collection) - console.log(` - ${data.length} records exported, writing to disk as v2-${collection}.json`) - fs.writeFileSync(`./v2-${collection}.json`, JSON.stringify(data, null, 2), 'utf-8') - } -} - -await loadV2Data() diff --git a/sites/backend/v2-v3/remove-set-images.mjs b/sites/backend/v2-v3/remove-set-images.mjs deleted file mode 100644 index 9bc49c69509..00000000000 --- a/sites/backend/v2-v3/remove-set-images.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { cloudflareImages as config } from '../src/config.mjs' -import axios from 'axios' - -const headers = { Authorization: `Bearer ${config.token}` } - -const result = await axios.get(`${config.api}?page=1&per_page=10000`, { headers }) - -const images = result.data.result.images.map((i) => i.id).filter((id) => id.slice(0, 4) === 'set-') -const total = images.length -if (total > 0) { - console.log(`${total} images to remove`) - let i = 1 - for (const id of images) { - console.log(`${i}/${total} : Removing ${id}`) - await axios.delete(`${config.api}/${id}`, { headers }) - i++ - } -} else console.log('No images to remove')