feat(backend): Bunch of changes for Docker

config/dependencies.yaml

@@ -202,23 +202,28 @@ yuri:
 
 backend:
   _:
-    '@aws-sdk/client-sesv2': '^3.200.0'
+    '@aws-sdk/client-sesv2': '3.200.0'
     '@prisma/client': '4.7.1'
-    'bcryptjs': '^2.4.3'
+    'bcryptjs': '2.4.3'
-    'crypto': '^1.0.1'
+    'cors': '2.8.5'
+    'crypto': '1.0.1'
+    'dotenv': '16.0.3'
     'express': '4.18.2'
-    'mustache': '^4.2.0'
+    'lodash.get': *_get
-    'otplib': '^12.0.1'
+    'mustache': '4.2.0'
-    'passport': '^0.6.0'
+    'otplib': '12.0.1'
-    'passport-http': '^0.3.0'
+    'passport': '0.6.0'
-    'passport-jwt': '^4.0.0'
+    'passport-http': '0.3.0'
-    'pino': '^8.7.0'
+    'passport-jwt': '4.0.0'
-    'qrcode': '^1.5.1'
+    'pino': '8.7.0'
+    'qrcode': '1.5.1'
   dev:
     'chai': *chai
-    'chai-http': '^4.3.0'
+    'chai-http': '4.3.0'
+    'esbuild': '0.16.9'
     'mocha': *mocha
-    'mocha-steps': '^1.3.0'
+    'mocha-steps': '1.3.0'
+    'nodemon': '2.0.20'
     'prisma': '4.7.1'
 
 dev:

config/scripts.yaml

@@ -49,12 +49,16 @@ snapseries:
 
 # Sites go here
 backend:
+  build: 'node build.mjs'
+  clean: 'rimraf dist'
   dev: 'nodemon src/index.mjs'
   initdb: 'npx prisma db push'
+  mbuild: 'NO_MINIFY=1 node build.mjs'
   newdb: 'node ./scripts/newdb.mjs'
   prettier: "npx prettier --write 'src/*.mjs' 'tests/*.mjs'"
   rmdb: 'node ./scripts/rmdb.mjs'
   test: 'npx mocha --require mocha-steps tests/index.mjs'
+  vbuild: 'VERBOSE=1 node build.mjs'
 
 dev:
   build: &nextBuild 'node --experimental-json-modules ../../node_modules/next/dist/bin/next build'

sites/backend/.gitignore

@@ -1,2 +1,7 @@
+# Protect auto-generated encryption keys
+encryption.key
+
+# SQLite databases
 *.sqlite
 *.sqlite-journal
+

sites/backend/Dockerfile

@@ -1,37 +1,44 @@
 ## Stage 1: Builder
-FROM node:alpine as builder
+FROM node:16.15-slim as builder
 
 ## Set workdir
 WORKDIR /backend
 
-## Install build toolchain
-#RUN apk add --no-cache python make g++
-
 ## Install node dependencies
 COPY package* ./
+COPY prisma .
+RUN apt-get update && apt-get install -y openssl
 RUN npm install pm2 && npm ci
+RUN ls -l /backend/node_modules/prisma/libquery_engine-debian-openssl-1.1.x.so.node
+RUN npx prisma generate
 
 ## Build app
 COPY package.json package.json
 COPY src src
+COPY prisma prisma
+COPY local-config.mjs local-config.mjs
 COPY build.mjs build.mjs
 RUN npm run build
 
 ## Stage 2: App
-FROM node:alpine as app
+FROM node:16.15-slim as app
 
 ## Set workdir
 WORKDIR /backend
 
 ## Copy built node modules and binaries without including the toolchain
 COPY --from=builder /backend/node_modules/ /backend/node_modules/
-COPY --from=builder /backend/build/ /backend/build/
+COPY --from=builder /backend/dist/ /backend/dist/
 COPY --from=builder /backend/package.json /backend/package.json
+COPY --from=builder /backend/prisma /backend
+COPY --from=builder /backend/prisma/schema.sqlite /backend/db.sqlite
+COPY --from=builder /backend/local-config.mjs /backend/
+RUN mkdir -p /backend/src/landing
+COPY --from=builder /backend/src/landing/* /backend/src/landing/
 
 ## Add a user to run the app
-RUN addgroup -S freesewing \
+RUN useradd --home-dir /backend --comment FreeSewing --no-create-home --uid 20000 freesewing
-  && adduser -S freesewing \
+RUN chown -R freesewing /backend
-  && chown -R freesewing /backend
 
 ## Drop privleges and run app
 USER freesewing

sites/backend/build.mjs

@@ -12,12 +12,23 @@ const banner = `/**
 
 // Shared esbuild options
 const options = {
-  banner: { js: banner },
+  banner: {
+    js: `// See: https://github.com/evanw/esbuild/issues/1921
+import { createRequire } from 'module';
+import path from 'path';
+import { fileURLToPath } from 'url';
+const require = createRequire(import.meta.url);
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+${banner}
+`,
+  },
   bundle: true,
   entryPoints: ['src/index.mjs'],
   format: 'esm',
   outfile: 'dist/index.mjs',
-  external: [],
+  external: ['./local-config.mjs'],
   metafile: process.env.VERBOSE ? true : false,
   minify: process.env.NO_MINIFY ? false : true,
   sourcemap: true,

sites/backend/local-config.mjs

@@ -0,0 +1,16 @@
+/*
+ * This method allows you to change/override the backend config
+ *
+ * It takes the initial (base) config object.
+ * It must return the (modified) config object.
+ *
+ * Note that you can configure a lot via environment variables
+ * but if you prefer to keep certain aspects of the config in
+ * code, you can override this file.
+ *
+ * If you're running this in Docker, you can volume-mount only this file.
+ * This gives you full control over the container configuration.
+ */
+export function postConfig(config) {
+  return config
+}

sites/backend/package.json

@@ -14,37 +14,42 @@
     "url": "https://freesewing.org/patrons/join"
   },
   "scripts": {
+    "build": "node build.mjs",
+    "clean": "rimraf dist",
     "dev": "nodemon src/index.mjs",
-    "build": "node --experimental-json-modules build.mjs",
-    "test": "npx mocha --require mocha-steps tests/index.mjs",
     "initdb": "npx prisma db push",
+    "mbuild": "NO_MINIFY=1 node build.mjs",
     "newdb": "node ./scripts/newdb.mjs",
     "prettier": "npx prettier --write 'src/*.mjs' 'tests/*.mjs'",
     "rmdb": "node ./scripts/rmdb.mjs",
-    "test": "npx mocha --require mocha-steps tests/index.mjs"
+    "test": "npx mocha --require mocha-steps tests/index.mjs",
+    "vbuild": "VERBOSE=1 node build.mjs"
   },
   "peerDependencies": {},
   "dependencies": {
-    "@aws-sdk/client-sesv2": "^3.200.0",
+    "@aws-sdk/client-sesv2": "3.200.0",
     "@prisma/client": "4.7.1",
-    "bcryptjs": "^2.4.3",
+    "bcryptjs": "2.4.3",
-    "crypto": "^1.0.1",
+    "cors": "2.8.5",
-    "esbuild": "^0.16.8",
+    "crypto": "1.0.1",
+    "dotenv": "16.0.3",
     "express": "4.18.2",
-    "mustache": "^4.2.0",
+    "lodash.get": "4.4.2",
-    "otplib": "^12.0.1",
+    "mustache": "4.2.0",
-    "passport": "^0.6.0",
+    "otplib": "12.0.1",
-    "passport-http": "^0.3.0",
+    "passport": "0.6.0",
-    "passport-jwt": "^4.0.0",
+    "passport-http": "0.3.0",
-    "pino": "^8.7.0",
+    "passport-jwt": "4.0.0",
-    "qrcode": "^1.5.1",
+    "pino": "8.7.0",
-    "cors": "latest"
+    "qrcode": "1.5.1"
   },
   "devDependencies": {
     "chai": "^4.2.0",
-    "chai-http": "^4.3.0",
+    "chai-http": "4.3.0",
+    "esbuild": "0.16.9",
     "mocha": "^10.0.0",
-    "mocha-steps": "^1.3.0",
+    "mocha-steps": "1.3.0",
+    "nodemon": "2.0.20",
     "prisma": "4.7.1"
   },
   "engines": {

sites/backend/prisma/schema.prisma

@@ -1,5 +1,6 @@
 generator client {
   provider = "prisma-client-js"
+  binaryTargets = ["native", "debian-openssl-1.1.x"]
 }
 
 datasource db {

sites/backend/src/config.mjs

@@ -2,13 +2,22 @@ import chalk from 'chalk'
 // Load environment variables
 import dotenv from 'dotenv'
 import { asJson } from './utils/index.mjs'
+import { randomString } from './utils/crypto.mjs'
 import { measurements } from './measurements.mjs'
+import get from 'lodash.get'
+import { readFileSync, writeFileSync } from 'node:fs'
+import { postConfig } from '../local-config.mjs'
 dotenv.config()
 
 // Allow these 2 to be imported
 export const port = process.env.BACKEND_PORT || 3000
 export const api = process.env.BACKEND_URL || `http://localhost:${port}`
 
+// Generate/Check encryption key only once
+const encryptionKey = process.env.BACKEND_ENC_KEY
+  ? process.env.BACKEND_ENC_KEY
+  : randomEncryptionKey()
+
 // All environment variables are strings
 // This is a helper method to turn them into a boolean
 const envToBool = (input = 'no') => {
@@ -17,7 +26,7 @@ const envToBool = (input = 'no') => {
 }
 
 // Construct config object
-const config = {
+const baseConfig = {
   // Feature flags
   use: {
     github: envToBool(process.env.BACKEND_ENABLE_GITHUB),
@@ -45,13 +54,13 @@ const config = {
     pattern: process.env.BACKEND_AVATAR_PATTERN || 'https://freesewing.org/avatar.svg',
   },
   db: {
-    url: process.env.BACKEND_DB_URL,
+    url: process.env.BACKEND_DB_URL || './db.sqlite',
   },
   encryption: {
-    key: process.env.BACKEND_ENC_KEY,
+    key: encryptionKey,
   },
   jwt: {
-    secretOrKey: process.env.BACKEND_ENC_KEY,
+    secretOrKey: encryptionKey,
     issuer: process.env.BACKEND_JWT_ISSUER || 'freesewing.org',
     audience: process.env.BACKEND_JWT_ISSUER || 'freesewing.org',
     expiresIn: process.env.BACKEND_JWT_EXPIRY || '7d',
@@ -84,8 +93,8 @@ const config = {
  */
 
 // Github config
-if (config.use.github)
+if (baseConfig.use.github)
-  config.github = {
+  baseConfig.github = {
     token: process.env.BACKEND_GITHUB_TOKEN,
     api: 'https://api.github.com',
     bot: {
@@ -116,17 +125,17 @@ if (config.use.github)
   }
 
 // Unit test config
-if (config.use.tests.base)
+if (baseConfig.use.tests.base)
-  config.tests = {
+  baseConfig.tests = {
     domain: process.env.BACKEND_TEST_DOMAIN || 'freesewing.dev',
   }
 
 // Sanity config
-if (config.use.sanity)
+if (baseConfig.use.sanity)
-  config.sanity = {
+  baseConfig.sanity = {
     project: process.env.SANITY_PROJECT,
     dataset: process.env.SANITY_DATASET || 'production',
-    token: process.env.SANITY_TOKEN,
+    token: process.env.SANITY_TOKEN || 'fixmeSetSanityToken',
     version: process.env.SANITY_VERSION || 'v2022-10-31',
     api: `https://${process.env.SANITY_PROJECT || 'missing-project-id'}.api.sanity.io/${
       process.env.SANITY_VERSION || 'v2022-10-31'
@@ -134,8 +143,8 @@ if (config.use.sanity)
   }
 
 // AWS SES config (for sending out emails)
-if (config.use.ses)
+if (baseConfig.use.ses)
-  config.aws = {
+  baseConfig.aws = {
     ses: {
       region: process.env.BACKEND_AWS_SES_REGION || 'us-east-1',
       from: process.env.BACKEND_AWS_SES_FROM || 'FreeSewing <info@freesewing.org>',
@@ -151,8 +160,8 @@ if (config.use.ses)
   }
 
 // Oauth config for Github as a provider
-if (config.use.oauth?.github)
+if (baseConfig.use.oauth?.github)
-  config.oauth.github = {
+  baseConfig.oauth.github = {
     clientId: process.env.BACKEND_OAUTH_GITHUB_CLIENT_ID,
     clientSecret: process.env.BACKEND_OAUTH_GITHUB_CLIENT_SECRET,
     tokenUri: 'https://github.com/login/oauth/access_token',
@@ -161,24 +170,27 @@ if (config.use.oauth?.github)
   }
 
 // Oauth config for Google as a provider
-if (config.use.oauth?.google)
+if (baseConfig.use.oauth?.google)
-  config.oauth.google = {
+  baseConfig.oauth.google = {
     clientId: process.env.BACKEND_OAUTH_GOOGLE_CLIENT_ID,
     clientSecret: process.env.BACKEND_OAUTH_GOOGLE_CLIENT_SECRET,
     tokenUri: 'https://oauth2.googleapis.com/token',
     dataUri: 'https://people.googleapis.com/v1/people/me?personFields=emailAddresses,names,photos',
   }
 
+// Load local config
+const config = postConfig(baseConfig)
+
 // Exporting this stand-alone config
 export const sanity = config.sanity || {}
 export const website = config.website
 
 const vars = {
-  BACKEND_DB_URL: 'required',
+  BACKEND_DB_URL: ['required', 'db.url'],
   BACKEND_PORT: 'optional',
   BACKEND_WEBSITE_DOMAIN: 'optional',
   BACKEND_WEBSITE_SCHEME: 'optional',
-  BACKEND_ENC_KEY: 'requiredSecret',
+  BACKEND_ENC_KEY: ['requiredSecret', 'encryption.key'],
   BACKEND_JWT_ISSUER: 'optional',
   BACKEND_JWT_EXPIRY: 'optional',
   // Feature flags
@@ -249,12 +261,19 @@ export function verifyConfig(silent = false) {
   const errors = []
   const ok = []
 
-  for (const [key, type] of Object.entries(vars)) {
+  for (let [key, type] of Object.entries(vars)) {
+    let configPath = false
+    let val
+    if (Array.isArray(type)) [type, configPath] = type
     if (['required', 'requiredSecret'].includes(type)) {
-      if (typeof process.env[key] === 'undefined' || emptyString(process.env[key])) errors.push(key)
+      if (typeof process.env[key] === 'undefined' || emptyString(process.env[key])) {
+        // Allow falling back to defaults for required config
+        if (configPath) val = get(config, configPath)
+        if (typeof val === 'undefined') errors.push(key)
+      }
       if (type === 'requiredSecret')
         ok.push(`🔒 ${chalk.yellow(key)}: ` + chalk.grey('***redacted***'))
-      else ok.push(`✅ ${chalk.green(key)}: ${chalk.grey(process.env[key])}`)
+      else ok.push(`✅ ${chalk.green(key)}: ${chalk.grey(val)}`)
     } else {
       if (typeof process.env[key] !== 'undefined' && !emptyString(process.env[key])) {
         ok.push(`✅ ${chalk.green(key)}: ${chalk.grey(process.env[key])}`)
@@ -284,32 +303,54 @@ export function verifyConfig(silent = false) {
   }
 
   if (envToBool(process.env.BACKEND_ENABLE_DUMP_CONFIG_AT_STARTUP)) {
-    console.log(
+    const dump = {
-      chalk.cyan.bold('Dumping configuration:\n'),
+      ...config,
-      asJson(
+      encryption: {
-        {
+        ...config.encryption,
-          ...config,
+        key: config.encryption.key.slice(0, 4) + '**redacted**' + config.encryption.key.slice(-4),
-          encryption: {
+      },
-            ...config.encryption,
+      jwt: {
-            key:
+        secretOrKey:
-              config.encryption.key.slice(0, 4) + '**redacted**' + config.encryption.key.slice(-4),
+          config.jwt.secretOrKey.slice(0, 4) + '**redacted**' + config.jwt.secretOrKey.slice(-4),
-          },
+      },
-          jwt: {
+    }
-            secretOrKey:
+    if (config.sanity)
-              config.jwt.secretOrKey.slice(0, 4) +
+      dump.sanity = {
-              '**redacted**' +
+        ...config.sanity,
-              config.jwt.secretOrKey.slice(-4),
+        token: config.sanity.token.slice(0, 4) + '**redacted**' + config.sanity.token.slice(-4),
-          },
+      }
-          sanity: {
+    console.log(chalk.cyan.bold('Dumping configuration:\n'), asJson(dump, null, 2))
-            ...config.sanity,
-            token: config.sanity.token.slice(0, 4) + '**redacted**' + config.sanity.token.slice(-4),
-          },
-        },
-        null,
-        2
-      )
-    )
   }
 
   return config
 }
+
+/*
+ * Generates a random key
+ *
+ * This is a convenience method, typically used in a scenario where people want
+ * to kick the tires by spinning up a Docker container running this backend.
+ * The backend won't start without a valid encryption key. So rather than add
+ * this roadblock to such users, it will auto-generate an encryption key and
+ * write it to disk.
+ */
+function randomEncryptionKey() {
+  const filename = 'encryption.key'
+  console.log(chalk.yellow('⚠️  No encryption key provided'))
+  let key = false
+  try {
+    console.log(chalk.dim('Checking for prior auto-generated encryption key'))
+    key = readFileSync(filename, 'utf-8')
+  } catch (err) {
+    console.log(chalk.dim('No prior auto-generated encryption key found.'))
+  }
+  if (key) {
+    console.log(chalk.green('✅ Prior encryption key found.'))
+  } else {
+    console.log(chalk.green('✅ Generating new random encryption key'))
+    key = randomString(64)
+    writeFileSync(filename, key)
+  }
+
+  return key
+}

sites/backend/src/utils/crypto.mjs

@@ -6,7 +6,9 @@ import { asJson } from './index.mjs'
 /*
  * Hashes an email address (or other string)
  */
-export const hash = (string) => createHash('sha256').update(string).digest('hex')
+export function hash(string) {
+  return createHash('sha256').update(string).digest('hex')
+}
 
 /*
  * Generates a random string
@@ -14,7 +16,9 @@ export const hash = (string) => createHash('sha256').update(string).digest('hex'
  * This is not used in anything cryptographic. It is only used as a temporary
  * username to avoid username collisions or to generate (long) API key secrets
  */
-export const randomString = (bytes = 8) => randomBytes(bytes).toString('hex')
+export function randomString(bytes = 8) {
+  return randomBytes(bytes).toString('hex')
+}
 
 /*
  * Returns an object holding encrypt() and decrypt() methods
@@ -23,7 +27,7 @@ export const randomString = (bytes = 8) => randomBytes(bytes).toString('hex')
  * which makes things easier to read/understand for contributors, as well
  * as allowing scrutiny of the implementation in a single file.
  */
-export const encryption = (stringKey, salt = 'FreeSewing') => {
+export function encryption(stringKey, salt = 'FreeSewing') {
   // Shout-out to the OG crypto bros Joan and Vincent
   const algorithm = 'aes-256-cbc'