21ee1c4a36
When we don't care about the language, use grep directly rather than find first in prebuild script. This is to sidestep an issue where the vercel build fails because the find+grep combo fails (even though it does find the results).
344 lines
10 KiB
JavaScript
344 lines
10 KiB
JavaScript
import fs from 'node:fs'
|
|
import path from 'node:path'
|
|
import { exec } from 'node:child_process'
|
|
import orderBy from 'lodash.orderby'
|
|
import { loadUserInfo } from './users.mjs'
|
|
|
|
/*
|
|
* Shared header to include in written .mjs files
|
|
*/
|
|
export const header = `/*
|
|
* This file was auto-generated by the prebuild script
|
|
* Any changes you make to it will be lost on the next (pre)build.
|
|
*/
|
|
`
|
|
|
|
/*
|
|
* Strips quptes from the start/end of a string
|
|
*/
|
|
const stripQuotes = (str) => {
|
|
str = str.trim()
|
|
if (str.slice(0, 1) === '"') str = str.slice(1)
|
|
if (str.slice(-1) === '"') str = str.slice(0, -1)
|
|
|
|
return str.trim()
|
|
}
|
|
|
|
/*
|
|
* This is the fast and low-tech way to some frontmatter from all files in a folder
|
|
*/
|
|
const loadFolderFrontmatter = async (key, site, folder, transform = false, lang = false) => {
|
|
const prefix = site === 'org' ? `${folder}/` : ''
|
|
/*
|
|
* Figure out what directory to spawn the child process in
|
|
*/
|
|
const cwd = await path.resolve(process.cwd(), '..', '..', 'markdown', site, folder)
|
|
/*
|
|
* When going through a small number of files in a flat directory (eg. blog posts) a
|
|
* recursive grep through all files is faster.
|
|
* But the biggest task is combing through all the org documentation and for this
|
|
* it's much faster to first run find to limit the number of files to open
|
|
*/
|
|
//const cmd = `find . -type f -name "${lang ? lang : '*'}.md" -exec grep "^${key}:" -ism 1 {} +`
|
|
const cmd = lang
|
|
? `find . -type f -name "${lang ? lang : '*'}.md" -exec grep "^${key}:" -ism 1 {} +`
|
|
: `grep -R "^${key}:" -ism 1 .`
|
|
const grep = exec(cmd, { cwd, maxBuffer: 2048 * 1024 }, (error, stdout, stderr) => {
|
|
if (error) console.error('Exec error:', { cwd, cmd, error, stderr })
|
|
|
|
return stdout
|
|
})
|
|
|
|
/*
|
|
* Stdout is buffered, so we need to gather all of it
|
|
*/
|
|
let stdout = ''
|
|
for await (const data of grep.stdout) stdout += data
|
|
|
|
/*
|
|
* Turn all matches into an array
|
|
*/
|
|
const matches = stdout.split('\n')
|
|
|
|
/*
|
|
* Turns matches into structured data
|
|
*/
|
|
const pages = {}
|
|
for (let match of matches) {
|
|
/*
|
|
* Trim some of the irrelevant path info prior to splitting on '.md:{key}:'
|
|
*/
|
|
const chunks = match
|
|
.split(`markdown/${site}/${site === 'dev' ? '' : folder + '/'}`)
|
|
.pop()
|
|
.split(`.md:${key}:`)
|
|
|
|
if (chunks.length === 2 && chunks[0].length > 1) {
|
|
/*
|
|
* Figure out the language and make sure we have an key for that language
|
|
*/
|
|
const lang = chunks[0].slice(-2)
|
|
if (!pages[lang]) pages[lang] = {}
|
|
|
|
/*
|
|
* Add page to our object with slug as key and title as value
|
|
*/
|
|
let slug = prefix + chunks[0].slice(0, -3).replace('./', '')
|
|
if (slug === prefix) slug = slug.slice(0, -1)
|
|
if (slug !== 'docs/.')
|
|
pages[lang][slug] = transform
|
|
? transform(stripQuotes(chunks[1]), slug, lang)
|
|
: stripQuotes(chunks[1])
|
|
}
|
|
}
|
|
|
|
return pages
|
|
}
|
|
|
|
/*
|
|
* Merges in order key on those slugs that have it set
|
|
*/
|
|
const mergeOrder = (titles, order, withSlug = false) => {
|
|
const pages = {}
|
|
for (const lang in titles) {
|
|
pages[lang] = {}
|
|
for (const [slug, t] of Object.entries(titles[lang])) {
|
|
pages[lang][slug] = { t }
|
|
if (order.en[slug]) pages[lang][slug].o = order.en[slug]
|
|
if (withSlug) pages[lang][slug].s = slug
|
|
}
|
|
}
|
|
|
|
return pages
|
|
}
|
|
|
|
/*
|
|
* Fixes the date format to be yyyymmdd
|
|
*/
|
|
const formatDate = (date, slug, lang) => {
|
|
date = date.split('-')
|
|
if (date.length === 1) date = date[0].split('.')
|
|
if (date.length === 1) {
|
|
if (date[0].length === 8) return date[0]
|
|
else console.log(`Could not format date ${date} from ${slug} (${lang})`)
|
|
} else {
|
|
if (date[0].length === 4) return date.join('')
|
|
else return date.reverse().join('')
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Loads all new users and adds them to the store
|
|
*/
|
|
const loadUsers = async (list, store) => {
|
|
// Weed out doubles in list
|
|
for (const user of [...new Set([...list])]) {
|
|
const id = Number(user)
|
|
if (id && typeof store.users[id] === 'undefined') store.users[id] = await loadUserInfo(id)
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Loads all docs files, titles and order
|
|
*/
|
|
const loadDocs = async (site) => {
|
|
const folder = site === 'org' ? 'docs' : ''
|
|
const titles = await loadFolderFrontmatter('title', site, folder)
|
|
// Order is the same for all languages, so only grab EN files
|
|
const order = await loadFolderFrontmatter('order', site, folder, false, 'en')
|
|
|
|
return mergeOrder(titles, order)
|
|
}
|
|
|
|
/*
|
|
* Loads jargon and terms
|
|
*/
|
|
const loadJargon = async (site, docs) => {
|
|
const folder = site === 'org' ? 'docs' : ''
|
|
const jargon = await loadFolderFrontmatter('jargon', site, folder)
|
|
const terms = await loadFolderFrontmatter('terms', site, folder)
|
|
|
|
const data = {}
|
|
for (const lang in jargon) {
|
|
data[lang] = {}
|
|
for (const slug in jargon[lang]) {
|
|
data[lang][docs[lang][slug].t.toLowerCase()] = slug
|
|
if (terms[lang]?.[slug]) {
|
|
for (const term of terms[lang][slug].split(',').map((term) => term.trim())) {
|
|
data[lang][term.toLowerCase()] = slug
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return data
|
|
}
|
|
|
|
/*
|
|
* Loads all blog posts, titles and order
|
|
*/
|
|
const loadBlog = async (store) => {
|
|
const titles = await loadFolderFrontmatter('title', 'org', 'blog')
|
|
// Order is the same for all languages, so only grab EN files
|
|
const order = await loadFolderFrontmatter('date', 'org', 'blog', formatDate, 'en')
|
|
// Author is the same for all languages, so only grab EN files
|
|
const authors = await loadFolderFrontmatter('author', 'org', 'blog', false, 'en')
|
|
// Load user accounts of authors
|
|
await loadUsers(Object.values(authors.en), store)
|
|
|
|
// Merge titles and order for EN
|
|
const merged = {}
|
|
for (const slug in titles.en)
|
|
merged[slug] = {
|
|
t: titles.en[slug],
|
|
o: order.en[slug],
|
|
s: slug,
|
|
a: authors.en[slug],
|
|
}
|
|
// Order based on post data (descending)
|
|
const ordered = orderBy(merged, 'o', 'desc')
|
|
|
|
// Apply same order to all languages
|
|
const posts = {}
|
|
const meta = {}
|
|
|
|
for (const lang of Object.keys(titles)) {
|
|
posts[lang] = {}
|
|
for (const post of ordered) {
|
|
const sortkey = 9999999999 - post.o
|
|
posts[lang][post.s] = { t: post.t, o: sortkey }
|
|
if (lang === 'en') meta[post.s] = { a: post.a, d: post.o }
|
|
}
|
|
}
|
|
|
|
return { posts, meta }
|
|
}
|
|
|
|
/*
|
|
* Loads all showcase posts, titles, designs and order
|
|
*/
|
|
const loadShowcase = async (store) => {
|
|
const titles = await loadFolderFrontmatter('title', 'org', 'showcase')
|
|
// Order is the same for all languages, so only grab EN files
|
|
const order = await loadFolderFrontmatter('date', 'org', 'showcase', formatDate, 'en')
|
|
// Author is the same for all languages, so only grab EN files
|
|
const authors = await loadFolderFrontmatter('author', 'org', 'showcase', false, 'en')
|
|
// Load user accounts of authors
|
|
await loadUsers(Object.values(authors.en), store)
|
|
|
|
// Merge titles and order for EN
|
|
const merged = {}
|
|
for (const slug in titles.en)
|
|
merged[slug] = {
|
|
t: titles.en[slug],
|
|
o: order.en[slug],
|
|
s: slug,
|
|
m: authors.en[slug],
|
|
}
|
|
// Order based on post data (descending)
|
|
const ordered = orderBy(merged, 'o', 'desc')
|
|
|
|
// Apply same order to all languages
|
|
const posts = {}
|
|
const meta = {}
|
|
|
|
for (const lang of Object.keys(titles)) {
|
|
posts[lang] = {}
|
|
for (const post of ordered) {
|
|
posts[lang][post.s] = { t: post.t }
|
|
if (lang === 'en') meta[post.s] = { m: post.m, d: post.o }
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Create list of showcase slugs per design
|
|
*/
|
|
const designShowcases = {}
|
|
// Designs is the same for all languages, so only grab EN files
|
|
const designs = await loadFolderFrontmatter('designs', 'org', 'showcase', false, 'en')
|
|
for (const [slug, list] of Object.entries(designs.en)) {
|
|
for (const design of JSON.parse(list)) {
|
|
if (typeof designShowcases[design] === 'undefined') designShowcases[design] = []
|
|
designShowcases[design].push(slug.split('/').pop())
|
|
}
|
|
}
|
|
|
|
return { posts, meta, designShowcases }
|
|
}
|
|
|
|
/*
|
|
* Loads all newsletter posts, titles and order
|
|
*/
|
|
const loadNewsletter = async () => {
|
|
const titles = await loadFolderFrontmatter('title', 'org', 'newsletter')
|
|
// Order is the same for all languages, so only grab EN files
|
|
const order = await loadFolderFrontmatter('edition', 'org', 'newsletter', false, 'en')
|
|
|
|
return mergeOrder(titles, order)
|
|
}
|
|
|
|
/*
|
|
* Write out prebuild files
|
|
*/
|
|
const writeFiles = async (type, site, pages) => {
|
|
let allPaths = ``
|
|
for (const lang in pages) {
|
|
fs.writeFileSync(
|
|
path.resolve('..', site, 'prebuild', `${type}.${lang}.mjs`),
|
|
`${header}export const pages = ${JSON.stringify(pages[lang])}`
|
|
)
|
|
allPaths += `import { pages as ${lang} } from './${type}.${lang}.mjs'` + '\n'
|
|
}
|
|
// Write umbrella file
|
|
fs.writeFileSync(
|
|
path.resolve('..', site, 'prebuild', `${type}.mjs`),
|
|
`${allPaths}${header}
|
|
|
|
export const pages = { ${Object.keys(pages).join(',')} }`
|
|
)
|
|
}
|
|
|
|
/*
|
|
* Write out a single prebuild file
|
|
*/
|
|
const writeFile = async (filename, exportname, site, content) => {
|
|
fs.writeFileSync(
|
|
path.resolve('..', site, 'prebuild', `${filename}.mjs`),
|
|
`${header}export const ${exportname} = ${JSON.stringify(content)}`
|
|
)
|
|
}
|
|
|
|
/*
|
|
* Main method that does what needs doing for the docs
|
|
*/
|
|
export const prebuildDocs = async (store) => {
|
|
store.docs = await loadDocs(store.site)
|
|
await writeFiles('docs', store.site, store.docs)
|
|
|
|
// Handle jargon
|
|
store.jargon = await loadJargon(store.site, store.docs)
|
|
fs.writeFileSync(
|
|
path.resolve('..', store.site, 'prebuild', `jargon.mjs`),
|
|
`${header}
|
|
export const site = "${store.site}"
|
|
export const jargon = ${JSON.stringify(store.jargon, null, 2)}`
|
|
)
|
|
}
|
|
|
|
/*
|
|
* Main method that does what needs doing for the blog/showcase/newsletter posts
|
|
*/
|
|
export const prebuildPosts = async (store) => {
|
|
store.posts = {
|
|
blog: await loadBlog(store),
|
|
showcase: await loadShowcase(store),
|
|
newsletter: { posts: await loadNewsletter(store) },
|
|
}
|
|
await writeFiles('blog', 'org', store.posts.blog.posts)
|
|
await writeFiles('showcase', 'org', store.posts.showcase.posts)
|
|
await writeFiles('newsletter', 'org', store.posts.newsletter)
|
|
await writeFile('blog-meta', 'meta', 'org', store.posts.blog.meta)
|
|
await writeFile('showcase-meta', 'meta', 'org', store.posts.showcase.meta)
|
|
await writeFile('design-examples', 'examples', 'org', store.posts.showcase.designShowcases)
|
|
await writeFile('authors', 'authors', 'org', store.users)
|
|
}
|