const fs = require('fs') const { readdir } = require('fs').promises const path = require('path') const { resolve } = require('path') const bent = require('bent') const getJSON = bent('json') const _ = require('lodash') const md = require('markdown-it')({ typographer: true }) const yamlFront = require('yaml-front-matter') const { DateTime } = require('luxon') const Feed = require('feed').Feed const low = require('lowdb') const FileSync = require('lowdb/adapters/FileSync') const adapter = new FileSync('db.json') const db = low(adapter) db.defaults({ posts: [] }) .write() md.use(require("markdown-it-anchor"), { "level": 2, "permalink": true, "permalinkClass": 'header-anchor', "permalinkSymbol": 'ΒΆ', "permalinkBefore": false }) async function* getFiles(dir) { const dirents = await readdir(dir, { withFileTypes: true }) for (const dirent of dirents) { const res = resolve(dir, dirent.name) if (dirent.isDirectory()) { yield* getFiles(res) } else { yield res } } } const getBlogPosts = async (opts) => { if (!opts) { opts = {} } opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true return db._.reverse(db.get('posts') .filter({ type: 'blog', published: opts.publishedOnly }) .sortBy('date') .value()) } const getBlogPostsFromFiles = async (opts) => { if (!opts) { opts = {} } opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true let data = [] for await (const f of getFiles(path.join(__dirname, '../', 'content', 'blog'))) { const rawContent = fs.readFileSync(f, 'utf8') const fm = yamlFront.loadFront(rawContent) if (opts.publishedOnly && !fm.published) { continue } data.push({ type: 'blog', title: fm.title, author: fm.author, urlrel: `/post/${fm.slug}`, url: `https://yarmo.eu/post/${fm.slug}`, slug: fm.slug, date: fm.date, date_formatted: DateTime.fromFormat(fm.date, 'yyyy-LL-dd hh:mm:ss').setLocale("en").toLocaleString(DateTime.DATE_MED), published: fm.published, discussion: fm.discussion, content: md.render(fm.__content) }) } return data.reverse() } const getNotes = async (opts) => { if (!opts) { opts = {} } opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true return db._.reverse(db.get('posts') .filter({ type: 'note', published: opts.publishedOnly }) .sortBy('date') .value()) } const getNotesFromFiles = async (opts) => { if (!opts) { opts = {} } opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true let data = [] for await (const f of getFiles(path.join(__dirname, '../', 'content', 'notes'))) { const rawContent = fs.readFileSync(f, 'utf8') const fm = yamlFront.loadFront(rawContent) if (opts.publishedOnly && !fm.published) { continue } data.push({ type: 'note', title: fm.title, author: fm.author, urlrel: `/post/${fm.slug}`, url: `https://yarmo.eu/post/${fm.slug}`, slug: fm.slug, date: fm.date, date_formatted: DateTime.fromFormat(fm.date, 'yyyy-LL-dd hh:mm:ss').setLocale("en").setZone('utc').toLocaleString(DateTime.DATE_MED), published: fm.published, discussion: fm.discussion, content: md.render(fm.__content) }) } return data.reverse() } const getPost = async (slug) => { let post = db.get('posts') .find({ slug: slug }) .value() return post } const getWebmentions = async (url) => { let data try { const data_1 = await getJSON(`https://webm.yarmo.eu/get?target=${url}`) const data_2 = await getJSON(`https://webm.yarmo.eu/get?target=${url.replace('/post/', '/blog/')}`) const data_3 = await getJSON(`https://webm.yarmo.eu/get?target=${url.replace('/post/', '/notes/')}`) const dataRaw = data_1.concat(data_2).concat(data_3) data = _.map(dataRaw, (x) => { x.date = DateTime.fromISO(x.created_at).setLocale("en").setZone('utc').toLocaleString(DateTime.DATE_MED), x.time = DateTime.fromISO(x.created_at).setLocale("en").setZone('utc').toLocaleString(DateTime.TIME_24_WITH_SHORT_OFFSET) return x }) } catch(e) { data = [] } return data } const getRSS = async (opts) => { if (!opts) { opts = {} } opts.include = 'include' in opts ? opts.include : ['blog', 'notes'] opts.channel = 'channel' in opts ? opts.channel : 'all' opts.format = 'format' in opts ? opts.format : 'rss' const feed = new Feed({ title: "Yarmo's blog and notes", description: "Blog posts and notes feed for yarmo.eu discussing open source, privacy and selfhosted stuff", id: "https://yarmo.eu", link: "https://yarmo.eu", language: "en", favicon: "https://yarmo.eu/favicon.png", copyright: "All rights reserved 2020 Yarmo Mackenbach", updated: new Date(Date.now()), feedLinks: { rss: `https://yarmo.eu/feed/${opts.channel}`, json: `https://yarmo.eu/feed/${opts.channel}.json`, atom: `https://yarmo.eu/feed/${opts.channel}.atom` }, author: { name: "Yarmo Mackenbach", email: "yarmo@yarmo.eu", link: "https://yarmo.eu" } }) feed.addCategory("Technology") feed.addCategory("FOSS") feed.addCategory("Decentralization") feed.addCategory("Privacy") feed.addCategory("Identity") feed.addCategory("Cryptography") feed.addCategory("Selfhosted") let posts = [] if (opts.include.includes('blog')) { posts = posts.concat(await getBlogPosts()) } if (opts.include.includes('notes')) { posts = posts.concat(await getNotes()) } posts = _.sortBy(posts, ['date']).reverse() posts.forEach((item, i) => { feed.addItem({ title: item.title, id: item.url, link: item.url, description: item.content, content: item.content, author: [ { name: "Yarmo Mackenbach", email: "yarmo@yarmo.eu", link: "https://yarmo.eu" } ], date: new Date(item.date) }) }) let response switch (opts.format) { case 'rss': case 'xml': default: response = feed.rss2() break; case 'atom': response = feed.atom1() break; case 'json': response = feed.json1() break; } return response } const updateDB = async () => { db.read() db.set('posts', []) .write() let data data = await getBlogPostsFromFiles({ publishedOnly: false }) data = data.concat(await getNotesFromFiles({ publishedOnly: false })) data.forEach((item, i) => { if (db.get('posts').find({ slug: item.slug }).size().value() == 0) { db.get('posts') .push(item) .write() } else { db.get('posts') .find({ slug: item.slug }) .assign(item) .write() } }) } module.exports = { getFiles: getFiles, getBlogPosts: getBlogPosts, getBlogPostsFromFiles: getBlogPostsFromFiles, getNotes: getNotes, getNotesFromFiles: getNotesFromFiles, getPost: getPost, getWebmentions: getWebmentions, getRSS: getRSS, updateDB: updateDB }