yarmo.eu/server/util.js

const fs = require('fs')
const { readdir } = require('fs').promises
const path = require('path')
const { resolve } = require('path')
const bent = require('bent')
const getJSON = bent('json')
const _ = require('lodash')
const md = require('markdown-it')({ typographer: true })
const yamlFront = require('yaml-front-matter')
const { DateTime } = require('luxon')
const Feed = require('feed').Feed
const low = require('lowdb')
const FileSync = require('lowdb/adapters/FileSync')

const adapter = new FileSync('db.json')
const db = low(adapter)
db.defaults({ posts: [] })
  .write()

md.use(require("markdown-it-anchor"), {
  "level": 2,
  "permalink": true,
  "permalinkClass": 'header-anchor',
  "permalinkSymbol": '¶',
  "permalinkBefore": false
})

async function* getFiles(dir) {
  const dirents = await readdir(dir, { withFileTypes: true })
  for (const dirent of dirents) {
    const res = resolve(dir, dirent.name)
    if (dirent.isDirectory()) {
      yield* getFiles(res)
    } else {
      yield res
    }
  }
}

const getBlogPosts = async (opts) => {
  if (!opts) { opts = {} }
  opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true

  return db._.reverse(db.get('posts')
                         .filter({ type: 'blog', published: opts.publishedOnly })
                         .sortBy('date')
                         .value())
}
const getBlogPostsFromFiles = async (opts) => {
  if (!opts) { opts = {} }
  opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true

  let data = []
  for await (const f of getFiles(path.join(__dirname, '../', 'content', 'blog'))) {
    const rawContent = fs.readFileSync(f, 'utf8')
    const fm = yamlFront.loadFront(rawContent)

    if (opts.publishedOnly && !fm.published) { continue }

    data.push({
      type: 'blog',
      title: fm.title,
      author: fm.author,
      urlrel: `/post/${fm.slug}`,
      url: `https://yarmo.eu/post/${fm.slug}`,
      slug: fm.slug,
      date: fm.date,
      date_formatted: DateTime.fromFormat(fm.date, 'yyyy-LL-dd hh:mm:ss').setLocale("en").toLocaleString(DateTime.DATE_MED),
      published: fm.published,
      discussion: fm.discussion,
      content: md.render(fm.__content)
    })
  }
  return data.reverse()
}

const getNotes = async (opts) => {
  if (!opts) { opts = {} }
  opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true

  return db._.reverse(db.get('posts')
                         .filter({ type: 'note', published: opts.publishedOnly })
                         .sortBy('date')
                         .value())
}
const getNotesFromFiles = async (opts) => {
  if (!opts) { opts = {} }
  opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true

  let data = []
  for await (const f of getFiles(path.join(__dirname, '../', 'content', 'notes'))) {
    const rawContent = fs.readFileSync(f, 'utf8')
    const fm = yamlFront.loadFront(rawContent)

    if (opts.publishedOnly && !fm.published) { continue }

    data.push({
      type: 'note',
      title: fm.title,
      author: fm.author,
      urlrel: `/post/${fm.slug}`,
      url: `https://yarmo.eu/post/${fm.slug}`,
      slug: fm.slug,
      date: fm.date,
      date_formatted: DateTime.fromFormat(fm.date, 'yyyy-LL-dd hh:mm:ss').setLocale("en").setZone('utc').toLocaleString(DateTime.DATE_MED),
      published: fm.published,
      discussion: fm.discussion,
      content: md.render(fm.__content)
    })
  }
  return data.reverse()
}

const getPost = async (slug) => {
  let post = db.get('posts')
                .find({ slug: slug })
                .value()

  return post
}

const getWebmentions = async (url) => {
  let data
  try {
    const data_1 = await getJSON(`https://webm.yarmo.eu/get?target=${url}`)
    const data_2 = await getJSON(`https://webm.yarmo.eu/get?target=${url.replace('/post/', '/blog/')}`)
    const data_3 = await getJSON(`https://webm.yarmo.eu/get?target=${url.replace('/post/', '/notes/')}`)

      const dataRaw = data_1.concat(data_2).concat(data_3)
      data = _.map(dataRaw, (x) => {
        x.date = DateTime.fromISO(x.created_at).setLocale("en").setZone('utc').toLocaleString(DateTime.DATE_MED),
        x.time = DateTime.fromISO(x.created_at).setLocale("en").setZone('utc').toLocaleString(DateTime.TIME_24_WITH_SHORT_OFFSET)
        return x
      })
  } catch(e) {
    data = []
  }

  return data
}

const getRSS = async (opts) => {
  if (!opts) { opts = {} }
  opts.include = 'include' in opts ? opts.include : ['blog', 'notes']
  opts.channel = 'channel' in opts ? opts.channel : 'all'
  opts.format = 'format' in opts ? opts.format : 'rss'

  const feed = new Feed({
    title: "Yarmo's blog and notes",
    description: "Blog posts and notes feed for yarmo.eu discussing open source, privacy and selfhosted stuff",
    id: "https://yarmo.eu",
    link: "https://yarmo.eu",
    language: "en",
    favicon: "https://yarmo.eu/favicon.png",
    copyright: "All rights reserved 2020 Yarmo Mackenbach",
    updated: new Date(Date.now()),
    feedLinks: {
      rss: `https://yarmo.eu/feed/${opts.channel}`,
      json: `https://yarmo.eu/feed/${opts.channel}.json`,
      atom: `https://yarmo.eu/feed/${opts.channel}.atom`
    },
    author: {
      name: "Yarmo Mackenbach",
      email: "yarmo@yarmo.eu",
      link: "https://yarmo.eu"
    }
  })

  feed.addCategory("Technology")
  feed.addCategory("FOSS")
  feed.addCategory("Decentralization")
  feed.addCategory("Privacy")
  feed.addCategory("Identity")
  feed.addCategory("Cryptography")
  feed.addCategory("Selfhosted")

  let posts = []
  if (opts.include.includes('blog')) {
    posts = posts.concat(await getBlogPosts())
  }
  if (opts.include.includes('notes')) {
    posts = posts.concat(await getNotes())
  }

  posts = _.sortBy(posts, ['date']).reverse()

  posts.forEach((item, i) => {
    feed.addItem({
      title: item.title,
      id: item.url,
      link: item.url,
      description: item.content,
      content: item.content,
      author: [
        {
          name: "Yarmo Mackenbach",
          email: "yarmo@yarmo.eu",
          link: "https://yarmo.eu"
        }
      ],
      date: new Date(item.date)
    })
  })

  let response
  switch (opts.format) {
    case 'rss':
    case 'xml':
    default:
      response = feed.rss2()
      break;
    case 'atom':
      response = feed.atom1()
      break;
    case 'json':
      response = feed.json1()
      break;
  }

  return response
}

const updateDB = async () => {
  db.read()

  db.set('posts', [])
    .write()

  let data
  data = await getBlogPostsFromFiles({ publishedOnly: false })
  data = data.concat(await getNotesFromFiles({ publishedOnly: false }))

  data.forEach((item, i) => {
    if (db.get('posts').find({ slug: item.slug }).size().value() == 0) {
      db.get('posts')
        .push(item)
        .write()
    } else {
      db.get('posts')
        .find({ slug: item.slug })
        .assign(item)
        .write()
    }
  })
}

module.exports = {
  getFiles: getFiles,
  getBlogPosts: getBlogPosts,
  getBlogPostsFromFiles: getBlogPostsFromFiles,
  getNotes: getNotes,
  getNotesFromFiles: getNotesFromFiles,
  getPost: getPost,
  getWebmentions: getWebmentions,
  getRSS: getRSS,
  updateDB: updateDB
}