Add db caching for posts

.gitignore

@@ -1,2 +1,3 @@
 .well-known
 node_modules
+db.json

index.js

@@ -1,6 +1,7 @@
 const express = require('express')
 const app = express()
 const fs = require('fs')
+const util = require('./server/util')
 require('dotenv').config()
 
 app.set('env', process.env.NODE_ENV || "production")
@@ -12,6 +13,8 @@ app.use('/', require('./routes/static'))
 app.use('/feed', require('./routes/feed'))
 app.use('/rss', require('./routes/feed'))
 
+util.updateDB()
+
 app.listen(app.get('port'), () => {
     console.log(`Node server listening at http://localhost:${app.get('port')}`)
 })

nodemon.json

@@ -1,5 +1,5 @@
 {
-  "ignore": [],
+  "ignore": ["db.json"],
   "env": {
     "NODE_ENV": "development"
   }

package-lock.json

@@ -679,8 +679,7 @@
     "graceful-fs": {
       "version": "4.2.4",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz",
-      "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==",
-      "dev": true
+      "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw=="
     },
     "has-flag": {
       "version": "3.0.0",
@@ -939,6 +938,18 @@
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
       "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
     },
+    "lowdb": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-1.0.0.tgz",
+      "integrity": "sha512-2+x8esE/Wb9SQ1F9IHaYWfsC9FIecLOPrK4g17FGEayjUWH172H6nwicRovGvSE2CPZouc2MCIqCI7h9d+GftQ==",
+      "requires": {
+        "graceful-fs": "^4.1.3",
+        "is-promise": "^2.1.0",
+        "lodash": "4",
+        "pify": "^3.0.0",
+        "steno": "^0.4.1"
+      }
+    },
     "lowercase-keys": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
@@ -1188,6 +1199,11 @@
       "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
       "dev": true
     },
+    "pify": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
+      "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
+    },
     "prepend-http": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz",
@@ -1522,6 +1538,14 @@
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
       "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
     },
+    "steno": {
+      "version": "0.4.4",
+      "resolved": "https://registry.npmjs.org/steno/-/steno-0.4.4.tgz",
+      "integrity": "sha1-BxEFvfwobmYVwEA8J+nXtdy4Vcs=",
+      "requires": {
+        "graceful-fs": "^4.1.3"
+      }
+    },
     "string-width": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",

package.json

@@ -22,6 +22,7 @@
     "feed": "^4.2.1",
     "jstransformer-markdown-it": "^2.1.0",
     "lodash": "^4.17.20",
+    "lowdb": "^1.0.0",
     "luxon": "^1.25.0",
     "markdown-it-anchor": "^5.3.0",
     "markdown-it-table-of-contents": "^0.4.4",

routes/main.js

@@ -55,4 +55,9 @@ router.get('/pgp', (req, res) => {
   res.render('pgp', { title: 'PGP — yarmo.eu' })
 })
 
+// router.get('/updateDB', async (req, res) => {
+//   const response = await util.updateDB()
+//   res.send(response)
+// })
+
 module.exports = router

server/util.js

@@ -9,6 +9,13 @@ const md = require('markdown-it')({ typographer: true })
 const yamlFront = require('yaml-front-matter')
 const { DateTime } = require('luxon')
 const Feed = require('feed').Feed
+const low = require('lowdb')
+const FileSync = require('lowdb/adapters/FileSync')
+
+const adapter = new FileSync('db.json')
+const db = low(adapter)
+db.defaults({ posts: [] })
+  .write()
 
 md.use(require("markdown-it-anchor"), {
   "level": 2,
@@ -34,6 +41,15 @@ const getBlogPosts = async (opts) => {
   if (!opts) { opts = {} }
   opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true
 
+  return db._.reverse(db.get('posts')
+                         .filter({ type: 'blog', published: opts.publishedOnly })
+                         .sortBy('date')
+                         .value())
+}
+const getBlogPostsFromFiles = async (opts) => {
+  if (!opts) { opts = {} }
+  opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true
+
   let data = []
   for await (const f of getFiles(path.join(__dirname, '../', 'content', 'blog'))) {
     const rawContent = fs.readFileSync(f, 'utf8')
@@ -62,6 +78,15 @@ const getNotes = async (opts) => {
   if (!opts) { opts = {} }
   opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true
 
+  return db._.reverse(db.get('posts')
+                         .filter({ type: 'blog', published: opts.publishedOnly })
+                         .sortBy('date')
+                         .value())
+}
+const getNotesFromFiles = async (opts) => {
+  if (!opts) { opts = {} }
+  opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true
+
   let data = []
   for await (const f of getFiles(path.join(__dirname, '../', 'content', 'notes'))) {
     const rawContent = fs.readFileSync(f, 'utf8')
@@ -87,15 +112,19 @@ const getNotes = async (opts) => {
 }
 
 const getPost = async (slug) => {
-  let post = null, posts = await getBlogPosts()
-  posts = _.filter(posts, (p) => { return slug == p.slug })
-  post = posts.length > 0 ? posts[0] : null
+  // let post = null, posts = await getBlogPosts()
+  // posts = _.filter(posts, (p) => { return slug == p.slug })
+  // post = posts.length > 0 ? posts[0] : null
+  //
+  // if (!post) {
+  //   posts = await getNotes()
+  //   posts = _.filter(posts, (p) => { return slug == p.slug })
+  //   post = posts.length > 0 ? posts[0] : null
+  // }
 
-  if (!post) {
-    posts = await getNotes()
-    posts = _.filter(posts, (p) => { return slug == p.slug })
-    post = posts.length > 0 ? posts[0] : null
-  }
+  let post = db.get('posts')
+                .find({ slug: slug })
+                .value()
 
   return post
 }
@@ -196,11 +225,35 @@ const getRSS = async (opts) => {
   return response
 }
 
+const updateDB = async () => {
+  db.read()
+
+  let data
+  data = await getBlogPostsFromFiles({ publishedOnly: false })
+  data = data.concat(await getNotesFromFiles({ publishedOnly: false }))
+
+  data.forEach((item, i) => {
+    if (db.get('posts').find({ slug: item.slug }).size().value() == 0) {
+      db.get('posts')
+        .push(item)
+        .write()
+    } else {
+      db.get('posts')
+        .find({ slug: item.slug })
+        .assign(item)
+        .write()
+    }
+  })
+}
+
 module.exports = {
   getFiles: getFiles,
   getBlogPosts: getBlogPosts,
+  getBlogPostsFromFiles: getBlogPostsFromFiles,
   getNotes: getNotes,
+  getNotesFromFiles: getNotesFromFiles,
   getPost: getPost,
   getWebmentions: getWebmentions,
-  getRSS: getRSS
+  getRSS: getRSS,
+  updateDB: updateDB
 }