Browse Source

Add db caching for posts

main
Yarmo Mackenbach 1 month ago
parent
commit
8d5fc5c20e
7 changed files with 102 additions and 15 deletions
  1. +2
    -1
      .gitignore
  2. +3
    -0
      index.js
  3. +1
    -1
      nodemon.json
  4. +26
    -2
      package-lock.json
  5. +1
    -0
      package.json
  6. +5
    -0
      routes/main.js
  7. +64
    -11
      server/util.js

+ 2
- 1
.gitignore View File

@ -1,2 +1,3 @@
.well-known
node_modules
node_modules
db.json

+ 3
- 0
index.js View File

@ -1,6 +1,7 @@
const express = require('express')
const app = express()
const fs = require('fs')
const util = require('./server/util')
require('dotenv').config()
app.set('env', process.env.NODE_ENV || "production")
@ -12,6 +13,8 @@ app.use('/', require('./routes/static'))
app.use('/feed', require('./routes/feed'))
app.use('/rss', require('./routes/feed'))
util.updateDB()
app.listen(app.get('port'), () => {
console.log(`Node server listening at http://localhost:${app.get('port')}`)
})

+ 1
- 1
nodemon.json View File

@ -1,5 +1,5 @@
{
"ignore": [],
"ignore": ["db.json"],
"env": {
"NODE_ENV": "development"
}

+ 26
- 2
package-lock.json View File

@ -679,8 +679,7 @@
"graceful-fs": {
"version": "4.2.4",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz",
"integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==",
"dev": true
"integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw=="
},
"has-flag": {
"version": "3.0.0",
@ -939,6 +938,18 @@
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
},
"lowdb": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/lowdb/-/lowdb-1.0.0.tgz",
"integrity": "sha512-2+x8esE/Wb9SQ1F9IHaYWfsC9FIecLOPrK4g17FGEayjUWH172H6nwicRovGvSE2CPZouc2MCIqCI7h9d+GftQ==",
"requires": {
"graceful-fs": "^4.1.3",
"is-promise": "^2.1.0",
"lodash": "4",
"pify": "^3.0.0",
"steno": "^0.4.1"
}
},
"lowercase-keys": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
@ -1188,6 +1199,11 @@
"integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
"dev": true
},
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
},
"prepend-http": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz",
@ -1522,6 +1538,14 @@
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
},
"steno": {
"version": "0.4.4",
"resolved": "https://registry.npmjs.org/steno/-/steno-0.4.4.tgz",
"integrity": "sha1-BxEFvfwobmYVwEA8J+nXtdy4Vcs=",
"requires": {
"graceful-fs": "^4.1.3"
}
},
"string-width": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",

+ 1
- 0
package.json View File

@ -22,6 +22,7 @@
"feed": "^4.2.1",
"jstransformer-markdown-it": "^2.1.0",
"lodash": "^4.17.20",
"lowdb": "^1.0.0",
"luxon": "^1.25.0",
"markdown-it-anchor": "^5.3.0",
"markdown-it-table-of-contents": "^0.4.4",

+ 5
- 0
routes/main.js View File

@ -55,4 +55,9 @@ router.get('/pgp', (req, res) => {
res.render('pgp', { title: 'PGP — yarmo.eu' })
})
// router.get('/updateDB', async (req, res) => {
// const response = await util.updateDB()
// res.send(response)
// })
module.exports = router

+ 64
- 11
server/util.js View File

@ -9,6 +9,13 @@ const md = require('markdown-it')({ typographer: true })
const yamlFront = require('yaml-front-matter')
const { DateTime } = require('luxon')
const Feed = require('feed').Feed
const low = require('lowdb')
const FileSync = require('lowdb/adapters/FileSync')
const adapter = new FileSync('db.json')
const db = low(adapter)
db.defaults({ posts: [] })
.write()
md.use(require("markdown-it-anchor"), {
"level": 2,
@ -34,6 +41,15 @@ const getBlogPosts = async (opts) => {
if (!opts) { opts = {} }
opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true
return db._.reverse(db.get('posts')
.filter({ type: 'blog', published: opts.publishedOnly })
.sortBy('date')
.value())
}
const getBlogPostsFromFiles = async (opts) => {
if (!opts) { opts = {} }
opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true
let data = []
for await (const f of getFiles(path.join(__dirname, '../', 'content', 'blog'))) {
const rawContent = fs.readFileSync(f, 'utf8')
@ -62,6 +78,15 @@ const getNotes = async (opts) => {
if (!opts) { opts = {} }
opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true
return db._.reverse(db.get('posts')
.filter({ type: 'blog', published: opts.publishedOnly })
.sortBy('date')
.value())
}
const getNotesFromFiles = async (opts) => {
if (!opts) { opts = {} }
opts.publishedOnly = 'publishedOnly' in opts ? opts.publishedOnly : true
let data = []
for await (const f of getFiles(path.join(__dirname, '../', 'content', 'notes'))) {
const rawContent = fs.readFileSync(f, 'utf8')
@ -87,15 +112,19 @@ const getNotes = async (opts) => {
}
const getPost = async (slug) => {
let post = null, posts = await getBlogPosts()
posts = _.filter(posts, (p) => { return slug == p.slug })
post = posts.length > 0 ? posts[0] : null
if (!post) {
posts = await getNotes()
posts = _.filter(posts, (p) => { return slug == p.slug })
post = posts.length > 0 ? posts[0] : null
}
// let post = null, posts = await getBlogPosts()
// posts = _.filter(posts, (p) => { return slug == p.slug })
// post = posts.length > 0 ? posts[0] : null
//
// if (!post) {
// posts = await getNotes()
// posts = _.filter(posts, (p) => { return slug == p.slug })
// post = posts.length > 0 ? posts[0] : null
// }
let post = db.get('posts')
.find({ slug: slug })
.value()
return post
}
@ -111,7 +140,7 @@ const getWebmentions = async (url) => {
x.time = DateTime.fromISO(x.created_at).setLocale("en").setZone('utc').toLocaleString(DateTime.TIME_24_WITH_SHORT_OFFSET)
return x
})
return data
}
@ -196,11 +225,35 @@ const getRSS = async (opts) => {
return response
}
const updateDB = async () => {
db.read()
let data
data = await getBlogPostsFromFiles({ publishedOnly: false })
data = data.concat(await getNotesFromFiles({ publishedOnly: false }))
data.forEach((item, i) => {
if (db.get('posts').find({ slug: item.slug }).size().value() == 0) {
db.get('posts')
.push(item)
.write()
} else {
db.get('posts')
.find({ slug: item.slug })
.assign(item)
.write()
}
})
}
module.exports = {
getFiles: getFiles,
getBlogPosts: getBlogPosts,
getBlogPostsFromFiles: getBlogPostsFromFiles,
getNotes: getNotes,
getNotesFromFiles: getNotesFromFiles,
getPost: getPost,
getWebmentions: getWebmentions,
getRSS: getRSS
getRSS: getRSS,
updateDB: updateDB
}

Loading…
Cancel
Save