forked from nm3clol/nm3clol-express-app
Refactored breadcrumbs feature into modular view component, and moved page route handler into its own router.
This commit is contained in:
parent
3a06bec2b7
commit
76a460abcd
4
app/helpers/breadcrumbs.mts
Normal file
4
app/helpers/breadcrumbs.mts
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
export interface Breadcrumb {
|
||||||
|
title: string;
|
||||||
|
url: string;
|
||||||
|
}
|
|
@ -5,6 +5,7 @@ import { config } from '../config.mjs';
|
||||||
import markdownIt from 'markdown-it';
|
import markdownIt from 'markdown-it';
|
||||||
import markdownItAttrs from 'markdown-it-attrs';
|
import markdownItAttrs from 'markdown-it-attrs';
|
||||||
import momentJs from 'moment-timezone';
|
import momentJs from 'moment-timezone';
|
||||||
|
import { inspect } from 'util';
|
||||||
const moment = momentJs.tz.setDefault("UTC");
|
const moment = momentJs.tz.setDefault("UTC");
|
||||||
const md = markdownIt({
|
const md = markdownIt({
|
||||||
html: true,
|
html: true,
|
||||||
|
@ -189,6 +190,8 @@ export default {
|
||||||
printReadme,
|
printReadme,
|
||||||
stripWebVTT,
|
stripWebVTT,
|
||||||
renderArchive,
|
renderArchive,
|
||||||
|
config,
|
||||||
|
inspect,
|
||||||
md,
|
md,
|
||||||
moment,
|
moment,
|
||||||
};
|
};
|
|
@ -1,9 +0,0 @@
|
||||||
export interface SolrDocument {
|
|
||||||
id: string;
|
|
||||||
sha256sum: string;
|
|
||||||
url: string;
|
|
||||||
content_length: number;
|
|
||||||
content_type: string;
|
|
||||||
text: string;
|
|
||||||
_version_?: number;
|
|
||||||
}
|
|
274
app/page/router.mts
Normal file
274
app/page/router.mts
Normal file
|
@ -0,0 +1,274 @@
|
||||||
|
console.log(`Loading nm3clol-express-app directory router module...`);
|
||||||
|
|
||||||
|
import { config } from '../config.mjs';
|
||||||
|
import express from 'express';
|
||||||
|
import serve from './vercel-serve.mjs';
|
||||||
|
import path from 'path';
|
||||||
|
import { globSync } from 'glob';
|
||||||
|
import matter from 'gray-matter';
|
||||||
|
import ejs from 'ejs';
|
||||||
|
import helpers from '../helpers/functions.mjs';
|
||||||
|
import fs from 'fs';
|
||||||
|
import { readFile } from 'fs/promises';
|
||||||
|
import { Breadcrumb } from '../helpers/breadcrumbs.mjs';
|
||||||
|
|
||||||
|
export default function () {
|
||||||
|
const pageRouter = express.Router();
|
||||||
|
|
||||||
|
// // Serve static files (CSS, JavaScript, images, etc.)
|
||||||
|
// app.use(serve('../public', {
|
||||||
|
// dotfiles: 'ignore',
|
||||||
|
// index: false,
|
||||||
|
// }));
|
||||||
|
|
||||||
|
// app.get('/', (req, res) => {
|
||||||
|
// res.send('Hello World!');
|
||||||
|
// })
|
||||||
|
|
||||||
|
// console.log("Setting route for /ads.txt");
|
||||||
|
// app.get('/ads.txt', (req, res) => {
|
||||||
|
// res.setHeader("Content-Type", "text/plain");
|
||||||
|
// res.setHeader("Cache-Control", "no-cache");
|
||||||
|
// res.send(`google.com, pub-8937572456576531, DIRECT, f08c47fec0942fa0`);
|
||||||
|
// });
|
||||||
|
|
||||||
|
console.log(`Serving /robots.txt from memory.`);
|
||||||
|
pageRouter.get('/robots.txt', (req, res) => {
|
||||||
|
res.setHeader("Content-Type", "text/plain");
|
||||||
|
res.setHeader("Cache-Control", "no-cache");
|
||||||
|
// TODO: Implement Site Map feature and provide sitemap url in robots.txt
|
||||||
|
res.send(
|
||||||
|
`User-agent: *
|
||||||
|
Allow: /
|
||||||
|
|
||||||
|
# TODO: Implement Site Map feature and provide sitemap url in robots.txt
|
||||||
|
#sitemap: https://no-moss-3-carbo-landfill-library.online/sitemap.xml`
|
||||||
|
);//end of res.send() for robots.txt
|
||||||
|
});
|
||||||
|
|
||||||
|
// Endpoints for all the site's pages.
|
||||||
|
console.log(`Scanning for pages in ${config.pagesPath} to create routes.`);
|
||||||
|
globSync('**/*.md', {
|
||||||
|
cwd: config.pagesPath,
|
||||||
|
matchBase: true,
|
||||||
|
follow: true,
|
||||||
|
}).forEach((filePath) => {
|
||||||
|
const expressRoutePathFromFilePath = (filePath: string) => {
|
||||||
|
filePath = filePath.substring(0, filePath.length - path.extname(filePath).length).replaceAll(path.sep, path.posix.sep);
|
||||||
|
if (!filePath.startsWith('/') && filePath.length > 0) {
|
||||||
|
filePath = `/${filePath}`;
|
||||||
|
}
|
||||||
|
return filePath;
|
||||||
|
};
|
||||||
|
const route = expressRoutePathFromFilePath(filePath);
|
||||||
|
const fullFilePath = path.join(config.pagesPath, filePath);
|
||||||
|
let paths = route.split(path.posix.sep);
|
||||||
|
console.log(`Serving ${route} route as a page at ${fullFilePath}.`);
|
||||||
|
pageRouter.get(route, async (req, res) => {
|
||||||
|
const fm = matter.read(fullFilePath);
|
||||||
|
const fmData = { fm: fm.data, excerpt: fm.excerpt };
|
||||||
|
const content = helpers.md.render(fm.content, fmData );
|
||||||
|
let breadcrumbs: Breadcrumb[] = [];
|
||||||
|
paths.forEach((path, index) => {
|
||||||
|
if (index == 0) {
|
||||||
|
breadcrumbs.push({ title: config.siteName, url: '/' });
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
breadcrumbs.push({ title: fmData.fm['title']||path.replaceAll('_', ' ').replaceAll('-', ' '), url: helpers.trimSlashes(helpers.leftTrimFirstDirectory(path)) });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const renderData = { breadcrumbs, content, filePath, fullFilePath, paths, req, route, ...fmData };
|
||||||
|
res.render("page", { h: helpers, ...renderData });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// console.log("Scanning for documents to create routes.");
|
||||||
|
// glob.globSync('**/*{.pdf,.docx,.xlsx,.pptx,.doc,.xls,.ppt}', {
|
||||||
|
// cwd: path.join(config.publicPath),
|
||||||
|
// matchBase: true,
|
||||||
|
// follow: true,
|
||||||
|
// }).forEach((filePath) => {
|
||||||
|
// const expressRoutePathFromFilePath = (filePath) => {
|
||||||
|
// return filePath.substring(0, filePath.length - path.extname(filePath).length).replaceAll(path.sep, path.posix.sep);
|
||||||
|
// };
|
||||||
|
// const route = expressRoutePathFromFilePath(filePath);
|
||||||
|
// const fullFilePath = path.join(config.publicPath, filePath);
|
||||||
|
// let paths = route.split(path.posix.sep);
|
||||||
|
// paths[0] = 'public';
|
||||||
|
// console.log(`Setting route for ${route}`);
|
||||||
|
// app.get(route, async (req, res) => {
|
||||||
|
// const fm = matter.read(fullFilePath);
|
||||||
|
// const fmData = { fm: fm.data, excerpt: fm.excerpt };
|
||||||
|
// const content = helpers.md.render(fm.content, fmData );
|
||||||
|
// const renderData = { content, route, filePath, fullFilePath, req, paths, ...fmData };
|
||||||
|
// res.render("page", { h: helpers, ...renderData });
|
||||||
|
// });
|
||||||
|
// });
|
||||||
|
|
||||||
|
//TODO: Rewrite this facility so that it utilizes Git index as a filesystem.
|
||||||
|
console.log("Scanning for web archive HTML documents to create routes.");
|
||||||
|
globSync('Web_Site_Archives/**/*{.htm,.html}', {
|
||||||
|
cwd: config.publicPath,
|
||||||
|
matchBase: true,
|
||||||
|
follow: true,
|
||||||
|
}).forEach((filePath) => {
|
||||||
|
const expressRoutePathFromFilePath = (filePath: string) => {
|
||||||
|
return '/' + filePath.replaceAll(path.sep, path.posix.sep);
|
||||||
|
};
|
||||||
|
const route = expressRoutePathFromFilePath(filePath);
|
||||||
|
const fullFilePath = path.join(config.publicPath, filePath);
|
||||||
|
let paths = route.split(path.posix.sep);
|
||||||
|
paths[0] = 'public';
|
||||||
|
console.log(`Setting route for ${route}`);
|
||||||
|
pageRouter.get(route, async (req, res) => {
|
||||||
|
const html = fs.readFileSync(fullFilePath).toString();
|
||||||
|
const renderData = { route, filePath, fullFilePath, req, paths, html };
|
||||||
|
res.render("archive", { h: helpers, ...renderData });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
//TODO: Rewrite this facility so that it utilizes Git index as a filesystem.
|
||||||
|
console.log("Scanning for archived videos to create routes.");
|
||||||
|
globSync(['Russell_County/Board_of_Supervisors/YouTube_Archive/**/*.info.json', 'Virginia_Energy/YouTube_Archive/**/*.info.json', 'Virginia_Governor/**/*.info.json'], {
|
||||||
|
cwd: config.publicPath,
|
||||||
|
matchBase: true,
|
||||||
|
follow: true,
|
||||||
|
}).forEach((filePath: string) => {
|
||||||
|
const expressRoutePathFromFilePath = (filePath: string) => {
|
||||||
|
return path.posix.sep+filePath.substring(0, filePath.lastIndexOf(path.sep)).replaceAll(path.sep, path.posix.sep);
|
||||||
|
};
|
||||||
|
const dirFromFilePath = (filePath: string) => {
|
||||||
|
return filePath.substring(0, filePath.lastIndexOf(path.sep));
|
||||||
|
}
|
||||||
|
const directory = dirFromFilePath(filePath);
|
||||||
|
let videoURL = ""+globSync("*.{mpg,mpeg,mp4,mkv,webm}", {
|
||||||
|
cwd: path.join(config.publicPath, directory),
|
||||||
|
matchBase: true,
|
||||||
|
follow: true,
|
||||||
|
}).pop();
|
||||||
|
let subtitleURL = ""+globSync("*.en.vtt", {
|
||||||
|
cwd: path.join(config.publicPath, directory),
|
||||||
|
matchBase: true,
|
||||||
|
follow: true,
|
||||||
|
}).pop();
|
||||||
|
let subtitleFile = path.join(config.publicPath, directory, subtitleURL);
|
||||||
|
const route = encodeURI(expressRoutePathFromFilePath(filePath));
|
||||||
|
let paths = filePath
|
||||||
|
.substring(0, filePath.lastIndexOf(path.sep) > 0 ? filePath.lastIndexOf(path.sep) : filePath.length-1)
|
||||||
|
.split(path.sep)
|
||||||
|
.map((name, idx, aPaths) => {
|
||||||
|
let url = aPaths.slice(0, idx+1).join(path.posix.sep);
|
||||||
|
return {
|
||||||
|
name,
|
||||||
|
url,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const fullFilePath = path.join(config.publicPath, filePath);
|
||||||
|
console.log(`Setting route for ${route}`);
|
||||||
|
pageRouter.get(route, async (req, res) => {
|
||||||
|
if (!req.path.endsWith('/')) {
|
||||||
|
res.redirect(req.path + '/');
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
let info = JSON.parse((await readFile(fullFilePath)).toString());
|
||||||
|
let subtitleVTT = fs.existsSync(subtitleFile) ? (await readFile(subtitleFile)).toString() : '';
|
||||||
|
const renderData = { route, filePath, fullFilePath, req, paths, directory: path.join('public', directory), videoURL, subtitleURL, subtitleVTT, info };
|
||||||
|
res.render("video-player", { h: helpers, ...renderData });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
//app.get('/OCR-Encoded-PDFs/Russell-County-Web-Site_2024-02-13_19_50_Modified-With-OCR-Encoding**', rewriter.rewrite('/Web_Site_Archives/Russell_County_Web_Site-2024-02-13_19_50_Modified_With_OCR_Encoding/$1'));
|
||||||
|
|
||||||
|
console.log(`Serving /vendor/**/* route for all files in ${path.join(config.assetsPath, 'vendor')}`);;
|
||||||
|
pageRouter.get('/vendor/**/*', async (req, res) => {
|
||||||
|
await serve(req, res, {
|
||||||
|
public: config.assetsPath,
|
||||||
|
symlinks: true,
|
||||||
|
trailingSlash: true,
|
||||||
|
cleanUrls: false,
|
||||||
|
renderSingle: false,
|
||||||
|
unlisted: [
|
||||||
|
".DS_Store",
|
||||||
|
".git",
|
||||||
|
"Thumbs.db",
|
||||||
|
"README*",
|
||||||
|
],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`Serving /css/*.css route for all files in ${path.join(config.assetsPath, 'css')}`);;
|
||||||
|
pageRouter.get('/css/*.css', async (req, res) => {
|
||||||
|
await serve(req, res, {
|
||||||
|
public: config.assetsPath,
|
||||||
|
symlinks: true,
|
||||||
|
trailingSlash: true,
|
||||||
|
cleanUrls: false,
|
||||||
|
renderSingle: false,
|
||||||
|
unlisted: [
|
||||||
|
".DS_Store",
|
||||||
|
".git",
|
||||||
|
"Thumbs.db",
|
||||||
|
"README*",
|
||||||
|
],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`Serving /svg/*.svg route for all files in ${path.join(config.assetsPath, 'svg')}`);;
|
||||||
|
pageRouter.get('/svg/*.svg', async (req, res) => {
|
||||||
|
await serve(req, res, {
|
||||||
|
public: config.assetsPath,
|
||||||
|
symlinks: true,
|
||||||
|
trailingSlash: true,
|
||||||
|
cleanUrls: false,
|
||||||
|
renderSingle: false,
|
||||||
|
unlisted: [
|
||||||
|
".DS_Store",
|
||||||
|
".git",
|
||||||
|
"Thumbs.db",
|
||||||
|
"README*",
|
||||||
|
],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
//TODO: Rewrite this facility so that it utilizes Git index as a filesystem.
|
||||||
|
console.log(`Serving * default route for all files in ${config.publicPath}`);;
|
||||||
|
pageRouter.get('*', async (req, res) => {
|
||||||
|
await serve(req, res, {
|
||||||
|
public: config.publicPath,
|
||||||
|
symlinks: true,
|
||||||
|
trailingSlash: true,
|
||||||
|
cleanUrls: false,
|
||||||
|
renderSingle: false,
|
||||||
|
unlisted: [
|
||||||
|
".*", //dot files/folders
|
||||||
|
"Thumbs.db"
|
||||||
|
],
|
||||||
|
redirects: [
|
||||||
|
{
|
||||||
|
source: "/:year(\d{4})-:mo(\d{2})-:dd(\d{2})_:hh(\d{2})_:mm(\d{2})/",
|
||||||
|
destination: "/Web_Site_Archives/Russell_County_Web_Site-:year-:mo-:dd_:hh_:mm/"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
source: "/OCR-Encoded-PDFs",
|
||||||
|
destination: "/Web_Site_Archives"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
source: "/OCR-Encoded-PDFs/Russell-County-Web-Site_2024-02-13_19_50_Modified-With-OCR-Encoding.zip",
|
||||||
|
destination: "/Web_Site_Archives/Russell_County_Web_Site-2024-02-13_19_50_Modified_With_OCR_Encoding.zip"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
source: "/OCR-Encoded-PDFs/Russell-County-Web-Site_2024-02-13_19_50_Modified-With-OCR-Encoding/:u(.*)",
|
||||||
|
destination: "/Web_Site_Archives/Russell_County_Web_Site-2024-02-13_19_50_Modified_With_OCR_Encoding:u"
|
||||||
|
},
|
||||||
|
{ source: '/YouTube Channel', destination: '/Russell_County/Board_of_Supervisors/YouTube_Archive/@russellcountyvirginia8228' },
|
||||||
|
// { source: '/YouTube Channel.zip', destination: '/Russell_County_BOS/YouTube_Channel.zip' },
|
||||||
|
// { source: '/YouTube Channel/:u?', destination: '/Russell_County_BOS/YouTube_Channel/:u' },
|
||||||
|
{ source: '/Project Reclaim [WI19KR9Ogwg].mkv', destination: '/YouTube_Archives/@VADMME/Project Reclaim [WI19KR9Ogwg].mkv' },
|
||||||
|
]
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return pageRouter;
|
||||||
|
};
|
|
@ -19,8 +19,9 @@ import parseRange from 'range-parser';
|
||||||
import { ServerResponse, OutgoingHttpHeader, OutgoingHttpHeaders } from 'http';
|
import { ServerResponse, OutgoingHttpHeader, OutgoingHttpHeaders } from 'http';
|
||||||
import { Request } from 'express';
|
import { Request } from 'express';
|
||||||
import ejs from 'ejs';
|
import ejs from 'ejs';
|
||||||
import { config } from './config.mjs';
|
import { config } from '../config.mjs';
|
||||||
import helpers from './helpers/functions.mjs';
|
import helpers from '../helpers/functions.mjs';
|
||||||
|
import { Breadcrumb } from '../helpers/breadcrumbs.mjs';
|
||||||
|
|
||||||
export interface Dictionary<T> {
|
export interface Dictionary<T> {
|
||||||
[Key: string]: T;
|
[Key: string]: T;
|
||||||
|
@ -168,8 +169,22 @@ export interface ServeErrorTemplateParameters {
|
||||||
}
|
}
|
||||||
|
|
||||||
export const directoryTemplate = (vals: ServeDirectoryTemplateParameters) => {
|
export const directoryTemplate = (vals: ServeDirectoryTemplateParameters) => {
|
||||||
|
let breadcrumbs: Breadcrumb[] = [];
|
||||||
|
if (vals.paths.length == 1 && helpers.getDirectoryName(vals.paths[0].name)) {
|
||||||
|
breadcrumbs.push({ title: config.siteWelcomeMessage, url: '/' });
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
vals.paths.forEach((path, index, paths) => {
|
||||||
|
if (index == 0) {
|
||||||
|
breadcrumbs.push({ title: config.siteName, url: '/' });
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
breadcrumbs.push({ title: helpers.getDirectoryName(path.name).replaceAll('_', ' '), url: `/${path.url}` });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
ejs.renderFile(path.join(config.viewsPath, 'directory.ejs'), { h: helpers, ...vals }, (err, str) => {
|
ejs.renderFile(path.join(config.viewsPath, 'directory.ejs'), { breadcrumbs, h: helpers, ...vals }, (err, str) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
reject(err);
|
reject(err);
|
||||||
} else {
|
} else {
|
|
@ -1,171 +0,0 @@
|
||||||
console.log(`Loading nm3clol-express-app search router module...`);
|
|
||||||
|
|
||||||
import express from 'express';
|
|
||||||
import { parse, toString } from 'lucene';
|
|
||||||
import { createClient, Query } from 'solr-client';
|
|
||||||
import { SearchResponse } from 'solr-client/dist/lib/solr.js';
|
|
||||||
import { config } from '../config.mjs';
|
|
||||||
import helpers from '../helpers/functions.mjs';
|
|
||||||
|
|
||||||
interface Dictionary<T> {
|
|
||||||
[Key: string]: T;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Highlight {
|
|
||||||
text: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
interface WithHighlighting {
|
|
||||||
highlighting: Dictionary<Highlight>;
|
|
||||||
}
|
|
||||||
|
|
||||||
const router = express.Router();
|
|
||||||
|
|
||||||
router.get('/', (req: express.Request, res: express.Response) => {
|
|
||||||
// Extract paging parameters from request query parameters
|
|
||||||
let { q = '', page = 1, pageSize = 10 } = req.query;
|
|
||||||
// Sanitize query, with particular emphasis on one problem area where soft keyboards are creating fancy quotes but we need basic quotes
|
|
||||||
if (typeof q != "undefined") {
|
|
||||||
if (typeof q != "string") {
|
|
||||||
q = (q as string[]).join(' ');
|
|
||||||
}
|
|
||||||
q = q?.replaceAll(/[“”“”„„‟❝❞〝〞〟"❠⹂🙶🙷🙸]/g, '\"').replaceAll(/[‘’‘’'‚‛❛❜❟]/g, '\'');
|
|
||||||
}
|
|
||||||
if (page instanceof String) page = parseInt(page as string);
|
|
||||||
if (pageSize instanceof String) pageSize = parseInt(pageSize as string);
|
|
||||||
// Cap at 100 max per page
|
|
||||||
pageSize = Math.min(pageSize as number, 100);
|
|
||||||
// Calculate start offset for pagination
|
|
||||||
const start = (page as number - 1) * pageSize;
|
|
||||||
if (!q || (typeof q === 'string' && q.trim() == "")) {
|
|
||||||
res.render('search-error', { h: helpers, query: q, error: { code: 400, message: 'Search query is required.'} });
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
// Parse query
|
|
||||||
let parsedQuery = parse(q);
|
|
||||||
// Construct a Solr q field query string based on the extracted components
|
|
||||||
let qQuery = toString(parsedQuery);
|
|
||||||
// Generate a Solr query based on the query strings and additional parameters
|
|
||||||
let solrQuery = new Query().df('text').q(qQuery).start(start).rows(10).hl({
|
|
||||||
on: true,
|
|
||||||
q: qQuery,
|
|
||||||
fl: '*',
|
|
||||||
snippets: 5,
|
|
||||||
formatter: 'simple',
|
|
||||||
simplePre: `<b class="result-highlight">`,
|
|
||||||
simplePost: `</b>`,
|
|
||||||
highlightMultiTerm: true,
|
|
||||||
usePhraseHighlighter: true,
|
|
||||||
});
|
|
||||||
// Create a Solr client
|
|
||||||
const solrClient = createClient({ host: config.solrDocsHost, port: config.solrDocsPort, core: config.solrDocsCore });
|
|
||||||
solrClient.search(solrQuery)
|
|
||||||
.then((solrResponse: SearchResponse<unknown>|WithHighlighting) => {
|
|
||||||
const solrResponseAsSearchResponse = solrResponse as SearchResponse<unknown>;
|
|
||||||
const solrResponseWithHighlighting = solrResponse as WithHighlighting;
|
|
||||||
//console.log(require('util').inspect(solrResponse, { showHidden: true, depth: null, colors: true }));
|
|
||||||
// overcome broken hl simplePre/simplePost implementation
|
|
||||||
let overrideHighlighting: Dictionary<Highlight> = {};
|
|
||||||
Object.keys(solrResponseWithHighlighting.highlighting).forEach((highlight_key: string) => {
|
|
||||||
overrideHighlighting[highlight_key] = solrResponseWithHighlighting.highlighting[highlight_key];
|
|
||||||
if (overrideHighlighting[highlight_key].text && overrideHighlighting[highlight_key].text.length > 0) {
|
|
||||||
overrideHighlighting[highlight_key].text = overrideHighlighting[highlight_key].text.map( (text) => {
|
|
||||||
return text.replaceAll("<em>", `<b class="result-highlight">`).replaceAll("</em>", "</b>")
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
solrResponseWithHighlighting.highlighting = overrideHighlighting;
|
|
||||||
// Calculate total number of results (needed for pagination)
|
|
||||||
const totalResults = solrResponseAsSearchResponse.response.numFound;
|
|
||||||
// Calculate total number of pages
|
|
||||||
const totalPages = Math.ceil(totalResults / pageSize);
|
|
||||||
res.render('search-results', {
|
|
||||||
h: helpers,
|
|
||||||
query: qQuery,
|
|
||||||
page,
|
|
||||||
pageSize,
|
|
||||||
totalResults,
|
|
||||||
totalPages,
|
|
||||||
solrQuery: solrQuery,
|
|
||||||
...solrResponse
|
|
||||||
});
|
|
||||||
// res.render('search-error', { h: helpers, query: sanitizedQuery, error: { code: 400, message: 'Search query is required.'} });
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
if (typeof error === 'object' && error instanceof Error) {
|
|
||||||
// check for error from throw new Error(`Request HTTP error ${response.statusCode}: ${text}`) in solr.ts from
|
|
||||||
// solr-node-client dependency
|
|
||||||
const detectRequestHttpErrorRegExLit = /^Request HTTP error (?<statusCode>\d{1,3}): (?<text>\{.*\}$)/s;
|
|
||||||
const detectRequestHttpErrorRegExp = new RegExp(detectRequestHttpErrorRegExLit);
|
|
||||||
const matchRequestHttpErrorRegExpInError = error.message.match(detectRequestHttpErrorRegExp);
|
|
||||||
const statusCode = (matchRequestHttpErrorRegExpInError && matchRequestHttpErrorRegExpInError.groups && matchRequestHttpErrorRegExpInError.groups.statusCode);
|
|
||||||
const text = (matchRequestHttpErrorRegExpInError && matchRequestHttpErrorRegExpInError.groups && matchRequestHttpErrorRegExpInError.groups.text);
|
|
||||||
if (text) {
|
|
||||||
let solrRequestHttpInternalError = JSON.parse(text);
|
|
||||||
error = {
|
|
||||||
message: "Solr Client Request HTTP Error",
|
|
||||||
code: statusCode,
|
|
||||||
innerError: solrRequestHttpInternalError
|
|
||||||
};
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
error = {
|
|
||||||
message: error
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
res.render('search-error', { h: helpers, query: qQuery, error });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// // Sanitize search query to prevent code injection
|
|
||||||
// try {
|
|
||||||
// // Validate search query
|
|
||||||
// if (!query) {
|
|
||||||
// //return res.status(400).json({ error: 'q parameter is required' });
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
// else {
|
|
||||||
// // Send search query to Solr
|
|
||||||
// const response = await axios.get(solrUrl + '/select', {
|
|
||||||
// params: {
|
|
||||||
// q: `text:${sanitizedQuery}`, // Query string with field name
|
|
||||||
// hl: 'true',
|
|
||||||
// 'hl.method': 'unified',
|
|
||||||
// 'hl.fl': '*',
|
|
||||||
// 'hl.snippets': 5,
|
|
||||||
// 'hl.tag.pre': '<strong class=\"result-highlight\">',
|
|
||||||
// 'hl.tag.post': '</strong>',
|
|
||||||
// 'hl.usePhraseHighlighter': true,
|
|
||||||
// start, // Start offset for pagination
|
|
||||||
// rows: 10, // Number of rows to return
|
|
||||||
// wt: 'json', // Response format (JSON)
|
|
||||||
// },
|
|
||||||
// });
|
|
||||||
//
|
|
||||||
// // Extract search results from Solr response
|
|
||||||
// const searchResults = response.data.response.docs;
|
|
||||||
// const highlightedSnippets = response.data.highlighting;
|
|
||||||
|
|
||||||
// // Calculate total number of results (needed for pagination)
|
|
||||||
// const totalResults = response.data.response.numFound;
|
|
||||||
|
|
||||||
// // Calculate total number of pages
|
|
||||||
// const totalPages = Math.ceil(totalResults / pageSize);
|
|
||||||
|
|
||||||
// // Send search results as JSON response
|
|
||||||
// //res.json('search-results', { query, searchResults, highlightedSnippets, page, pageSize, totalResults, totalPages });
|
|
||||||
// res.render('search-results', { h: helpers, query: sanitizedQuery, searchResults, highlightedSnippets, page, pageSize, totalResults, totalPages });
|
|
||||||
// }
|
|
||||||
// } catch (error) {
|
|
||||||
// // console.error('Error searching Solr:', error.message);
|
|
||||||
// // res.status(500).json({ error: 'Internal server error' });
|
|
||||||
// res.render('search-error', { h: helpers, query: sanitizedQuery, error });
|
|
||||||
// }
|
|
||||||
});
|
|
||||||
|
|
||||||
export default {
|
|
||||||
router,
|
|
||||||
// solrUrl,
|
|
||||||
// sanitizeQuery,
|
|
||||||
};
|
|
189
app/search/router.mts
Normal file
189
app/search/router.mts
Normal file
|
@ -0,0 +1,189 @@
|
||||||
|
console.log(`Loading nm3clol-express-app search router module...`);
|
||||||
|
|
||||||
|
import express from 'express';
|
||||||
|
import { parse, toString } from 'lucene';
|
||||||
|
import { createClient, Query } from 'solr-client';
|
||||||
|
import { SearchResponse } from 'solr-client/dist/lib/solr.js';
|
||||||
|
import { config } from '../config.mjs';
|
||||||
|
import helpers from '../helpers/functions.mjs';
|
||||||
|
import { Breadcrumb } from '../helpers/breadcrumbs.mjs';
|
||||||
|
|
||||||
|
interface Dictionary<T> {
|
||||||
|
[Key: string]: T;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Highlight {
|
||||||
|
text: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface WithHighlighting {
|
||||||
|
highlighting: Dictionary<Highlight>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function () {
|
||||||
|
const searchRouter = express.Router();
|
||||||
|
|
||||||
|
searchRouter.get('/', (req: express.Request, res: express.Response) => {
|
||||||
|
// Extract paging parameters from request query parameters
|
||||||
|
let { q = '', page = 1, pageSize = 10 } = req.query;
|
||||||
|
// Sanitize query, with particular emphasis on one problem area where soft keyboards are creating fancy quotes but we need basic quotes
|
||||||
|
if (typeof q != "undefined") {
|
||||||
|
if (typeof q != "string") {
|
||||||
|
q = (q as string[]).join(' ');
|
||||||
|
}
|
||||||
|
q = q?.replaceAll(/[“”“”„„‟❝❞〝〞〟"❠⹂🙶🙷🙸]/g, '\"').replaceAll(/[‘’‘’'‚‛❛❜❟]/g, '\'');
|
||||||
|
}
|
||||||
|
if (page instanceof String) page = parseInt(page as string);
|
||||||
|
if (pageSize instanceof String) pageSize = parseInt(pageSize as string);
|
||||||
|
// Cap at 100 max per page
|
||||||
|
pageSize = Math.min(pageSize as number, 100);
|
||||||
|
// Calculate start offset for pagination
|
||||||
|
const start = (page as number - 1) * pageSize;
|
||||||
|
if (!q || (typeof q === 'string' && q.trim() == "")) {
|
||||||
|
// Build breadcrumbs
|
||||||
|
const breadcrumbs: Breadcrumb[] = [
|
||||||
|
{ title: `${config.siteName}`, url: '/' },
|
||||||
|
{ title: `Search Error`, url: req.url }
|
||||||
|
];
|
||||||
|
// Render ejs page to output
|
||||||
|
res.render('search-error', { breadcrumbs, h: helpers, query: q, error: { code: 400, message: 'Search query is required.'} });
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
// Parse query
|
||||||
|
let parsedQuery = parse(q);
|
||||||
|
// Construct a Solr q field query string based on the extracted components
|
||||||
|
let qQuery = toString(parsedQuery);
|
||||||
|
// Generate a Solr query based on the query strings and additional parameters
|
||||||
|
let solrQuery = new Query().df('text').q(qQuery).start(start).rows(10).hl({
|
||||||
|
on: true,
|
||||||
|
q: qQuery,
|
||||||
|
fl: '*',
|
||||||
|
snippets: 5,
|
||||||
|
formatter: 'simple',
|
||||||
|
simplePre: `<b class="result-highlight">`,
|
||||||
|
simplePost: `</b>`,
|
||||||
|
highlightMultiTerm: true,
|
||||||
|
usePhraseHighlighter: true,
|
||||||
|
});
|
||||||
|
// Create a Solr client
|
||||||
|
const solrClient = createClient({ host: config.solrDocsHost, port: config.solrDocsPort, core: config.solrDocsCore });
|
||||||
|
solrClient.search(solrQuery)
|
||||||
|
.then((solrResponse: SearchResponse<unknown>|WithHighlighting) => {
|
||||||
|
const solrResponseAsSearchResponse = solrResponse as SearchResponse<unknown>;
|
||||||
|
const solrResponseWithHighlighting = solrResponse as WithHighlighting;
|
||||||
|
//console.log(require('util').inspect(solrResponse, { showHidden: true, depth: null, colors: true }));
|
||||||
|
// overcome broken hl simplePre/simplePost implementation
|
||||||
|
let overrideHighlighting: Dictionary<Highlight> = {};
|
||||||
|
Object.keys(solrResponseWithHighlighting.highlighting).forEach((highlight_key: string) => {
|
||||||
|
overrideHighlighting[highlight_key] = solrResponseWithHighlighting.highlighting[highlight_key];
|
||||||
|
if (overrideHighlighting[highlight_key].text && overrideHighlighting[highlight_key].text.length > 0) {
|
||||||
|
overrideHighlighting[highlight_key].text = overrideHighlighting[highlight_key].text.map( (text) => {
|
||||||
|
return text.replaceAll("<em>", `<b class="result-highlight">`).replaceAll("</em>", "</b>")
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
solrResponseWithHighlighting.highlighting = overrideHighlighting;
|
||||||
|
// Calculate total number of results (needed for pagination)
|
||||||
|
const totalResults = solrResponseAsSearchResponse.response.numFound;
|
||||||
|
// Calculate total number of pages
|
||||||
|
const totalPages = Math.ceil(totalResults / pageSize);
|
||||||
|
// Build breadcrumbs
|
||||||
|
let breadcrumbs: Breadcrumb[] = [
|
||||||
|
{ title: `${config.siteName}`, url: '/' },
|
||||||
|
{ title: `Search Results for ${qQuery}`, url: req.url }
|
||||||
|
];
|
||||||
|
// Render ejs page to output
|
||||||
|
res.render('search-results', {
|
||||||
|
breadcrumbs,
|
||||||
|
h: helpers,
|
||||||
|
query: qQuery,
|
||||||
|
page,
|
||||||
|
pageSize,
|
||||||
|
solrQuery: solrQuery,
|
||||||
|
totalResults,
|
||||||
|
totalPages,
|
||||||
|
...solrResponse
|
||||||
|
});
|
||||||
|
// res.render('search-error', { h: helpers, query: sanitizedQuery, error: { code: 400, message: 'Search query is required.'} });
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
if (typeof error === 'object' && error instanceof Error) {
|
||||||
|
// check for error from throw new Error(`Request HTTP error ${response.statusCode}: ${text}`) in solr.ts from
|
||||||
|
// solr-node-client dependency
|
||||||
|
const detectRequestHttpErrorRegExLit = /^Request HTTP error (?<statusCode>\d{1,3}): (?<text>\{.*\}$)/s;
|
||||||
|
const detectRequestHttpErrorRegExp = new RegExp(detectRequestHttpErrorRegExLit);
|
||||||
|
const matchRequestHttpErrorRegExpInError = error.message.match(detectRequestHttpErrorRegExp);
|
||||||
|
const statusCode = (matchRequestHttpErrorRegExpInError && matchRequestHttpErrorRegExpInError.groups && matchRequestHttpErrorRegExpInError.groups.statusCode);
|
||||||
|
const text = (matchRequestHttpErrorRegExpInError && matchRequestHttpErrorRegExpInError.groups && matchRequestHttpErrorRegExpInError.groups.text);
|
||||||
|
if (text) {
|
||||||
|
let solrRequestHttpInternalError = JSON.parse(text);
|
||||||
|
error = {
|
||||||
|
message: "Solr Client Request HTTP Error",
|
||||||
|
code: statusCode,
|
||||||
|
innerError: solrRequestHttpInternalError
|
||||||
|
};
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
error = {
|
||||||
|
message: error
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Build breadcrumbs
|
||||||
|
const breadcrumbs: Breadcrumb[] = [
|
||||||
|
{ title: `${config.siteName}`, url: '/' },
|
||||||
|
{ title: `Search Error` + (qQuery ? ` for ${qQuery}` : ``), url: req.url }
|
||||||
|
];
|
||||||
|
// Render ejs page to output
|
||||||
|
res.render('search-error', { breadcrumbs, error, h: helpers, query: qQuery});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// // Sanitize search query to prevent code injection
|
||||||
|
// try {
|
||||||
|
// // Validate search query
|
||||||
|
// if (!query) {
|
||||||
|
// //return res.status(400).json({ error: 'q parameter is required' });
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
// else {
|
||||||
|
// // Send search query to Solr
|
||||||
|
// const response = await axios.get(solrUrl + '/select', {
|
||||||
|
// params: {
|
||||||
|
// q: `text:${sanitizedQuery}`, // Query string with field name
|
||||||
|
// hl: 'true',
|
||||||
|
// 'hl.method': 'unified',
|
||||||
|
// 'hl.fl': '*',
|
||||||
|
// 'hl.snippets': 5,
|
||||||
|
// 'hl.tag.pre': '<strong class=\"result-highlight\">',
|
||||||
|
// 'hl.tag.post': '</strong>',
|
||||||
|
// 'hl.usePhraseHighlighter': true,
|
||||||
|
// start, // Start offset for pagination
|
||||||
|
// rows: 10, // Number of rows to return
|
||||||
|
// wt: 'json', // Response format (JSON)
|
||||||
|
// },
|
||||||
|
// });
|
||||||
|
//
|
||||||
|
// // Extract search results from Solr response
|
||||||
|
// const searchResults = response.data.response.docs;
|
||||||
|
// const highlightedSnippets = response.data.highlighting;
|
||||||
|
|
||||||
|
// // Calculate total number of results (needed for pagination)
|
||||||
|
// const totalResults = response.data.response.numFound;
|
||||||
|
|
||||||
|
// // Calculate total number of pages
|
||||||
|
// const totalPages = Math.ceil(totalResults / pageSize);
|
||||||
|
|
||||||
|
// // Send search results as JSON response
|
||||||
|
// //res.json('search-results', { query, searchResults, highlightedSnippets, page, pageSize, totalResults, totalPages });
|
||||||
|
// res.render('search-results', { h: helpers, query: sanitizedQuery, searchResults, highlightedSnippets, page, pageSize, totalResults, totalPages });
|
||||||
|
// }
|
||||||
|
// } catch (error) {
|
||||||
|
// // console.error('Error searching Solr:', error.message);
|
||||||
|
// // res.status(500).json({ error: 'Internal server error' });
|
||||||
|
// res.render('search-error', { h: helpers, query: sanitizedQuery, error });
|
||||||
|
// }
|
||||||
|
});
|
||||||
|
|
||||||
|
return searchRouter;
|
||||||
|
};
|
25
app/search/solr-doc.mts
Normal file
25
app/search/solr-doc.mts
Normal file
|
@ -0,0 +1,25 @@
|
||||||
|
/**
|
||||||
|
* Needed until the conversion is completed.
|
||||||
|
*/
|
||||||
|
export interface IncorrectStyleSolrDocument {
|
||||||
|
id: string;
|
||||||
|
sha256sum: string[];
|
||||||
|
url: string[];
|
||||||
|
content_length: number[];
|
||||||
|
content_type: string[];
|
||||||
|
text: string[];
|
||||||
|
_version_?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Describes Solr full-text search properties for a document file in the public repository.
|
||||||
|
*/
|
||||||
|
export interface SolrDocument {
|
||||||
|
id: string;
|
||||||
|
sha256sum: string;
|
||||||
|
url: string;
|
||||||
|
content_length: number;
|
||||||
|
content_type: string;
|
||||||
|
text: string;
|
||||||
|
_version_?: number;
|
||||||
|
}
|
268
app/server.mts
268
app/server.mts
|
@ -2,22 +2,12 @@ console.log(`Starting up nm3clol-express-app...`);
|
||||||
|
|
||||||
import express from 'express';
|
import express from 'express';
|
||||||
import axios from 'axios';
|
import axios from 'axios';
|
||||||
|
import pageRouter from './page/router.mjs';
|
||||||
|
import searchRouter from './search/router.mjs';
|
||||||
|
import { config } from './config.mjs';
|
||||||
|
|
||||||
const app = express();
|
const app = express();
|
||||||
|
|
||||||
import serve from './vercel-serve.mjs';
|
|
||||||
import path from 'path';
|
|
||||||
import { globSync } from 'glob';
|
|
||||||
import matter from 'gray-matter';
|
|
||||||
import ejs from 'ejs';
|
|
||||||
import { config } from './config.mjs';
|
|
||||||
import helpers from './helpers/functions.mjs';
|
|
||||||
import search from './routes/search.mjs';
|
|
||||||
import fs from 'fs';
|
|
||||||
import { readFile } from 'fs/promises';
|
|
||||||
import { inspect } from 'util';
|
|
||||||
// import advancedSearch from '../routes/advanced-search';
|
|
||||||
|
|
||||||
console.log(`Running app configuration:`, config);
|
console.log(`Running app configuration:`, config);
|
||||||
|
|
||||||
// Set EJS as the view engine
|
// Set EJS as the view engine
|
||||||
|
@ -32,256 +22,14 @@ app.use(express.json());
|
||||||
// Middleware to rewrite requests
|
// Middleware to rewrite requests
|
||||||
//app.use(rewriter);
|
//app.use(rewriter);
|
||||||
|
|
||||||
// // Serve static files (CSS, JavaScript, images, etc.)
|
|
||||||
// app.use(serve('../public', {
|
|
||||||
// dotfiles: 'ignore',
|
|
||||||
// index: false,
|
|
||||||
// }));
|
|
||||||
|
|
||||||
// app.get('/', (req, res) => {
|
|
||||||
// res.send('Hello World!');
|
|
||||||
// })
|
|
||||||
|
|
||||||
// console.log("Setting route for /ads.txt");
|
|
||||||
// app.get('/ads.txt', (req, res) => {
|
|
||||||
// res.setHeader("Content-Type", "text/plain");
|
|
||||||
// res.setHeader("Cache-Control", "no-cache");
|
|
||||||
// res.send(`google.com, pub-8937572456576531, DIRECT, f08c47fec0942fa0`);
|
|
||||||
// });
|
|
||||||
|
|
||||||
console.log(`Serving /robots.txt from memory.`);
|
|
||||||
app.get('/robots.txt', (req, res) => {
|
|
||||||
res.setHeader("Content-Type", "text/plain");
|
|
||||||
res.setHeader("Cache-Control", "no-cache");
|
|
||||||
// TODO: Implement Site Map feature and provide sitemap url in robots.txt
|
|
||||||
res.send(
|
|
||||||
`User-agent: *
|
|
||||||
Allow: /
|
|
||||||
|
|
||||||
# TODO: Implement Site Map feature and provide sitemap url in robots.txt
|
|
||||||
#sitemap: https://no-moss-3-carbo-landfill-library.online/sitemap.xml`
|
|
||||||
);//end of res.send() for robots.txt
|
|
||||||
});
|
|
||||||
|
|
||||||
// Search endpoints
|
// Search endpoints
|
||||||
console.log(`Serving /search using search router.`);
|
console.log(`Assigning /search route to search router.`);
|
||||||
app.use('/search', search.router);
|
app.use('/search', searchRouter());
|
||||||
// app.use('/advanced-search', advancedSearch.router);
|
// app.use('/advanced-search', advancedSearch.router);
|
||||||
|
|
||||||
// Endpoints for all the site's pages.
|
// Search endpoints
|
||||||
console.log(`Scanning for pages in ${config.pagesPath} to create routes.`);
|
console.log(`Assigning / route to page router.`);
|
||||||
globSync('**/*.md', {
|
app.use('/', pageRouter());
|
||||||
cwd: config.pagesPath,
|
|
||||||
matchBase: true,
|
|
||||||
follow: true,
|
|
||||||
}).forEach((filePath) => {
|
|
||||||
const expressRoutePathFromFilePath = (filePath: string) => {
|
|
||||||
filePath = filePath.substring(0, filePath.length - path.extname(filePath).length).replaceAll(path.sep, path.posix.sep);
|
|
||||||
if (!filePath.startsWith('/') && filePath.length > 0) {
|
|
||||||
filePath = `/${filePath}`;
|
|
||||||
}
|
|
||||||
return filePath;
|
|
||||||
};
|
|
||||||
const route = expressRoutePathFromFilePath(filePath);
|
|
||||||
const fullFilePath = path.join(config.pagesPath, filePath);
|
|
||||||
let paths = route.split(path.posix.sep);
|
|
||||||
console.log(`Serving ${route} route as a page at ${fullFilePath}.`);
|
|
||||||
app.get(route, async (req, res) => {
|
|
||||||
const fm = matter.read(fullFilePath);
|
|
||||||
const fmData = { fm: fm.data, excerpt: fm.excerpt };
|
|
||||||
const content = helpers.md.render(fm.content, fmData );
|
|
||||||
const renderData = { content, route, filePath, fullFilePath, req, paths, ...fmData };
|
|
||||||
res.render("page", { h: helpers, ...renderData });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// console.log("Scanning for documents to create routes.");
|
|
||||||
// glob.globSync('**/*{.pdf,.docx,.xlsx,.pptx,.doc,.xls,.ppt}', {
|
|
||||||
// cwd: path.join(config.publicPath),
|
|
||||||
// matchBase: true,
|
|
||||||
// follow: true,
|
|
||||||
// }).forEach((filePath) => {
|
|
||||||
// const expressRoutePathFromFilePath = (filePath) => {
|
|
||||||
// return filePath.substring(0, filePath.length - path.extname(filePath).length).replaceAll(path.sep, path.posix.sep);
|
|
||||||
// };
|
|
||||||
// const route = expressRoutePathFromFilePath(filePath);
|
|
||||||
// const fullFilePath = path.join(config.publicPath, filePath);
|
|
||||||
// let paths = route.split(path.posix.sep);
|
|
||||||
// paths[0] = 'public';
|
|
||||||
// console.log(`Setting route for ${route}`);
|
|
||||||
// app.get(route, async (req, res) => {
|
|
||||||
// const fm = matter.read(fullFilePath);
|
|
||||||
// const fmData = { fm: fm.data, excerpt: fm.excerpt };
|
|
||||||
// const content = helpers.md.render(fm.content, fmData );
|
|
||||||
// const renderData = { content, route, filePath, fullFilePath, req, paths, ...fmData };
|
|
||||||
// res.render("page", { h: helpers, ...renderData });
|
|
||||||
// });
|
|
||||||
// });
|
|
||||||
|
|
||||||
//TODO: Rewrite this facility so that it utilizes Git index as a filesystem.
|
|
||||||
console.log("Scanning for web archive HTML documents to create routes.");
|
|
||||||
globSync('Web_Site_Archives/**/*{.htm,.html}', {
|
|
||||||
cwd: config.publicPath,
|
|
||||||
matchBase: true,
|
|
||||||
follow: true,
|
|
||||||
}).forEach((filePath) => {
|
|
||||||
const expressRoutePathFromFilePath = (filePath: string) => {
|
|
||||||
return '/' + filePath.replaceAll(path.sep, path.posix.sep);
|
|
||||||
};
|
|
||||||
const route = expressRoutePathFromFilePath(filePath);
|
|
||||||
const fullFilePath = path.join(config.publicPath, filePath);
|
|
||||||
let paths = route.split(path.posix.sep);
|
|
||||||
paths[0] = 'public';
|
|
||||||
console.log(`Setting route for ${route}`);
|
|
||||||
app.get(route, async (req, res) => {
|
|
||||||
const html = fs.readFileSync(fullFilePath).toString();
|
|
||||||
const renderData = { route, filePath, fullFilePath, req, paths, html };
|
|
||||||
res.render("archive", { h: helpers, ...renderData });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
|
|
||||||
//TODO: Rewrite this facility so that it utilizes Git index as a filesystem.
|
|
||||||
console.log("Scanning for archived videos to create routes.");
|
|
||||||
globSync(['Russell_County/Board_of_Supervisors/YouTube_Archive/**/*.info.json', 'Virginia_Energy/YouTube_Archive/**/*.info.json', 'Virginia_Governor/**/*.info.json'], {
|
|
||||||
cwd: config.publicPath,
|
|
||||||
matchBase: true,
|
|
||||||
follow: true,
|
|
||||||
}).forEach((filePath: string) => {
|
|
||||||
const expressRoutePathFromFilePath = (filePath: string) => {
|
|
||||||
return path.posix.sep+filePath.substring(0, filePath.lastIndexOf(path.sep)).replaceAll(path.sep, path.posix.sep);
|
|
||||||
};
|
|
||||||
const dirFromFilePath = (filePath: string) => {
|
|
||||||
return filePath.substring(0, filePath.lastIndexOf(path.sep));
|
|
||||||
}
|
|
||||||
const directory = dirFromFilePath(filePath);
|
|
||||||
let videoURL = ""+globSync("*.{mpg,mpeg,mp4,mkv,webm}", {
|
|
||||||
cwd: path.join(config.publicPath, directory),
|
|
||||||
matchBase: true,
|
|
||||||
follow: true,
|
|
||||||
}).pop();
|
|
||||||
let subtitleURL = ""+globSync("*.en.vtt", {
|
|
||||||
cwd: path.join(config.publicPath, directory),
|
|
||||||
matchBase: true,
|
|
||||||
follow: true,
|
|
||||||
}).pop();
|
|
||||||
let subtitleFile = path.join(config.publicPath, directory, subtitleURL);
|
|
||||||
const route = encodeURI(expressRoutePathFromFilePath(filePath));
|
|
||||||
let paths = filePath
|
|
||||||
.substring(0, filePath.lastIndexOf(path.sep) > 0 ? filePath.lastIndexOf(path.sep) : filePath.length-1)
|
|
||||||
.split(path.sep)
|
|
||||||
.map((name, idx, aPaths) => {
|
|
||||||
let url = aPaths.slice(0, idx+1).join(path.posix.sep);
|
|
||||||
return {
|
|
||||||
name,
|
|
||||||
url,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
const fullFilePath = path.join(config.publicPath, filePath);
|
|
||||||
console.log(`Setting route for ${route}`);
|
|
||||||
app.get(route, async (req, res) => {
|
|
||||||
if (!req.path.endsWith('/')) {
|
|
||||||
res.redirect(req.path + '/');
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
let info = JSON.parse((await readFile(fullFilePath)).toString());
|
|
||||||
let subtitleVTT = fs.existsSync(subtitleFile) ? (await readFile(subtitleFile)).toString() : '';
|
|
||||||
const renderData = { inspect, route, filePath, fullFilePath, req, paths, directory: path.join('public', directory), videoURL, subtitleURL, subtitleVTT, info };
|
|
||||||
res.render("video-player", { h: helpers, ...renderData });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
//app.get('/OCR-Encoded-PDFs/Russell-County-Web-Site_2024-02-13_19_50_Modified-With-OCR-Encoding**', rewriter.rewrite('/Web_Site_Archives/Russell_County_Web_Site-2024-02-13_19_50_Modified_With_OCR_Encoding/$1'));
|
|
||||||
|
|
||||||
console.log(`Serving /vendor/**/* route for all files in ${path.join(config.assetsPath, 'vendor')}`);;
|
|
||||||
app.get('/vendor/**/*', async (req, res) => {
|
|
||||||
await serve(req, res, {
|
|
||||||
public: config.assetsPath,
|
|
||||||
symlinks: true,
|
|
||||||
trailingSlash: true,
|
|
||||||
cleanUrls: false,
|
|
||||||
renderSingle: false,
|
|
||||||
unlisted: [
|
|
||||||
".DS_Store",
|
|
||||||
".git",
|
|
||||||
"Thumbs.db",
|
|
||||||
"README*",
|
|
||||||
],
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(`Serving /css/*.css route for all files in ${path.join(config.assetsPath, 'css')}`);;
|
|
||||||
app.get('/css/*.css', async (req, res) => {
|
|
||||||
await serve(req, res, {
|
|
||||||
public: config.assetsPath,
|
|
||||||
symlinks: true,
|
|
||||||
trailingSlash: true,
|
|
||||||
cleanUrls: false,
|
|
||||||
renderSingle: false,
|
|
||||||
unlisted: [
|
|
||||||
".DS_Store",
|
|
||||||
".git",
|
|
||||||
"Thumbs.db",
|
|
||||||
"README*",
|
|
||||||
],
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(`Serving /svg/*.svg route for all files in ${path.join(config.assetsPath, 'svg')}`);;
|
|
||||||
app.get('/svg/*.svg', async (req, res) => {
|
|
||||||
await serve(req, res, {
|
|
||||||
public: config.assetsPath,
|
|
||||||
symlinks: true,
|
|
||||||
trailingSlash: true,
|
|
||||||
cleanUrls: false,
|
|
||||||
renderSingle: false,
|
|
||||||
unlisted: [
|
|
||||||
".DS_Store",
|
|
||||||
".git",
|
|
||||||
"Thumbs.db",
|
|
||||||
"README*",
|
|
||||||
],
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
//TODO: Rewrite this facility so that it utilizes Git index as a filesystem.
|
|
||||||
console.log(`Serving * default route for all files in ${config.publicPath}`);;
|
|
||||||
app.get('*', async (req, res) => {
|
|
||||||
await serve(req, res, {
|
|
||||||
public: config.publicPath,
|
|
||||||
symlinks: true,
|
|
||||||
trailingSlash: true,
|
|
||||||
cleanUrls: false,
|
|
||||||
renderSingle: false,
|
|
||||||
unlisted: [
|
|
||||||
".*", //dot files/folders
|
|
||||||
"Thumbs.db"
|
|
||||||
],
|
|
||||||
redirects: [
|
|
||||||
{
|
|
||||||
source: "/:year(\d{4})-:mo(\d{2})-:dd(\d{2})_:hh(\d{2})_:mm(\d{2})/",
|
|
||||||
destination: "/Web_Site_Archives/Russell_County_Web_Site-:year-:mo-:dd_:hh_:mm/"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
source: "/OCR-Encoded-PDFs",
|
|
||||||
destination: "/Web_Site_Archives"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
source: "/OCR-Encoded-PDFs/Russell-County-Web-Site_2024-02-13_19_50_Modified-With-OCR-Encoding.zip",
|
|
||||||
destination: "/Web_Site_Archives/Russell_County_Web_Site-2024-02-13_19_50_Modified_With_OCR_Encoding.zip"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
source: "/OCR-Encoded-PDFs/Russell-County-Web-Site_2024-02-13_19_50_Modified-With-OCR-Encoding/:u(.*)",
|
|
||||||
destination: "/Web_Site_Archives/Russell_County_Web_Site-2024-02-13_19_50_Modified_With_OCR_Encoding:u"
|
|
||||||
},
|
|
||||||
{ source: '/YouTube Channel', destination: '/Russell_County/Board_of_Supervisors/YouTube_Archive/@russellcountyvirginia8228' },
|
|
||||||
// { source: '/YouTube Channel.zip', destination: '/Russell_County_BOS/YouTube_Channel.zip' },
|
|
||||||
// { source: '/YouTube Channel/:u?', destination: '/Russell_County_BOS/YouTube_Channel/:u' },
|
|
||||||
{ source: '/Project Reclaim [WI19KR9Ogwg].mkv', destination: '/YouTube_Archives/@VADMME/Project Reclaim [WI19KR9Ogwg].mkv' },
|
|
||||||
]
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Start server
|
// Start server
|
||||||
app.listen(config.appHttpPort, () => {
|
app.listen(config.appHttpPort, () => {
|
||||||
|
|
|
@ -13,25 +13,7 @@
|
||||||
|
|
||||||
<main class="container">
|
<main class="container">
|
||||||
<header>
|
<header>
|
||||||
<h1 class="mt-5" style="font-family: 'Covered By Your Grace'">
|
<%- include('./includes/breadcrumbs.ejs') %>
|
||||||
<% paths.forEach(function(value, index) { %>
|
|
||||||
<% if (h.shouldShowDirectorySeparator(index)) { %>
|
|
||||||
<span class="separator">› </span>
|
|
||||||
<% } %>
|
|
||||||
<% if (h.shouldShowSiteWelcomeMessage(paths)) { %>
|
|
||||||
<i> </i>
|
|
||||||
<%= h.getSiteWelcomeMessage() %>
|
|
||||||
<% } else if (h.shouldOmitLinkOnLastBreadcrumb(paths, index)) { %>
|
|
||||||
<%= h.trimSlashes(value.name).replaceAll('_', ' ') %>
|
|
||||||
<% } else if (index == 0) { %>
|
|
||||||
<a href="/"><%= h.getSiteName() %></a>
|
|
||||||
<% } else { %>
|
|
||||||
<a href="/<%= value.url %>">
|
|
||||||
<%= h.getDirectoryName(value.name).replaceAll('_', ' ') %>
|
|
||||||
</a>
|
|
||||||
<% } %>
|
|
||||||
<% }); %>
|
|
||||||
</h1>
|
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
<% if (h.directoryContainsReadme(directory)) {%>
|
<% if (h.directoryContainsReadme(directory)) {%>
|
||||||
|
@ -54,12 +36,6 @@
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<%- include('./includes/bottom-navbar.ejs') %>
|
<%- include('./includes/bottom-navbar.ejs') %>
|
||||||
|
<%- include('./includes/bottom-scripts.ejs') %>
|
||||||
<!-- Bootstrap JS (optional, if you need Bootstrap JS features) -->
|
|
||||||
<script src="https://daball.me/vendor/jquery/jquery.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/popper.js/dist/popper.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/jquery-easing/jquery.easing.min.js"></script>
|
|
||||||
<script src="https://daball.me/layouts/blog/js/blog.min.js"></script>
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
6
app/views/includes/bottom-scripts.ejs
Normal file
6
app/views/includes/bottom-scripts.ejs
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
<!-- Bootstrap JS (optional, if you need Bootstrap JS features) -->
|
||||||
|
<script src="https://daball.me/vendor/jquery/jquery.min.js"></script>
|
||||||
|
<script src="https://daball.me/vendor/popper.js/dist/popper.min.js"></script>
|
||||||
|
<script src="https://daball.me/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
|
||||||
|
<script src="https://daball.me/vendor/jquery-easing/jquery.easing.min.js"></script>
|
||||||
|
<script src="https://daball.me/layouts/blog/js/blog.min.js"></script>
|
12
app/views/includes/breadcrumbs.ejs
Normal file
12
app/views/includes/breadcrumbs.ejs
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
<h1 class="mt-5" style="font-family: 'Covered By Your Grace'">
|
||||||
|
<% breadcrumbs.forEach(function(breadcrumb, index) { %>
|
||||||
|
<% if (index > 0 && index < breadcrumbs.length) { %>
|
||||||
|
<span class="separator">› </span>
|
||||||
|
<% } %>
|
||||||
|
<% if (index === breadcrumbs.length-1) { %>
|
||||||
|
<%=breadcrumb.title%>
|
||||||
|
<% } else { %>
|
||||||
|
<a href="<%=breadcrumb.url%>"><%=breadcrumb.title%></a>
|
||||||
|
<% } %>
|
||||||
|
<% }) %>
|
||||||
|
</h1>
|
|
@ -13,22 +13,7 @@
|
||||||
|
|
||||||
<main class="container">
|
<main class="container">
|
||||||
<header>
|
<header>
|
||||||
<h1 class="mt-5" style="font-family: 'Covered By Your Grace'">
|
<%- include('./includes/breadcrumbs.ejs') %>
|
||||||
<% paths.forEach(function(value, index) { %>
|
|
||||||
<% if (h.shouldShowDirectorySeparator({index})) { %>
|
|
||||||
<span class="separator">› </span>
|
|
||||||
<% } %>
|
|
||||||
<% if (h.shouldOmitLinkOnLastBreadcrumb({paths, index})) { %>
|
|
||||||
<%= (typeof fm.title !== 'undefined') ? `${fm.title}` : value %>
|
|
||||||
<% } else if (index == 0) { %>
|
|
||||||
<a href="/"><%= h.getSiteName() %></a>
|
|
||||||
<% } else { %>
|
|
||||||
<a href="/<%= h.trimSlashes(h.leftTrimFirstDirectory(value)) %>">
|
|
||||||
<%= (value == 'public' ? h.getSiteName() : value) %>
|
|
||||||
</a>
|
|
||||||
<% } %>
|
|
||||||
<% }) %>
|
|
||||||
</h1>
|
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
<% if (typeof content !== 'undefined') {%>
|
<% if (typeof content !== 'undefined') {%>
|
||||||
|
@ -57,12 +42,6 @@
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<%- include('./includes/bottom-navbar.ejs') %>
|
<%- include('./includes/bottom-navbar.ejs') %>
|
||||||
|
<%- include('./includes/bottom-scripts.ejs') %>
|
||||||
<!-- Bootstrap JS (optional, if you need Bootstrap JS features) -->
|
|
||||||
<script src="https://daball.me/vendor/jquery/jquery.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/popper.js/dist/popper.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/jquery-easing/jquery.easing.min.js"></script>
|
|
||||||
<script src="https://daball.me/layouts/blog/js/blog.min.js"></script>
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
|
@ -9,11 +9,7 @@
|
||||||
<%- include('./includes/no-trash-svg.ejs') %>
|
<%- include('./includes/no-trash-svg.ejs') %>
|
||||||
<main class="container">
|
<main class="container">
|
||||||
<header>
|
<header>
|
||||||
<h1 class="mt-5" style="font-family: 'Covered By Your Grace'">
|
<%- include('./includes/breadcrumbs.ejs') %>
|
||||||
<a href="/"><%=h.getSiteName()%></a>
|
|
||||||
<span class="separator">› </span>
|
|
||||||
Search Error<% if ((typeof query != undefined) && query != '') { %> for <%- query %><% } %>
|
|
||||||
</h1>
|
|
||||||
</header>
|
</header>
|
||||||
<div class="mt-3 p-3">
|
<div class="mt-3 p-3">
|
||||||
<p>
|
<p>
|
||||||
|
@ -34,12 +30,6 @@
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<%- include('./includes/bottom-navbar.ejs') %>
|
<%- include('./includes/bottom-navbar.ejs') %>
|
||||||
|
<%- include('./includes/bottom-scripts.ejs') %>
|
||||||
<!-- Bootstrap JS (optional, if you need Bootstrap JS features) -->
|
|
||||||
<script src="https://daball.me/vendor/jquery/jquery.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/popper.js/dist/popper.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/jquery-easing/jquery.easing.min.js"></script>
|
|
||||||
<script src="https://daball.me/layouts/blog/js/blog.min.js"></script>
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
|
@ -9,11 +9,7 @@
|
||||||
<%- include('./includes/no-trash-svg.ejs') %>
|
<%- include('./includes/no-trash-svg.ejs') %>
|
||||||
<main class="container">
|
<main class="container">
|
||||||
<header>
|
<header>
|
||||||
<h1 class="mt-5" style="font-family: 'Covered By Your Grace'">
|
<%- include('./includes/breadcrumbs.ejs') %>
|
||||||
<a href="/"><%=h.getSiteName()%></a>
|
|
||||||
<span class="separator">› </span>
|
|
||||||
Search Results for <%- query %>
|
|
||||||
</h1>
|
|
||||||
</header>
|
</header>
|
||||||
<div class="container">
|
<div class="container">
|
||||||
<p>
|
<p>
|
||||||
|
@ -72,12 +68,6 @@
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<%- include('./includes/bottom-navbar.ejs') %>
|
<%- include('./includes/bottom-navbar.ejs') %>
|
||||||
|
<%- include('./includes/bottom-scripts.ejs') %>
|
||||||
<!-- Bootstrap JS (optional, if you need Bootstrap JS features) -->
|
|
||||||
<script src="https://daball.me/vendor/jquery/jquery.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/popper.js/dist/popper.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/jquery-easing/jquery.easing.min.js"></script>
|
|
||||||
<script src="https://daball.me/layouts/blog/js/blog.min.js"></script>
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
|
@ -62,7 +62,7 @@
|
||||||
<a href="<%=encodeURI(videoURL)%>" target="_blank">Video (.<%=info.ext%>)</a>
|
<a href="<%=encodeURI(videoURL)%>" target="_blank">Video (.<%=info.ext%>)</a>
|
||||||
| <a href="<%=encodeURI(subtitleURL)%>" target="_blank">Subtitles (.vtt)</a>
|
| <a href="<%=encodeURI(subtitleURL)%>" target="_blank">Subtitles (.vtt)</a>
|
||||||
</p>
|
</p>
|
||||||
<!-- <pre><%=inspect(info)%></pre> -->
|
<!-- <pre><%=h.inspect(info)%></pre> -->
|
||||||
<%}%>
|
<%}%>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -87,12 +87,6 @@
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<%- include('./includes/bottom-navbar.ejs') %>
|
<%- include('./includes/bottom-navbar.ejs') %>
|
||||||
|
<%- include('./includes/bottom-scripts.ejs') %>
|
||||||
<!-- Bootstrap JS (optional, if you need Bootstrap JS features) -->
|
|
||||||
<script src="https://daball.me/vendor/jquery/jquery.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/popper.js/dist/popper.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
|
|
||||||
<script src="https://daball.me/vendor/jquery-easing/jquery.easing.min.js"></script>
|
|
||||||
<script src="https://daball.me/layouts/blog/js/blog.min.js"></script>
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
|
@ -6,13 +6,13 @@ import fs, { WriteStream } from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import crypto from 'crypto';
|
import crypto from 'crypto';
|
||||||
import url from 'url';
|
import url from 'url';
|
||||||
import { TikaClient } from './app/TikaClient/index.mjs';
|
import { TikaClient } from './app/tika/index.mjs';
|
||||||
import { Readable, Writable } from 'stream';
|
import { Readable, Writable } from 'stream';
|
||||||
import dotenv from 'dotenv';
|
import dotenv from 'dotenv';
|
||||||
import dotenvExpand from 'dotenv-expand';
|
import dotenvExpand from 'dotenv-expand';
|
||||||
import process from 'process';
|
import process from 'process';
|
||||||
import { config } from './app/config.mjs';
|
import { config } from './app/config.mjs';
|
||||||
import { SolrDocument } from './app/models/solr-doc.mjs';
|
import { SolrDocument } from './app/search/solr-doc.mjs';
|
||||||
// import ts from 'gulp-typescript';
|
// import ts from 'gulp-typescript';
|
||||||
import run from 'gulp-run';
|
import run from 'gulp-run';
|
||||||
import { deleteAsync } from 'del';
|
import { deleteAsync } from 'del';
|
||||||
|
|
Loading…
Reference in New Issue
Block a user