diff --git a/README.md b/README.md index 533e6227..4804b461 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Solr Test URL: [http://localhost:8983/](http://localhost:8983/) Tika Test URL: [http://localhost:9998/](http://localhost:9998/) -In case of trouble with accessing them from outside the localhost, please check and ensure the exposed Solr port is allowed through your firewall for the web server host to access. Please permit the Tika and the Solr port through your firewall in order to for any npm workers to request the plaintext for documents. +In case of trouble with accessing them from outside the localhost, please check and ensure the exposed Solr port is allowed through your firewall for the web server host to access. Please permit the Tika and the Solr port through your firewall for any npm workers that need to request the plaintext for documents outside of Docker and submit them to the Solr search index. # Installing ffmpeg diff --git a/app/server.js b/app/server.js index af2b9802..569f1566 100644 --- a/app/server.js +++ b/app/server.js @@ -9,6 +9,7 @@ const ejs = require('ejs'); const helpers = require('../views/helpers/functions'); const search = require('../routes/search'); const fs = require('fs'); +require('dotenv').config(); // const advancedSearch = require('../routes/advanced-search'); // Port number for HTTP server diff --git a/docker/fullstack/docker-compose.yml b/docker/fullstack/docker-compose.yml index d8983909..d6199c6d 100644 --- a/docker/fullstack/docker-compose.yml +++ b/docker/fullstack/docker-compose.yml @@ -35,6 +35,8 @@ services: nm3clol: build: ../nm3clol.Dockerfile container_name: nm3clol + environment: + - PORT=$APP_HTTP_LISTEN_PORT ports: - "3000:3000" diff --git a/docker/nm3clol-caddy/docker-compose.yml b/docker/nm3clol-caddy/docker-compose.yml index 29004e70..d9cba0d7 100644 --- a/docker/nm3clol-caddy/docker-compose.yml +++ b/docker/nm3clol-caddy/docker-compose.yml @@ -4,6 +4,8 @@ services: nm3clol: build: ../nm3clol.Dockerfile container_name: nm3clol + environment: + - PORT=$APP_HTTP_LISTEN_PORT ports: - "3000:3000" diff --git a/docker/nm3clol/docker-compose.yml b/docker/nm3clol/docker-compose.yml index aeb6a282..ac2b396c 100644 --- a/docker/nm3clol/docker-compose.yml +++ b/docker/nm3clol/docker-compose.yml @@ -4,5 +4,7 @@ services: nm3clol: build: ../nm3clol.Dockerfile container_name: nm3clol + environment: + - PORT=$APP_HTTP_LISTEN_PORT ports: - "3000:3000" diff --git a/docker/solr-data/vacode_core/managed-schema.xml b/docker/solr-data/vacode_core/conf/managed-schema.xml similarity index 100% rename from docker/solr-data/vacode_core/managed-schema.xml rename to docker/solr-data/vacode_core/conf/managed-schema.xml diff --git a/docker/solr-data/vacode_core/protwords.txt b/docker/solr-data/vacode_core/conf/protwords.txt similarity index 100% rename from docker/solr-data/vacode_core/protwords.txt rename to docker/solr-data/vacode_core/conf/protwords.txt diff --git a/docker/solr-data/vacode_core/schema.xml b/docker/solr-data/vacode_core/conf/schema.xml similarity index 100% rename from docker/solr-data/vacode_core/schema.xml rename to docker/solr-data/vacode_core/conf/schema.xml diff --git a/docker/solr-data/vacode_core/solrconfig.xml b/docker/solr-data/vacode_core/conf/solrconfig.xml similarity index 100% rename from docker/solr-data/vacode_core/solrconfig.xml rename to docker/solr-data/vacode_core/conf/solrconfig.xml diff --git a/docker/solr-data/vacode_core/stopwords.txt b/docker/solr-data/vacode_core/conf/stopwords.txt similarity index 100% rename from docker/solr-data/vacode_core/stopwords.txt rename to docker/solr-data/vacode_core/conf/stopwords.txt diff --git a/docker/solr-data/vacode_core/synonyms.txt b/docker/solr-data/vacode_core/conf/synonyms.txt similarity index 100% rename from docker/solr-data/vacode_core/synonyms.txt rename to docker/solr-data/vacode_core/conf/synonyms.txt diff --git a/gulpfile.js b/gulpfile.js index 53c6a53f..ca396d9c 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -8,17 +8,31 @@ const crypto = require('crypto'); const url = require('url') const { TikaClient } = require('./app/TikaClient/build'); const { Readable, Writable } = require('stream'); +require('dotenv').config(); const relPathToFiles = './public'; -const baseUrl = 'https://no-moss-3-carbo-landfill-library.online'; // URL of the document to download and index -const tikaUrl = 'http://solr.services.cleveland.daball.me:9998'; // URL of the Tika instance -const solrUrl = 'http://solr.services.cleveland.daball.me:8983/solr/my_core'; // URL of your Solr instance -const solrVirginiaLawUrl = 'http://solr.services.cleveland.daball.me:8983/solr/va_code'; // URL of your Solr instance + +const defaultSiteUrl = 'http://localhost'; +const defaultTikaUrl = 'http://tika:9998'; +const defaultSolrDocsUrl = 'http://solr:8983/solr/nm3clol_core'; +const defaultSolrLawUrl = 'http://solr:8983/solr/vacode_core'; +const baseUrl = (process.env.SITE_URL||defaultSiteUrl).endsWith('/') + ? (process.env.SITE_URL||defaultSiteUrl).substring(0, (process.env.SITE_URL||defaultSiteUrl).length - 2) + : process.env.SITE_URL||defaultSiteUrl; // URL of the document to download and index +const tikaUrl = (process.env.TIKA_URL||defaultTikaUrl).endsWith('/') + ? (process.env.TIKA_URL||defaultTikaUrl).substring(0, (process.env.TIKA_URL||defaultTikaUrl).length - 2) + : process.env.TIKA_URL||defaultTikaUrl; // URL of the Tika instance +const solrDocsUrl = (process.env.SOLR_DOCS_URL||defaultSolrDocsUrl).endsWith('/') + ? (process.env.SOLR_DOCS_URL||defaultSolrDocsUrl).substring(0, (process.env.SOLR_DOCS_URL||defaultSolrDocsUrl).length - 2) + : process.env.SOLR_DOCS_URL||defaultSolrDocsUrl; // URL of your Solr instance +const solrLawUrl = (process.env.SOLR_LAW_URL||defaultSolrLawUrl).endsWith('/') + ? (process.env.SOLR_LAW_URL||defaultSolrLawUrl).substring(0, (process.env.SOLR_LAW_URL||defaultSolrLawUrl).length - 2) + : process.env.SOLR_LAW_URL||defaultSolrLawUrl; // URL of your Solr instance // Task to clear out previous Solr data gulp.task('index:clear', async () => { await request({ - uri: `${solrUrl}/update?commit=true`, + uri: `${solrDocsUrl}/update?commit=true`, method: 'POST', body: { delete: { query: '*:*' } }, // Delete all documents json: true, @@ -27,7 +41,7 @@ gulp.task('index:clear', async () => { gulp.task('dbfromsolr', async () => { let docs = await request({ - uri: `${solrUrl}/select`, + uri: `${solrDocsUrl}/select`, qs: { q: '*:*', wt: 'json', @@ -79,7 +93,7 @@ async function retrieveVirginiaLawMetadataFromSolr(url) { // }); const fl = encodeURIComponent("sha256sum, content_length"); const q = encodeURIComponent("id:")+"\""+encodeURIComponent(url)+"\"";//encodeURIComponent(`id:"${url}"`); - const uri = `${solrVirginiaLawUrl}/select?q=${q}&fl=${fl}`; + const uri = `${solrLawUrl}/select?q=${q}&fl=${fl}`; const response = await request({ uri: `${uri}`, json: true }); return response && response.response && response.response.docs && response.response.docs[0]; } @@ -92,7 +106,7 @@ async function retrieveMetadataFromSolr(url) { // }); const fl = encodeURIComponent("sha256sum, content_length"); const q = encodeURIComponent("id:")+"\""+encodeURIComponent(url)+"\"";//encodeURIComponent(`id:"${url}"`); - const uri = `${solrUrl}/select?q=${q}&fl=${fl}`; + const uri = `${solrDocsUrl}/select?q=${q}&fl=${fl}`; const response = await request({ uri: `${uri}`, json: true }); return response && response.response && response.response.docs && response.response.docs[0]; } @@ -101,7 +115,7 @@ async function indexDocumentInSolr(document) { try { // Send document to Solr using the Solr REST API or a Solr client library // Example code to send document using Axios: - await axios.post(solrUrl + '/update/json/docs', document, { + await axios.post(solrDocsUrl + '/update/json/docs', document, { params: { commit: true, // Commit changes immediately }, @@ -115,7 +129,7 @@ async function indexLawDocumentInSolr(document) { try { // Send document to Solr using the Solr REST API or a Solr client library // Example code to send document using Axios: - await axios.post(solrVirginiaLawUrl + '/update/json/docs', document, { + await axios.post(solrLawUrl + '/update/json/docs', document, { params: { commit: true, // Commit changes immediately }, @@ -168,7 +182,7 @@ gulp.task('index:laws', async () => { const fileFullPath = path.join(cwd, file); - const url = `https://no-moss-3-carbo-landfill-library.online/${file.replaceAll(path.sep, '/')}`; + const url = `${baseUrl}/${file.replaceAll(path.sep, '/')}`; console.log('URL: ' + url); // Retrieve metadata of the file from Solr (if it exists) @@ -267,7 +281,7 @@ gulp.task('index:docs', async () => { const fileFullPath = path.join(cwd, file); - let url = `https://no-moss-3-carbo-landfill-library.online/${file.replaceAll(path.sep, '/')}`; + let url = `${baseUrl}/${file.replaceAll(path.sep, '/')}`; console.log('URL: ' + url); // Retrieve metadata of the file from Solr (if it exists) diff --git a/package-lock.json b/package-lock.json index c77c985b..2d7e9b18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "bytes": "3.0.0", "cheerio": "^1.0.0-rc.12", "content-disposition": "0.5.2", + "dotenv": "^16.4.5", "ejs": "^3.1.9", "express": "^4.18.3", "fast-url-parser": "1.1.3", @@ -5444,6 +5445,17 @@ "node": ">=8" } }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", diff --git a/package.json b/package.json index f4ed7a19..8af25d42 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "bytes": "3.0.0", "cheerio": "^1.0.0-rc.12", "content-disposition": "0.5.2", + "dotenv": "^16.4.5", "ejs": "^3.1.9", "express": "^4.18.3", "fast-url-parser": "1.1.3", diff --git a/routes/search.js b/routes/search.js index 62a73cfa..e660167f 100644 --- a/routes/search.js +++ b/routes/search.js @@ -2,7 +2,7 @@ const express = require('express'); const router = express.Router(); const { parse, toString } = require('lucene'); const { createClient, Query } = require('solr-client'); -const solrConfig = { host: 'solr.services.cleveland.daball.me', port: 8983, core: 'my_core' }; +const solrConfig = { host: process.env.SOLR_DOCS_HOST||'solr', port: process.env.SOLR_DOCS_PORT||8983, core: process.env.SOLR_DOCS_CORE_NAME||'nm3clol_core' }; const helpers = require('../views/helpers/functions'); router.get('/', (req, res) => { diff --git a/views/helpers/functions.js b/views/helpers/functions.js index 7cf21a69..88045add 100644 --- a/views/helpers/functions.js +++ b/views/helpers/functions.js @@ -19,7 +19,7 @@ const md = markdownit({ const moment = require('moment-timezone').tz.setDefault("UTC"); const getSiteName = () => { - return 'No Moss 3 Carbo Landfill Online Library'; + return process.env.SITE_NAME||'No Moss 3 Carbo Landfill Online Library'; } const trimSlashes = ({path}) => { @@ -89,7 +89,7 @@ const renderArchive = (html, paths) => { // Header and Footer content const headHeaderContent = ``; const headFooterContent = ` - +