From 555e9f52bd89f461d9433b1c0d1172cce4399971 Mon Sep 17 00:00:00 2001 From: David Ball Date: Fri, 24 May 2024 02:16:13 -0400 Subject: [PATCH] Updated scripts, YouTube cookies example, docker compose configuration, and documentation. --- README.md | 193 +++++++++++++++++- docker/README.md | 15 ++ .../TesseractOCRConfig.properties | 0 {solr => docker}/solr-data/log4j2.xml | 0 .../conf/lang/contractions_ca.txt | 0 .../conf/lang/contractions_fr.txt | 0 .../conf/lang/contractions_ga.txt | 0 .../conf/lang/contractions_it.txt | 0 .../nm3clol_core/conf/lang/hypenations_ga.txt | 0 .../nm3clol_core/conf/lang/stemdict_nl.txt | 0 .../nm3clol_core/conf/lang/stoptags_ja.txt | 0 .../nm3clol_core/conf/lang/stopwords_ar.txt | 0 .../nm3clol_core/conf/lang/stopwords_bg.txt | 0 .../nm3clol_core/conf/lang/stopwords_ca.txt | 0 .../nm3clol_core/conf/lang/stopwords_cz.txt | 0 .../nm3clol_core/conf/lang/stopwords_da.txt | 0 .../nm3clol_core/conf/lang/stopwords_de.txt | 0 .../nm3clol_core/conf/lang/stopwords_el.txt | 0 .../nm3clol_core/conf/lang/stopwords_en.txt | 0 .../nm3clol_core/conf/lang/stopwords_es.txt | 0 .../nm3clol_core/conf/lang/stopwords_et.txt | 0 .../nm3clol_core/conf/lang/stopwords_eu.txt | 0 .../nm3clol_core/conf/lang/stopwords_fa.txt | 0 .../nm3clol_core/conf/lang/stopwords_fi.txt | 0 .../nm3clol_core/conf/lang/stopwords_fr.txt | 0 .../nm3clol_core/conf/lang/stopwords_ga.txt | 0 .../nm3clol_core/conf/lang/stopwords_gl.txt | 0 .../nm3clol_core/conf/lang/stopwords_hi.txt | 0 .../nm3clol_core/conf/lang/stopwords_hu.txt | 0 .../nm3clol_core/conf/lang/stopwords_hy.txt | 0 .../nm3clol_core/conf/lang/stopwords_id.txt | 0 .../nm3clol_core/conf/lang/stopwords_it.txt | 0 .../nm3clol_core/conf/lang/stopwords_ja.txt | 0 .../nm3clol_core/conf/lang/stopwords_lv.txt | 0 .../nm3clol_core/conf/lang/stopwords_nl.txt | 0 .../nm3clol_core/conf/lang/stopwords_no.txt | 0 .../nm3clol_core/conf/lang/stopwords_pt.txt | 0 .../nm3clol_core/conf/lang/stopwords_ro.txt | 0 .../nm3clol_core/conf/lang/stopwords_ru.txt | 0 .../nm3clol_core/conf/lang/stopwords_sv.txt | 0 .../nm3clol_core/conf/lang/stopwords_th.txt | 0 .../nm3clol_core/conf/lang/stopwords_tr.txt | 0 .../nm3clol_core/conf/managed-schema.xml | 0 .../solr-data/nm3clol_core/conf/protwords.txt | 0 .../solr-data/nm3clol_core/conf/schema.xml | 0 .../nm3clol_core/conf/solrconfig.xml | 0 .../solr-data/nm3clol_core/conf/stopwords.txt | 0 .../solr-data/nm3clol_core/conf/synonyms.txt | 0 .../solr-data/nm3clol_core/core.properties | 0 .../vacode_core/conf/lang/contractions_ca.txt | 0 .../vacode_core/conf/lang/contractions_fr.txt | 0 .../vacode_core/conf/lang/contractions_ga.txt | 0 .../vacode_core/conf/lang/contractions_it.txt | 0 .../vacode_core/conf/lang/hypenations_ga.txt | 0 .../vacode_core/conf/lang/stemdict_nl.txt | 0 .../vacode_core/conf/lang/stoptags_ja.txt | 0 .../vacode_core/conf/lang/stopwords_ar.txt | 0 .../vacode_core/conf/lang/stopwords_bg.txt | 0 .../vacode_core/conf/lang/stopwords_ca.txt | 0 .../vacode_core/conf/lang/stopwords_cz.txt | 0 .../vacode_core/conf/lang/stopwords_da.txt | 0 .../vacode_core/conf/lang/stopwords_de.txt | 0 .../vacode_core/conf/lang/stopwords_el.txt | 0 .../vacode_core/conf/lang/stopwords_en.txt | 0 .../vacode_core/conf/lang/stopwords_es.txt | 0 .../vacode_core/conf/lang/stopwords_et.txt | 0 .../vacode_core/conf/lang/stopwords_eu.txt | 0 .../vacode_core/conf/lang/stopwords_fa.txt | 0 .../vacode_core/conf/lang/stopwords_fi.txt | 0 .../vacode_core/conf/lang/stopwords_fr.txt | 0 .../vacode_core/conf/lang/stopwords_ga.txt | 0 .../vacode_core/conf/lang/stopwords_gl.txt | 0 .../vacode_core/conf/lang/stopwords_hi.txt | 0 .../vacode_core/conf/lang/stopwords_hu.txt | 0 .../vacode_core/conf/lang/stopwords_hy.txt | 0 .../vacode_core/conf/lang/stopwords_id.txt | 0 .../vacode_core/conf/lang/stopwords_it.txt | 0 .../vacode_core/conf/lang/stopwords_ja.txt | 0 .../vacode_core/conf/lang/stopwords_lv.txt | 0 .../vacode_core/conf/lang/stopwords_nl.txt | 0 .../vacode_core/conf/lang/stopwords_no.txt | 0 .../vacode_core/conf/lang/stopwords_pt.txt | 0 .../vacode_core/conf/lang/stopwords_ro.txt | 0 .../vacode_core/conf/lang/stopwords_ru.txt | 0 .../vacode_core/conf/lang/stopwords_sv.txt | 0 .../vacode_core/conf/lang/stopwords_th.txt | 0 .../vacode_core/conf/lang/stopwords_tr.txt | 0 .../solr-data/vacode_core/core.properties | 0 .../solr-data/vacode_core/managed-schema.xml | 0 .../solr-data/vacode_core/protwords.txt | 0 .../solr-data/vacode_core/schema.xml | 0 .../solr-data/vacode_core/solrconfig.xml | 0 .../solr-data/vacode_core/stopwords.txt | 0 .../solr-data/vacode_core/synonyms.txt | 0 .../solr-tika/docker-compose.yml | 12 +- docker/solr/docker-compose.yml | 18 ++ {solr => docker}/tika-config.xml | 0 {solr => docker}/tika-data/log4j2.xml | 0 docker/tika/docker-compose.yml | 18 ++ mirror-virginia-law.ps1 | 1 + sync-my-youtube-videos.cmd | 3 - sync-youtube-videos.cmd | 9 +- youtube-cookies.example.txt | 2 +- 103 files changed, 250 insertions(+), 21 deletions(-) create mode 100644 docker/README.md rename {solr => docker}/TesseractOCRConfig.properties (100%) rename {solr => docker}/solr-data/log4j2.xml (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/contractions_ca.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/contractions_fr.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/contractions_ga.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/contractions_it.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/hypenations_ga.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stemdict_nl.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stoptags_ja.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_ar.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_bg.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_ca.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_cz.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_da.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_de.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_el.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_en.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_es.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_et.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_eu.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_fa.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_fi.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_fr.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_ga.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_gl.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_hi.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_hu.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_hy.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_id.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_it.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_ja.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_lv.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_nl.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_no.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_pt.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_ro.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_ru.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_sv.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_th.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/lang/stopwords_tr.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/managed-schema.xml (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/protwords.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/schema.xml (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/solrconfig.xml (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/stopwords.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/conf/synonyms.txt (100%) rename {solr => docker}/solr-data/nm3clol_core/core.properties (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/contractions_ca.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/contractions_fr.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/contractions_ga.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/contractions_it.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/hypenations_ga.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stemdict_nl.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stoptags_ja.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_ar.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_bg.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_ca.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_cz.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_da.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_de.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_el.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_en.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_es.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_et.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_eu.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_fa.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_fi.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_fr.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_ga.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_gl.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_hi.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_hu.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_hy.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_id.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_it.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_ja.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_lv.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_nl.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_no.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_pt.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_ro.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_ru.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_sv.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_th.txt (100%) rename {solr => docker}/solr-data/vacode_core/conf/lang/stopwords_tr.txt (100%) rename {solr => docker}/solr-data/vacode_core/core.properties (100%) rename {solr => docker}/solr-data/vacode_core/managed-schema.xml (100%) rename {solr => docker}/solr-data/vacode_core/protwords.txt (100%) rename {solr => docker}/solr-data/vacode_core/schema.xml (100%) rename {solr => docker}/solr-data/vacode_core/solrconfig.xml (100%) rename {solr => docker}/solr-data/vacode_core/stopwords.txt (100%) rename {solr => docker}/solr-data/vacode_core/synonyms.txt (100%) rename solr/docker-compose.solr-tika-tesseract.yml => docker/solr-tika/docker-compose.yml (71%) create mode 100644 docker/solr/docker-compose.yml rename {solr => docker}/tika-config.xml (100%) rename {solr => docker}/tika-data/log4j2.xml (100%) create mode 100644 docker/tika/docker-compose.yml delete mode 100644 sync-my-youtube-videos.cmd diff --git a/README.md b/README.md index 774fe651..4260b17e 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,75 @@ +# Installing Git + +On Ubuntu you can just do: + + apt install git + +Microsoft Documentation: [https://learn.microsoft.com/en-us/devops/develop/git/install-and-set-up-git](https://learn.microsoft.com/en-us/devops/develop/git/install-and-set-up-git) + +Git SCM Downloads: [https://git-scm.com/downloads](https://git-scm.com/downloads) + +# Installing Git LFS + +This enables the large file storage system. It needs to be done once per user account that uses it. + + git lfs install + +Documentation: [https://git-lfs.com/](https://git-lfs.com/) + # Cloning this Repository You can use git to clone this repository. git clone https://gitea.daball.me/No-Moss-3-Carbo-Landfill-Online-Library/no-moss-3-carbo-landfill-library.online.git -# Installing NPM Application Dependencies +# Pulling Git Source Updates -You can use npm to install the dependencies. +From inside the working directory, you can use git to pull source updates from the origin repository. - npm install + git pull -# Required: `yt-dlp` +# Pulling Git LFS Updates + +From inside the working directory, download all the large file storage updates from the origin server. + + git lfs pull + +# Installing Docker and Docker Compose (optional) + +Installing Docker is beyond the scope of this document. You may choose to install Docker to simplify running this web site. + +[https://docs.docker.com/get-docker/](https://docs.docker.com/get-docker/) + +# Installing Solr with Tika and Tesseract + +Installing Solr with Tika and Tesseract is beyond the scope of this document. It is the search engine I am currently using. + +From inside the solr folder, use Docker Compose bring up Solr and Tika. Tesseract will be installed in the Tika instance. + + docker compose up + +From inside the working directory, go into the solr folder and use Docker Compose to bring up the instances. + + cd solr + docker compose up + +Take note of your Docker hostname. Docker should be exposing port 8983 for the solr instance and port 9998 for the tika instance. + +Solr Test URL: [http://localhost:8983/](http://localhost:8983/) + +Tika Test URL: [http://localhost:9998/](http://localhost:9998/) + +In case of trouble with accessing them from outside the localhost, please check and ensure the exposed Solr port is allowed through your firewall for the web server host to access. Please permit the Tika and the Solr port through your firewall in order to for any npm workers to request the plaintext for documents. + +# Installing ffmpeg + +Before you can re-encode the yt-dlp output, you will need to install ffmpeg and have the `ffmpeg` binary available in your `PATH`. + +[https://ffmpeg.org/download.html](https://ffmpeg.org/download.html) + +# Installing yt-dlp + +You need to install yt-dlp and have the binary `yt-dlp` binary available in your `PATH`. Project: [https://github.com/yt-dlp/yt-dlp](https://github.com/yt-dlp/yt-dlp) @@ -18,6 +77,12 @@ Standalone Binaries: [https://github.com/yt-dlp/yt-dlp#release-files](https://gi Installation: [https://github.com/yt-dlp/yt-dlp/wiki/Installation](https://github.com/yt-dlp/yt-dlp/wiki/Installation) +# *Windows*: Reloading Environment + +In Windows you will need to log out and back in after updating your environment variables. In PowerShell you can reload your `PATH` environment variable using: + + $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") + # Downloading YouTube Archives You may need to use your web browser to login to YouTube. You can then copy each cookie value out of your web browser using the Developer Tools Network tab. Gather the correct values to build a `youtube-cookies.txt` file in your working directory using something like the `youtube-example-cookies.txt` template here: @@ -52,14 +117,122 @@ You can update the copy of the Virginia code: mirror-virginia-law.ps1 -# Install Docker (optional) +# Installing Node.js -Installing Docker is beyond the scope of this document. You may choose to install Docker to simplify running this web site. +In order to run the web app server, you must have Node.js installed. -# Install Solr with Tika and Tesseract +[https://nodejs.org/](https://nodejs.org/) -Installing Solr with Tika and Tesseract is beyond the scope of this document. It is the search engine I use. +# Installing NPM -**TODO**: Document installation with Docker. +Your Node.js should come with NPM. If you don't have it, see these directions: -**TODO**: Finish these instructions. +[https://docs.npmjs.com/downloading-and-installing-node-js-and-npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) + +# Installing Web App Server Dependencies + +You can use npm to install the web app server dependencies. + + npm install + +# Before Running the Node Server + +You may need to transpile the TypeScript to JavaScript. + + npm run-script transpile:ts + +# (Optional) Clearing the Search Index + +You can clear the search index using npm: + + npm run-script index:clear + +# (Optional) Rebuilding the Search Index + +You can clear and rebuild the search index in one go using npm: + + npm run-script index:reindex + +# Incrementally Building the Document Search Index + +You can scan all of the documents into the search index incrementally using npm. If the index is empty, all the documents will be scanned. If the file exists in the index, it's hash is checked before scanning the document's text. The presumption is that a second scan would produce the same text, and the scan is computationally expensive. Using npm, incrementally build the search index: + + npm run-script index:docs + +# Incrementally Building the Laws Search Index + +You can scan all of the laws into the search index incrementally using npm. If the index is empty, all the documents will be scanned. If the file exists in the index, it's hash is checked before scanning the document's text. The presumption is that a second scan would produce the same text, and the scan is computationally expensive. Using npm, incrementally build the search index: + + npm run-script index:laws + +# Running the Node Server + +You can run the web server using npm. + + npm run-script server + +Or you can use Node.js directly. + + node app/server.js + +# Reverse Proxy using Caddy + +If you want to use Caddy as a reverse proxy to the web application, try a Caddyfile like this: + + www.no-moss-3-carbo-landfill-library.online { + redir https://no-moss-3-carbo-landfill-library.online{uri} + } + + no-moss-3-carbo-landfill-library.online { + reverse_proxy localhost:3000 + } + +# Reverse Proxy using Nginx + +If you want to use Nginx as a reverse proxy to the web application, try a configuration file like this: + + server { + listen 80; + server_name www.no-moss-3-carbo-landfill-library.online; + location / { + rewrite ^/(.*) http://no-moss-3-carbo-landfill-library.online/$1 permanent; + } + } + server { + listen 80; + server_name no-moss-3-carbo-landfill-library.online; + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + } + +# Securing Nginx + +If you want a free SSL certification for Nginx try the certbot to keep your certificates up-to-date. + +[https://letsencrypt.org/docs/client-options/](https://letsencrypt.org/docs/client-options/) + +# Reverse Proxy Using IISNode + +If you want to use IIS to access the Node instance, this can be greatly simplified by using IISNode. + +IISNode requires the URL_rewrite module for IIS. + +[https://iis-umbraco.azurewebsites.net/downloads/microsoft/url-rewrite](https://iis-umbraco.azurewebsites.net/downloads/microsoft/url-rewrite) + +Then you can set up IISNode. + +[https://github.com/Azure/iisnode](https://github.com/Azure/iisnode) + +The included web.config is an example configuration for IISNode. + +# Post-Installation Considerations + +If running the web server application outside of Docker or IISNode, consider using a daemonizer such as PM2. + +[https://pm2.io/docs/runtime/guide/installation/](https://pm2.io/docs/runtime/guide/installation/) diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..b22b7e6d --- /dev/null +++ b/docker/README.md @@ -0,0 +1,15 @@ +# Docker Compose configurations + +Here are some Docker Compose configuration examples. + +## solr + +Minimal Solr instance. + +## solr-tika + +Solr and Tika with Tesseract together. + +## tika + +Minimal Tika with Tesseract instance. diff --git a/solr/TesseractOCRConfig.properties b/docker/TesseractOCRConfig.properties similarity index 100% rename from solr/TesseractOCRConfig.properties rename to docker/TesseractOCRConfig.properties diff --git a/solr/solr-data/log4j2.xml b/docker/solr-data/log4j2.xml similarity index 100% rename from solr/solr-data/log4j2.xml rename to docker/solr-data/log4j2.xml diff --git a/solr/solr-data/nm3clol_core/conf/lang/contractions_ca.txt b/docker/solr-data/nm3clol_core/conf/lang/contractions_ca.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/contractions_ca.txt rename to docker/solr-data/nm3clol_core/conf/lang/contractions_ca.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/contractions_fr.txt b/docker/solr-data/nm3clol_core/conf/lang/contractions_fr.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/contractions_fr.txt rename to docker/solr-data/nm3clol_core/conf/lang/contractions_fr.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/contractions_ga.txt b/docker/solr-data/nm3clol_core/conf/lang/contractions_ga.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/contractions_ga.txt rename to docker/solr-data/nm3clol_core/conf/lang/contractions_ga.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/contractions_it.txt b/docker/solr-data/nm3clol_core/conf/lang/contractions_it.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/contractions_it.txt rename to docker/solr-data/nm3clol_core/conf/lang/contractions_it.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/hypenations_ga.txt b/docker/solr-data/nm3clol_core/conf/lang/hypenations_ga.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/hypenations_ga.txt rename to docker/solr-data/nm3clol_core/conf/lang/hypenations_ga.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stemdict_nl.txt b/docker/solr-data/nm3clol_core/conf/lang/stemdict_nl.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stemdict_nl.txt rename to docker/solr-data/nm3clol_core/conf/lang/stemdict_nl.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stoptags_ja.txt b/docker/solr-data/nm3clol_core/conf/lang/stoptags_ja.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stoptags_ja.txt rename to docker/solr-data/nm3clol_core/conf/lang/stoptags_ja.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_ar.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_ar.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_ar.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_ar.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_bg.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_bg.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_bg.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_bg.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_ca.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_ca.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_ca.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_ca.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_cz.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_cz.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_cz.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_cz.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_da.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_da.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_da.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_da.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_de.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_de.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_de.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_de.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_el.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_el.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_el.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_el.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_en.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_en.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_en.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_en.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_es.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_es.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_es.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_es.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_et.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_et.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_et.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_et.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_eu.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_eu.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_eu.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_eu.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_fa.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_fa.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_fa.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_fa.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_fi.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_fi.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_fi.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_fi.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_fr.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_fr.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_fr.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_fr.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_ga.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_ga.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_ga.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_ga.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_gl.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_gl.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_gl.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_gl.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_hi.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_hi.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_hi.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_hi.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_hu.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_hu.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_hu.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_hu.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_hy.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_hy.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_hy.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_hy.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_id.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_id.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_id.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_id.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_it.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_it.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_it.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_it.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_ja.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_ja.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_ja.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_ja.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_lv.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_lv.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_lv.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_lv.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_nl.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_nl.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_nl.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_nl.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_no.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_no.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_no.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_no.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_pt.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_pt.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_pt.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_pt.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_ro.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_ro.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_ro.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_ro.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_ru.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_ru.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_ru.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_ru.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_sv.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_sv.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_sv.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_sv.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_th.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_th.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_th.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_th.txt diff --git a/solr/solr-data/nm3clol_core/conf/lang/stopwords_tr.txt b/docker/solr-data/nm3clol_core/conf/lang/stopwords_tr.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/lang/stopwords_tr.txt rename to docker/solr-data/nm3clol_core/conf/lang/stopwords_tr.txt diff --git a/solr/solr-data/nm3clol_core/conf/managed-schema.xml b/docker/solr-data/nm3clol_core/conf/managed-schema.xml similarity index 100% rename from solr/solr-data/nm3clol_core/conf/managed-schema.xml rename to docker/solr-data/nm3clol_core/conf/managed-schema.xml diff --git a/solr/solr-data/nm3clol_core/conf/protwords.txt b/docker/solr-data/nm3clol_core/conf/protwords.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/protwords.txt rename to docker/solr-data/nm3clol_core/conf/protwords.txt diff --git a/solr/solr-data/nm3clol_core/conf/schema.xml b/docker/solr-data/nm3clol_core/conf/schema.xml similarity index 100% rename from solr/solr-data/nm3clol_core/conf/schema.xml rename to docker/solr-data/nm3clol_core/conf/schema.xml diff --git a/solr/solr-data/nm3clol_core/conf/solrconfig.xml b/docker/solr-data/nm3clol_core/conf/solrconfig.xml similarity index 100% rename from solr/solr-data/nm3clol_core/conf/solrconfig.xml rename to docker/solr-data/nm3clol_core/conf/solrconfig.xml diff --git a/solr/solr-data/nm3clol_core/conf/stopwords.txt b/docker/solr-data/nm3clol_core/conf/stopwords.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/stopwords.txt rename to docker/solr-data/nm3clol_core/conf/stopwords.txt diff --git a/solr/solr-data/nm3clol_core/conf/synonyms.txt b/docker/solr-data/nm3clol_core/conf/synonyms.txt similarity index 100% rename from solr/solr-data/nm3clol_core/conf/synonyms.txt rename to docker/solr-data/nm3clol_core/conf/synonyms.txt diff --git a/solr/solr-data/nm3clol_core/core.properties b/docker/solr-data/nm3clol_core/core.properties similarity index 100% rename from solr/solr-data/nm3clol_core/core.properties rename to docker/solr-data/nm3clol_core/core.properties diff --git a/solr/solr-data/vacode_core/conf/lang/contractions_ca.txt b/docker/solr-data/vacode_core/conf/lang/contractions_ca.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/contractions_ca.txt rename to docker/solr-data/vacode_core/conf/lang/contractions_ca.txt diff --git a/solr/solr-data/vacode_core/conf/lang/contractions_fr.txt b/docker/solr-data/vacode_core/conf/lang/contractions_fr.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/contractions_fr.txt rename to docker/solr-data/vacode_core/conf/lang/contractions_fr.txt diff --git a/solr/solr-data/vacode_core/conf/lang/contractions_ga.txt b/docker/solr-data/vacode_core/conf/lang/contractions_ga.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/contractions_ga.txt rename to docker/solr-data/vacode_core/conf/lang/contractions_ga.txt diff --git a/solr/solr-data/vacode_core/conf/lang/contractions_it.txt b/docker/solr-data/vacode_core/conf/lang/contractions_it.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/contractions_it.txt rename to docker/solr-data/vacode_core/conf/lang/contractions_it.txt diff --git a/solr/solr-data/vacode_core/conf/lang/hypenations_ga.txt b/docker/solr-data/vacode_core/conf/lang/hypenations_ga.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/hypenations_ga.txt rename to docker/solr-data/vacode_core/conf/lang/hypenations_ga.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stemdict_nl.txt b/docker/solr-data/vacode_core/conf/lang/stemdict_nl.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stemdict_nl.txt rename to docker/solr-data/vacode_core/conf/lang/stemdict_nl.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stoptags_ja.txt b/docker/solr-data/vacode_core/conf/lang/stoptags_ja.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stoptags_ja.txt rename to docker/solr-data/vacode_core/conf/lang/stoptags_ja.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_ar.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_ar.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_ar.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_ar.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_bg.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_bg.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_bg.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_bg.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_ca.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_ca.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_ca.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_ca.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_cz.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_cz.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_cz.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_cz.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_da.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_da.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_da.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_da.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_de.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_de.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_de.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_de.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_el.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_el.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_el.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_el.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_en.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_en.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_en.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_en.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_es.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_es.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_es.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_es.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_et.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_et.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_et.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_et.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_eu.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_eu.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_eu.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_eu.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_fa.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_fa.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_fa.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_fa.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_fi.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_fi.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_fi.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_fi.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_fr.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_fr.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_fr.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_fr.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_ga.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_ga.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_ga.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_ga.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_gl.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_gl.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_gl.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_gl.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_hi.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_hi.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_hi.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_hi.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_hu.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_hu.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_hu.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_hu.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_hy.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_hy.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_hy.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_hy.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_id.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_id.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_id.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_id.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_it.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_it.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_it.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_it.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_ja.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_ja.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_ja.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_ja.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_lv.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_lv.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_lv.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_lv.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_nl.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_nl.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_nl.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_nl.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_no.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_no.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_no.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_no.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_pt.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_pt.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_pt.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_pt.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_ro.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_ro.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_ro.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_ro.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_ru.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_ru.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_ru.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_ru.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_sv.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_sv.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_sv.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_sv.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_th.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_th.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_th.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_th.txt diff --git a/solr/solr-data/vacode_core/conf/lang/stopwords_tr.txt b/docker/solr-data/vacode_core/conf/lang/stopwords_tr.txt similarity index 100% rename from solr/solr-data/vacode_core/conf/lang/stopwords_tr.txt rename to docker/solr-data/vacode_core/conf/lang/stopwords_tr.txt diff --git a/solr/solr-data/vacode_core/core.properties b/docker/solr-data/vacode_core/core.properties similarity index 100% rename from solr/solr-data/vacode_core/core.properties rename to docker/solr-data/vacode_core/core.properties diff --git a/solr/solr-data/vacode_core/managed-schema.xml b/docker/solr-data/vacode_core/managed-schema.xml similarity index 100% rename from solr/solr-data/vacode_core/managed-schema.xml rename to docker/solr-data/vacode_core/managed-schema.xml diff --git a/solr/solr-data/vacode_core/protwords.txt b/docker/solr-data/vacode_core/protwords.txt similarity index 100% rename from solr/solr-data/vacode_core/protwords.txt rename to docker/solr-data/vacode_core/protwords.txt diff --git a/solr/solr-data/vacode_core/schema.xml b/docker/solr-data/vacode_core/schema.xml similarity index 100% rename from solr/solr-data/vacode_core/schema.xml rename to docker/solr-data/vacode_core/schema.xml diff --git a/solr/solr-data/vacode_core/solrconfig.xml b/docker/solr-data/vacode_core/solrconfig.xml similarity index 100% rename from solr/solr-data/vacode_core/solrconfig.xml rename to docker/solr-data/vacode_core/solrconfig.xml diff --git a/solr/solr-data/vacode_core/stopwords.txt b/docker/solr-data/vacode_core/stopwords.txt similarity index 100% rename from solr/solr-data/vacode_core/stopwords.txt rename to docker/solr-data/vacode_core/stopwords.txt diff --git a/solr/solr-data/vacode_core/synonyms.txt b/docker/solr-data/vacode_core/synonyms.txt similarity index 100% rename from solr/solr-data/vacode_core/synonyms.txt rename to docker/solr-data/vacode_core/synonyms.txt diff --git a/solr/docker-compose.solr-tika-tesseract.yml b/docker/solr-tika/docker-compose.yml similarity index 71% rename from solr/docker-compose.solr-tika-tesseract.yml rename to docker/solr-tika/docker-compose.yml index 4baf4c75..8decc97b 100644 --- a/solr/docker-compose.solr-tika-tesseract.yml +++ b/docker/solr-tika/docker-compose.yml @@ -8,7 +8,7 @@ services: ports: - "8983:8983" volumes: - - ./solr-data:/var/solr + - ../solr-data:/var/solr # environment: # - SOLR_CORE=my_core command: @@ -22,13 +22,13 @@ services: restart: unless-stopped entrypoint: [ "/bin/sh", "-c", "exec java -cp \"/customocr:/tika-server-standard-2.9.2.jar:/tika-extras/*\" org.apache.tika.server.core.TikaServerCli -h 0.0.0.0 $$0 $$@"] environment: - - TIKA_LOG_PATH=/tika.log + - TIKA_LOG_PATH=/tika-logs command: -c /tika-config.xml volumes: - - ./tika-config.xml:/tika-config.xml - - ./tika-data/logs/tika.log:/tika.log - - ./tika-data/log4j2.xml:/log4j2.xml - - ./TesseractOCRConfig.properties:/TesseractOCRConfig.properties + - ../tika-config.xml:/tika-config.xml + - ../tika-data/logs:/tika-logs + - ../tika-data/log4j2.xml:/log4j2.xml + - ../TesseractOCRConfig.properties:/TesseractOCRConfig.properties ports: - "9998:9998" diff --git a/docker/solr/docker-compose.yml b/docker/solr/docker-compose.yml new file mode 100644 index 00000000..cef920d1 --- /dev/null +++ b/docker/solr/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + solr: + image: solr:latest + container_name: solr + restart: unless-stopped + ports: + - "8983:8983" + volumes: + - ../solr-data:/var/solr +# environment: +# - SOLR_CORE=my_core + command: + - solr-precreate +# - gettingstarted + user: "1000:995" + diff --git a/solr/tika-config.xml b/docker/tika-config.xml similarity index 100% rename from solr/tika-config.xml rename to docker/tika-config.xml diff --git a/solr/tika-data/log4j2.xml b/docker/tika-data/log4j2.xml similarity index 100% rename from solr/tika-data/log4j2.xml rename to docker/tika-data/log4j2.xml diff --git a/docker/tika/docker-compose.yml b/docker/tika/docker-compose.yml new file mode 100644 index 00000000..cef920d1 --- /dev/null +++ b/docker/tika/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + solr: + image: solr:latest + container_name: solr + restart: unless-stopped + ports: + - "8983:8983" + volumes: + - ../solr-data:/var/solr +# environment: +# - SOLR_CORE=my_core + command: + - solr-precreate +# - gettingstarted + user: "1000:995" + diff --git a/mirror-virginia-law.ps1 b/mirror-virginia-law.ps1 index 939c7584..3cdf10c4 100644 --- a/mirror-virginia-law.ps1 +++ b/mirror-virginia-law.ps1 @@ -2,6 +2,7 @@ mkdir public cd public mkdir Virginia_Law_Library cd Virginia_Law_Library +echo Downloading Virginia Law Library. wget https://law.lis.virginia.gov/library/constitution/constitution.epub -OutFile constitution.epub wget https://law.lis.virginia.gov/library/constitution/constitution.pdf -OutFile constitution.pdf wget https://law.lis.virginia.gov/CSV/Constitution.csv -OutFile Constitution.csv diff --git a/sync-my-youtube-videos.cmd b/sync-my-youtube-videos.cmd deleted file mode 100644 index 3cedb189..00000000 --- a/sync-my-youtube-videos.cmd +++ /dev/null @@ -1,3 +0,0 @@ -S:\bin\yt-dlp.exe --live-from-start --yes-playlist -N 8 -R infinite -c --no-force-overwrites --mtime --write-description --write-info-json --write-playlist-metafiles --write-comments --no-cookies-from-browser --cookies S:\srv\www\no-moss-3-carbo-landfill-library.online\youtube-cookies.txt --write-thumbnail --write-all-thumbnails --write-url-link --write-webloc-link --write-desktop-link --progress --video-multistreams --audio-multistreams --write-subs --write-auto-subs --embed-subs --embed-thumbnail --embed-metadata --embed-chapters --embed-info-json -o "S:\srv\www\no-moss-3-carbo-landfill-library.online\YouTube\%%(uploader_id)s\%%(upload_date>%%Y-%%m-%%d)s-%%(title)s\%%(id)s.%%(ext)s" "https://www.youtube.com/watch?v=WMEw18t9p1Q" -S:\bin\yt-dlp.exe --live-from-start --yes-playlist -N 8 -R infinite -c --no-force-overwrites --mtime --write-description --write-info-json --write-playlist-metafiles --write-comments --no-cookies-from-browser --cookies S:\srv\www\no-moss-3-carbo-landfill-library.online\youtube-cookies.txt --write-thumbnail --write-all-thumbnails --write-url-link --write-webloc-link --write-desktop-link --progress --video-multistreams --audio-multistreams --write-subs --write-auto-subs --embed-subs --embed-thumbnail --embed-metadata --embed-chapters --embed-info-json -o "S:\srv\www\no-moss-3-carbo-landfill-library.online\YouTube\%%(uploader_id)s\%%(upload_date>%%Y-%%m-%%d)s-%%(title)s\%%(id)s.%%(ext)s" "https://www.youtube.com/watch?v=lZfmk1RPdbk" -S:\bin\yt-dlp.exe --live-from-start --yes-playlist -N 8 -R infinite -c --no-force-overwrites --mtime --write-description --write-info-json --write-playlist-metafiles --write-comments --no-cookies-from-browser --cookies S:\srv\www\no-moss-3-carbo-landfill-library.online\youtube-cookies.txt --write-thumbnail --write-all-thumbnails --write-url-link --write-webloc-link --write-desktop-link --progress --video-multistreams --audio-multistreams --write-subs --write-auto-subs --embed-subs --embed-thumbnail --embed-metadata --embed-chapters --embed-info-json -o "S:\srv\www\no-moss-3-carbo-landfill-library.online\YouTube\%%(uploader_id)s\%%(upload_date>%%Y-%%m-%%d)s-%%(title)s\%%(id)s.%%(ext)s" "https://www.youtube.com/watch?v=RRGeIh_fh_M" diff --git a/sync-youtube-videos.cmd b/sync-youtube-videos.cmd index 5f457f07..025fbd0f 100644 --- a/sync-youtube-videos.cmd +++ b/sync-youtube-videos.cmd @@ -1,5 +1,12 @@ @echo off S:\bin\yt-dlp.exe -U +echo Downloading https://www.youtube.com/@russellcountyvirginia8228 S:\bin\yt-dlp.exe --live-from-start --yes-playlist -N 8 -R infinite -c --no-force-overwrites --mtime --write-description --write-info-json --write-playlist-metafiles --write-comments --no-cookies-from-browser --cookies S:\srv\www\no-moss-3-carbo-landfill-library.online\youtube-cookies.txt --write-thumbnail --write-all-thumbnails --write-url-link --write-webloc-link --write-desktop-link --progress --video-multistreams --audio-multistreams --write-subs --write-auto-subs --embed-subs --embed-thumbnail --embed-metadata --embed-chapters --embed-info-json -o "S:\srv\www\no-moss-3-carbo-landfill-library.online\YouTube\%%(uploader_id)s\%%(upload_date>%%Y-%%m-%%d)s-%%(title)s\%%(id)s.%%(ext)s" "https://www.youtube.com/@russellcountyvirginia8228" +echo Downloading https://www.youtube.com/@VADMME S:\bin\yt-dlp.exe --live-from-start --yes-playlist -N 8 -R infinite -c --no-force-overwrites --mtime --write-description --write-info-json --write-playlist-metafiles --write-comments --no-cookies-from-browser --cookies S:\srv\www\no-moss-3-carbo-landfill-library.online\youtube-cookies.txt --write-thumbnail --write-all-thumbnails --write-url-link --write-webloc-link --write-desktop-link --progress --video-multistreams --audio-multistreams --write-subs --write-auto-subs --embed-subs --embed-thumbnail --embed-metadata --embed-chapters --embed-info-json -o "S:\srv\www\no-moss-3-carbo-landfill-library.online\YouTube\%%(uploader_id)s\%%(upload_date>%%Y-%%m-%%d)s-%%(title)s\%%(id)s.%%(ext)s" "https://www.youtube.com/@VADMME" -npm run-script index:docs +echo Downloading https://www.youtube.com/watch?v=WMEw18t9p1Q +S:\bin\yt-dlp.exe --live-from-start --yes-playlist -N 8 -R infinite -c --no-force-overwrites --mtime --write-description --write-info-json --write-playlist-metafiles --write-comments --no-cookies-from-browser --cookies S:\srv\www\no-moss-3-carbo-landfill-library.online\youtube-cookies.txt --write-thumbnail --write-all-thumbnails --write-url-link --write-webloc-link --write-desktop-link --progress --video-multistreams --audio-multistreams --write-subs --write-auto-subs --embed-subs --embed-thumbnail --embed-metadata --embed-chapters --embed-info-json -o "S:\srv\www\no-moss-3-carbo-landfill-library.online\YouTube\%%(uploader_id)s\%%(upload_date>%%Y-%%m-%%d)s-%%(title)s\%%(id)s.%%(ext)s" "https://www.youtube.com/watch?v=WMEw18t9p1Q" +echo Downloading https://www.youtube.com/watch?v=lZfmk1RPdbk +S:\bin\yt-dlp.exe --live-from-start --yes-playlist -N 8 -R infinite -c --no-force-overwrites --mtime --write-description --write-info-json --write-playlist-metafiles --write-comments --no-cookies-from-browser --cookies S:\srv\www\no-moss-3-carbo-landfill-library.online\youtube-cookies.txt --write-thumbnail --write-all-thumbnails --write-url-link --write-webloc-link --write-desktop-link --progress --video-multistreams --audio-multistreams --write-subs --write-auto-subs --embed-subs --embed-thumbnail --embed-metadata --embed-chapters --embed-info-json -o "S:\srv\www\no-moss-3-carbo-landfill-library.online\YouTube\%%(uploader_id)s\%%(upload_date>%%Y-%%m-%%d)s-%%(title)s\%%(id)s.%%(ext)s" "https://www.youtube.com/watch?v=lZfmk1RPdbk" +echo Downloading https://www.youtube.com/watch?v=RRGeIh_fh_M +S:\bin\yt-dlp.exe --live-from-start --yes-playlist -N 8 -R infinite -c --no-force-overwrites --mtime --write-description --write-info-json --write-playlist-metafiles --write-comments --no-cookies-from-browser --cookies S:\srv\www\no-moss-3-carbo-landfill-library.online\youtube-cookies.txt --write-thumbnail --write-all-thumbnails --write-url-link --write-webloc-link --write-desktop-link --progress --video-multistreams --audio-multistreams --write-subs --write-auto-subs --embed-subs --embed-thumbnail --embed-metadata --embed-chapters --embed-info-json -o "S:\srv\www\no-moss-3-carbo-landfill-library.online\YouTube\%%(uploader_id)s\%%(upload_date>%%Y-%%m-%%d)s-%%(title)s\%%(id)s.%%(ext)s" "https://www.youtube.com/watch?v=RRGeIh_fh_M" diff --git a/youtube-cookies.example.txt b/youtube-cookies.example.txt index 5e5868af..ae34808c 100644 --- a/youtube-cookies.example.txt +++ b/youtube-cookies.example.txt @@ -1,5 +1,5 @@ # Netscape HTTP Cookie File -# This file is generated by yt-dlp. Do not edit. I think I might have once needed to create it first in Edge and copy my values from my logged in browser. +# This file is generated by yt-dlp. Do not edit. .youtube.com TRUE / TRUE ... GPS ... .youtube.com TRUE / FALSE ... PREF tz=...&f6=...&f7=...&hl=...