Initial commit of new flow

This commit is contained in:
Smaug123
2023-09-10 21:21:51 +01:00
commit a1072f203d
28 changed files with 701 additions and 0 deletions

9
.devcontainer/Dockerfile Normal file
View File

@@ -0,0 +1,9 @@
FROM pandoc/latex:2.14.0.1
RUN apk add --no-cache git=2.26.3-r0 texlive=20190410-r13 && \
tlmgr update --self && \
tlmgr install tikz-cd && \
tlmgr install mdframed && \
# Dependencies of mdframed \
tlmgr install zref && \
tlmgr install needspace

View File

@@ -0,0 +1,7 @@
{
"build": { "dockerfile": "Dockerfile" },
"extensions": ["james-yu.latex-workshop", "vscodevim.vim", "timonwong.shellcheck"],
"settings": {
}
}

30
.gitignore vendored Normal file
View File

@@ -0,0 +1,30 @@
/result
public/
.ionide/
hugo/static/images/galleries
images/**/*-thumb.jpg
*.log
*.fls
*.fdb_latexmk
*.out
*.aux
*.synctex.gz
*.swp
pdfs/*.pdf
hugo/static/misc/AdjointFunctorTheorems
hugo/static/misc/FriedbergMuchnik/
hugo/static/misc/ModularMachines/*.tex
hugo/static/misc/ModularMachines/*.pdf
hugo/static/misc/MonadicityTheorems/
hugo/static/misc/MultiplicativeDetProof/
hugo/static/misc/NonstandardAnalysis/
hugo/static/misc/ParametricBoundedLoeb2016/
hugo/static/misc/RepresentableFunctors/
hugo/static/misc/Tennenbaum/
hugo/static/misc/TokyoEntrance2016/
.DS_Store
.idea/

6
.gitmodules vendored Normal file
View File

@@ -0,0 +1,6 @@
[submodule "images"]
path = images
url = ../static-site-images
[submodule "extra-content"]
path = extra-content
url = ../extra-site-content

52
build.sh Executable file
View File

@@ -0,0 +1,52 @@
#!/bin/sh
rm -rf public
mkdir public || exit 1
DOCKERUSER="$(id -u):$(id -g)"
export DOCKERUSER
IMAGES="$(pwd)/images"
export IMAGES
PDFS="$(pwd)/pdfs"
export PDFS
HUGO="$(pwd)/hugo"
export HUGO
OUTPUT="$(pwd)/public"
export OUTPUT
docker build docker/hadolint -t build-hadolint || exit 1
docker run --user "$(id -u):$(id -g)" -v "$(pwd):/work" build-hadolint sh -c "/build.sh" || exit 1
docker build docker/shellcheck -t build-shellcheck || exit 1
docker run --user "$(id -u):$(id -g)" -v "$(pwd):/work" build-shellcheck sh -c "/build.sh /work" || exit 1
docker build docker/latex -t build-latex || exit 1
docker build docker/pictures -t build-pictures || exit 1
docker build docker/hugo -t build-hugo || exit 1
echo Building thumbnails.
docker run --user "$(id -u):$(id -g)" -v "$IMAGES:/output" build-pictures sh -c "/build.sh /output" || exit 1
while read -r d
do
mkdir -p "$HUGO/$d" || exit 1
DESIRED_LOCATION=$(basename "$d")
DEST=$(dirname "$d")
cp -r "$IMAGES/$DESIRED_LOCATION" "$HUGO/$DEST"
done < "$IMAGES/image-targets.txt"
echo Building LaTeX.
docker run --user "$(id -u):$(id -g)" -v "$PDFS:/inputs" build-latex sh -c "/build.sh /inputs" || exit 1
while read -r texfile
do
DIR="$HUGO"/$(dirname "$texfile")
TEXFILE_BASE=$(basename "$texfile")
PDFFILE=$(basename "$texfile" .tex).pdf
mkdir -p "$DIR"
cp "$PDFS/$TEXFILE_BASE" "$DIR/"
cp "$PDFS/$PDFFILE" "$DIR/"
done < "$PDFS/pdf-targets.txt"
echo Building site.
docker run --user "$(id -u):$(id -g)" -v "$HUGO:/hugo" -v "$OUTPUT:/output" build-hugo sh -c "/build.sh /hugo /output" || exit 1
echo Done.

58
build/all.sh Executable file
View File

@@ -0,0 +1,58 @@
#!/bin/bash
pdfs=$1
images=$2
ankiDecks=$3
buildHugo=$4
katex=$5
extraContent=$6
echo "Linking PDFs: $PDF_FOLDER/pdf-targets.txt"
while IFS= read -r texfile || [[ -n "$r" ]]
do
DIR=$(dirname "$texfile")
TEXFILE_BASE=$(basename "$texfile")
if [ -z "${TEXFILE_BASE}" ]; then
echo "Skipping empty line"
else
PDFFILE=$(basename "$texfile" .tex).pdf
mkdir -p "$DIR"
echo "$TEXFILE_BASE"
echo "$PDFFILE"
cp "${pdfs}/$TEXFILE_BASE" "$texfile"
cp "${pdfs}/$PDFFILE" "$DIR/"
fi
done < "${pdfs}/pdf-targets.txt"
echo "Linking thumbnails."
while IFS= read -r d || [[ -n "$d" ]]
do
if [ -n "${d}" ]; then
DIR=$(dirname "$d")
mkdir -p "$DIR" || exit 1
DESIRED_LOCATION=$(basename "$d")
echo "$d -> $DESIRED_LOCATION"
cp -r "${images}/$DESIRED_LOCATION" "$d"
fi
done < "${images}/image-targets.txt"
echo "Linking Anki decks."
mkdir static/AnkiDecks && cp -R "${ankiDecks}/." static/AnkiDecks
echo "Building site."
mkdir -p "themes/anatole/assets"
cp -r "$katex"/dist/fonts themes/anatole/assets/fonts
cp -r "$katex/dist/contrib" themes/anatole/assets/contrib
cp "$katex"/dist/*.js themes/anatole/assets/
cp "$katex"/dist/*.css themes/anatole/assets/
while IFS= read -r file_to_copy
do
echo "$file_to_copy"
sourcefile=$(echo "$file_to_copy" | cut -d ' ' -f 1)
destfile=$(echo "$file_to_copy" | cut -d ' ' -f 2-)
cp "$extraContent/$sourcefile" "$destfile"
done < "$extraContent/map.txt"
/bin/sh "${buildHugo}/run.sh" . ./output

15
build/anki.sh Executable file
View File

@@ -0,0 +1,15 @@
#!/bin/sh
input_folder=$1
anki=$2
out=$3
mkdir -p "$out"
for item in "$input_folder"/*.json; do
output_file=$(echo "$item" | sed 's/\.json$/.apkg/')
# Invoke the binary command on the item
echo "$item"
echo "$out/$output_file"
"$anki" render --output "$out/$output_file" "$item"
done

View File

@@ -0,0 +1,9 @@
FROM hadolint/hadolint:v2.10.0-beta-alpine
COPY "build.sh" "/build.sh"
RUN chmod +x /build.sh && \
addgroup -S hugo && adduser -S hugo -G hugo
USER hugo
ENTRYPOINT []

3
docker/hadolint/build.sh Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/sh
find /work -type f -name 'Dockerfile' -print0 | xargs -0 -n1 hadolint

19
docker/html/Dockerfile Normal file
View File

@@ -0,0 +1,19 @@
FROM debian:stable-20211011-slim AS builder
ADD https://github.com/validator/validator/releases/download/20.6.30/vnu.linux.zip .
ADD https://github.com/validator/validator/releases/download/20.6.30/vnu.linux.zip.sha1 .
COPY "build.sh" "/build.sh"
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update && apt-get install --no-install-recommends -y \
unzip=6.0-26 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& echo "$(cat vnu.linux.zip.sha1) vnu.linux.zip" | sha1sum -c - \
&& unzip ./vnu.linux.zip \
&& rm ./vnu.linux.zip* \
&& apt-get purge -y --auto-remove unzip \
&& chmod +x /build.sh \
&& addgroup --gid 1000 hugo && adduser --gid 1000 hugo
USER hugo

3
docker/html/build.sh Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/sh
find result/ -type f -name '*.html' -print0 | xargs -0 -n1 /nix/store/p7adf8zk6akdbjr60vb98fajxp5aaa7i-html-tidy-5.8.0/bin/tidy 2>/dev/null

12
docker/hugo/Dockerfile Normal file
View File

@@ -0,0 +1,12 @@
FROM alpine:3.15.4
RUN apk add --no-cache hugo=0.89.4-r2 git-2.34.8-r0
COPY "build.sh" "/build.sh"
RUN chmod +x /build.sh && \
mkdir /sentinels && \
chmod a+rwx /sentinels && \
addgroup -S hugo && adduser -S hugo -G hugo
USER hugo
ENTRYPOINT []

7
docker/hugo/build.sh Executable file
View File

@@ -0,0 +1,7 @@
#!/bin/sh
SOURCE_DIR=$(readlink -f "$1")
OUTPUT_DIR=$(readlink -f "$2")
rm -rf "${OUTPUT_DIR:?}/*" && \
hugo --minify --source "$SOURCE_DIR" --destination "$OUTPUT_DIR"

28
docker/latex/Dockerfile Normal file
View File

@@ -0,0 +1,28 @@
FROM pandoc/latex:2.18.0.0 AS basic
RUN apk add --no-cache texlive=20210325-r4
# texlive is huuuge and we don't want to keep re-downloading it; don't warn on the repeated RUN command
# hadolint ignore=DL3059
RUN tlmgr update --self && \
tlmgr install tikz-cd && \
tlmgr install mdframed && \
tlmgr install mathtools && \
# Dependencies of mdframed \
tlmgr install zref && \
tlmgr install needspace
FROM basic AS build
COPY "build.sh" "/build.sh"
RUN mkdir /sentinels && \
chmod a+rwx /sentinels && \
chmod +x /build.sh && \
addgroup -S pdftex && adduser -S pdftex -G pdftex
USER pdftex
WORKDIR /home/pdftex
ENTRYPOINT []

23
docker/latex/build.sh Executable file
View File

@@ -0,0 +1,23 @@
#!/bin/sh
USER_DIR=$(readlink -f "$1")
WORKDIR=$(mktemp -d -p "$USER_DIR")
cd "$WORKDIR" || exit 1
SHELL="/bin/sh"
# Build PDFs from LaTeX. Do the build twice to sort out any bookmarks.
# For some reason, using $0 instead of `sh` makes Hadolint warn about the single-quotes not expanding expressions
# shellcheck disable=SC2016
find "$USER_DIR" -type f -name '*.tex' -exec "$SHELL" -c '
if [ -f "${@%.*}.pdf" ]; then exit 0; fi;
output=$(dirname "$@")/$(basename "$@" .tex).pdf
echo "$@ - $output"
HOME=$(pwd) SOURCE_DATE_EPOCH=1622905527 pdflatex "$@" || exit 1
HOME=$(pwd) SOURCE_DATE_EPOCH=1622905527 pdflatex "$@" || exit 1
mv "$(basename "$output")" "$output" || exit 1
' -- {} \; || exit 1
cd "$USER_DIR" || exit 1
rm -r "$WORKDIR"

13
docker/load/Dockerfile Normal file
View File

@@ -0,0 +1,13 @@
FROM alpine:3.13.5
COPY "build.sh" "/build.sh"
RUN chmod +x /build.sh && \
mkdir /sentinels && \
chmod a+rwx /sentinels && \
addgroup -S load && adduser -S load -G load && \
mkdir /output && \
chown load /output
USER load
ENTRYPOINT []

10
docker/load/build.sh Executable file
View File

@@ -0,0 +1,10 @@
#!/bin/sh
rm -f /sentinels/load.txt
rm -rf -- /output/*
cp -Rf /git/. /output || exit 1
chmod -R a+rw /output || exit 1
touch /sentinels/load.txt

View File

@@ -0,0 +1,14 @@
FROM alpine:3.15.4
RUN apk add --no-cache imagemagick=7.1.0.16-r0
COPY "build.sh" "/build.sh"
RUN chmod +x /build.sh && \
mkdir /sentinels && \
chmod a+rwx /sentinels && \
addgroup -S pictures && adduser -S pictures -G pictures
USER pictures
WORKDIR /home/pictures
ENTRYPOINT []

13
docker/pictures/build.sh Executable file
View File

@@ -0,0 +1,13 @@
#!/bin/sh
TO_SCAN="$1"
SHELL="/bin/sh"
# For some reason, using $0 instead of `sh` makes Shellcheck warn about the single-quotes not expanding expressions
# shellcheck disable=SC2016
find "$TO_SCAN" -type f ! -name '*-thumb.jpg' -name '*.jpg' -exec "$SHELL" -c '
if [ -f "${@%.*}-thumb.jpg" ]; then exit 0; fi;
echo "$@"
convert "$@" -thumbnail 100x100^ -gravity center -extent 100x100 "${1%.*}-thumb.jpg"
' -- {} \; || exit 1

View File

@@ -0,0 +1,9 @@
FROM alpine:3.13.5
COPY "build.sh" "/build.sh"
RUN chmod +x /build.sh && \
mkdir /sentinels && \
chmod a+rwx /sentinels
ENTRYPOINT []

View File

@@ -0,0 +1,9 @@
FROM koalaman/shellcheck-alpine:v0.8.0
COPY "build.sh" "/build.sh"
RUN chmod +x /build.sh && \
addgroup -S shellcheck && adduser -S shellcheck -G shellcheck
USER shellcheck
ENTRYPOINT []

3
docker/shellcheck/build.sh Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/sh
find "$1" -type f -name '*.*sh' -print0 | xargs -0 -n1 shellcheck

9
external-links.txt Normal file
View File

@@ -0,0 +1,9 @@
https://www.patrickstevens.co.uk/slightly-silly-sylow-pseudo-sonnets/
https://www.patrickstevens.co.uk/misc/YonedaWithoutTears/YonedaWithoutTears.pdf
https://www.patrickstevens.co.uk/misc/ModularMachines/EmbedMMIntoTuringMachine.pdf
https://www.patrickstevens.co.uk/posts/2021-10-19-crates/
https://www.patrickstevens.co.uk/posts/2016-05-25-finitistic-reducibility/
https://www.patrickstevens.co.uk/posts/2016-04-13-independence-of-choice/
https://www.patrickstevens.co.uk/posts/2016-04-08-another-monty-hall-explanation/
https://www.patrickstevens.co.uk/misc/AdjointFunctorTheorems/AdjointFunctorTheorems.pdf
https://www.patrickstevens.co.uk/posts/2021-02-20-in-praise-of-dry-run/

1
extra-content Submodule

Submodule extra-content added at 62e61fa89e

129
flake.lock generated Normal file
View File

@@ -0,0 +1,129 @@
{
"nodes": {
"anki-compiler": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1694219801,
"narHash": "sha256-8KFSy+R0nwUeZ3U2WYvRRjEYEk8iLXwWM9onvz5pixE=",
"owner": "Smaug123",
"repo": "anki-dotnet",
"rev": "8d1904d5cea06c8c20f5712ba865ace2d61b6255",
"type": "github"
},
"original": {
"owner": "Smaug123",
"repo": "anki-dotnet",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1692799911,
"narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems_2"
},
"locked": {
"lastModified": 1692799911,
"narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1694021185,
"narHash": "sha256-v5Ie83yfsiQgp4GDRZFIsbkctEynfOdNOi67vBH12XM=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "3e233330d9f88f78c75c2a164a50807e44245007",
"type": "github"
},
"original": {
"owner": "nixos",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1688392541,
"narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"anki-compiler": "anki-compiler",
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs_2"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

200
flake.nix Normal file
View File

@@ -0,0 +1,200 @@
{
description = "Static site builder for patrickstevens.co.uk";
inputs = {
flake-utils.url = github:numtide/flake-utils;
nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11";
anki-compiler.url = "github:Smaug123/anki-dotnet";
};
outputs = {
self,
nixpkgs,
flake-utils,
anki-compiler,
}:
flake-utils.lib.eachDefaultSystem (system: let
pkgs = nixpkgs.legacyPackages.${system};
in let
texlive = pkgs.texlive.combine {
inherit (pkgs.texlive) scheme-medium mdframed etoolbox zref needspace tikz-cd;
};
in rec {
packages = flake-utils.lib.flattenTree {
gitAndTools = pkgs.gitAndTools;
};
defaultPackage = let
createShellScript = name: contents:
pkgs.stdenv.mkDerivation {
__contentAddressed = true;
pname = name;
version = "0.1.0";
src = contents;
buildInputs = [
pkgs.shellcheck
];
phases = ["configurePhase" "buildPhase" "installPhase"];
configurePhase = ''
${pkgs.shellcheck}/bin/shellcheck "${contents}"
'';
buildPhase = ''
cp "${contents}" run.sh
patchShebangs run.sh
sed -i 's_"/bin/sh"_"${pkgs.bash}/bin/sh"_' run.sh
'';
installPhase = ''
mkdir -p $out
mv run.sh $out/run.sh
'';
};
in let
buildLatex = createShellScript "latex" ./docker/latex/build.sh;
buildPictures = createShellScript "pictures" ./docker/pictures/build.sh;
buildHugo = createShellScript "hugo" ./docker/hugo/build.sh;
buildAnki = createShellScript "anki" ./build/anki.sh;
buildEverything = createShellScript "all" ./build/all.sh;
in let
katex = pkgs.stdenv.mkDerivation {
pname = "katex";
version = "0.1.0";
src = pkgs.fetchFromGitHub {
owner = "KaTeX";
repo = "KateX";
rev = "4f1d9166749ca4bd669381b84b45589f1500a476";
sha256 = "sha256-hDHo7JQAo+fGxQvY5OtXlfh+e6PjlVIQPTyCa3Fjg0Y=";
};
buildInputs = [pkgs.nodejs pkgs.yarn];
buildPhase = ''
export HOME=$(mktemp -d)
yarn --immutable
yarn build
find . -type f -name "katex.min.*"
'';
installPhase = ''
mkdir -p "$out/fonts"
cp ./fonts/* "$out/fonts"
cp -r ./dist "$out/dist"
'';
};
in let
pdfs = pkgs.stdenv.mkDerivation {
__contentAddressed = true;
pname = "patrickstevens.co.uk-latex";
version = "0.1.2";
src = pkgs.fetchFromGitHub {
owner = "Smaug123";
repo = "static-site-pdfs";
rev = "d8cf76c2f1f669e177cff5217f9ebbf763070d71";
sha256 = "sha256-BH8EoVP4jtqisjIuayDKxXrvjzyNrJJP/OJf6rH0zgE=";
};
buildInputs = [
texlive
buildLatex
];
buildPhase = ''
${pkgs.bash}/bin/sh ${buildLatex}/run.sh .
'';
installPhase = ''
mkdir -p $out
cp ./* $out
'';
};
in let
images = pkgs.stdenv.mkDerivation {
__contentAddressed = true;
pname = "patrickstevens.co.uk-images";
version = "0.1.0";
src = ./images;
buildInputs = [
pkgs.imagemagick
buildPictures
];
buildPhase = ''
${pkgs.bash}/bin/sh ${buildPictures}/run.sh .
'';
installPhase = ''
ls -la .
mkdir -p $out
mv ./* $out
'';
};
in let
extraContent = pkgs.stdenv.mkDerivation {
__contentAddressed = true;
pname = "patrickstevens.co.uk-extraContent";
version = "0.1.0";
src = ./extra-content;
buildInputs = [];
installPhase = ''
mkdir -p $out
cp -r ./. $out
'';
};
in let
ankiDecks = pkgs.stdenv.mkDerivation {
__contentAddressed = true;
pname = "patrickstevens.co.uk-anki";
version = "0.2.0";
src = pkgs.fetchFromGitHub {
owner = "Smaug123";
repo = "anki-decks";
rev = "589a52858d5dca419ad8866946b7418f30b36eff";
sha256 = "sha256-tc3Twev82WVFbHbEgLVwgcQnPaEYGAUHqw8lpj1Kuqk=";
};
buildInputs = [];
installPhase = ''
pwd
${./build/anki.sh} . "${anki-compiler.packages.${system}.default}/bin/AnkiStatic" "$out"
'';
};
in
pkgs.stdenv.mkDerivation {
__contentAddressed = true;
pname = "patrickstevens.co.uk";
version = "0.1.0";
src = pkgs.fetchFromGitHub {
owner = "Smaug123";
repo = "static-site-content";
rev = "67c4f084fc3425e86c1f98e3b0e5fa1b8049296d";
sha256 = "sha256-fOhvSnah74ZLYV130HOYnVduw5qpK32GikDdC6u/4gA=";
};
buildInputs = [
pkgs.hugo
buildHugo
images
pdfs
ankiDecks
katex
extraContent
pkgs.html-tidy
];
buildPhase = ''
${./build/all.sh} "${pdfs}" "${images}" "${ankiDecks}" "${buildHugo}" "${katex}" "${extraContent}"
'';
checkPhase = ''
echo "Linting HTML."
${pkgs.html-tidy}/bin/tidy
'';
installPhase = ''
mv output $out
'';
};
});
}

1
images Submodule

Submodule images added at c0bfc303ff

9
lint.sh Executable file
View File

@@ -0,0 +1,9 @@
#!/bin/sh
OUTPUT="$(pwd)/public"
export OUTPUT
docker build docker/html -t build-html || exit 1
echo Validating HTML.
docker run --user "$(id -u):$(id -g)" -v "$OUTPUT:/public" build-html sh -c "/build.sh" || exit 1