Fixes post-notification.

This is a major rewrite of how we're catching for trouble during
parsing and the posting of articles.  When fetching articles by
message-id, we don't need to enforce that a group has been chosen.

Anyfile

@@ -6,76 +6,82 @@ default: all
 all: loop.exe loop.lisp loop.asd scripts/build-exe.lisp \
 scripts/build-index-from-fs.lisp \
 scripts/cron-remove-inactive-users.lisp \
-scripts/migrate-add-creation-date.lisp
+scripts/migrate-add-creation-date.lisp \
+scripts/verify-index.lisp \
+scripts/find-duplicates.lisp \
+version.lisp TAGS
+
+version.lisp: loop.lisp
+	echo "(in-package :loop)" > version.lisp
+	echo "(defparameter *git-hash* \"$$(git rev-parse --short HEAD)\")" >> version.lisp
+
+TAGS: loop.lisp
+	etags $(PWD)/loop.lisp
+	sed -i 's|$(PWD)/loop.lisp|$(PWD)/loop.nw|g' TAGS
 
 loop.exe: scripts/build-exe.lisp loop.lisp loop.asd loop.nw
 	sbcl --script scripts/build-exe.lisp
 	(test -f loop.exe && cmp loop loop.exe) || cp loop loop.exe
 
 loop.lisp: loop.nw format-def
-	(any tangle -Rloop.lisp < loop.nw | sh format-def | \
+	(no tangle -Rloop.lisp < loop.nw | sh format-def > loop.tmp || \
-		dos2unix > loop.tmp || \
 		(rm loop.tmp && exit 1)) && \
 		mv loop.tmp loop.lisp
 
 format-def: loop.nw
-	(any tangle -Rformat-def < loop.nw | \
+	(no tangle -Rformat-def.sh < loop.nw > format-def.tmp || \
-		dos2unix > format-def.tmp || \
 		(rm format-def.tmp && exit 1)) && \
 		mv format-def.tmp format-def
 
-make-release: loop.nw
-	(any tangle -Rmake-release < loop.nw | \
-		dos2unix > make-release.tmp || \
-		(rm make-release.tmp && exit 1)) && \
-		mv make-release.tmp make-release
-
-release: make-release
-	./make-release $$(git log --oneline | head -1 | awk '{print $$1}') \
-           loop.nw > loop.tmp && mv loop.tmp loop.nw
-
 loop.asd: loop.nw
-	(any tangle -Rloop.asd < loop.nw | dos2unix > loop-asd.tmp || \
+	(no tangle -Rloop.asd < loop.nw > loop-asd.tmp || \
 		(rm loop-asd.tmp && exit 1)) && \
 		mv loop-asd.tmp loop.asd
 
 scripts/build-exe.lisp: loop.asd loop.lisp loop.nw
-	(any tangle -Rbuild-exe.lisp < loop.nw | dos2unix > build-exe.tmp || \
+	(no tangle -Rbuild-exe.lisp < loop.nw > build-exe.tmp || \
 		(rm build-exe.tmp && exit 1)) && \
 	    mv build-exe.tmp scripts/build-exe.lisp
 
 scripts/build-index-from-fs.lisp: loop.nw
-	(any tangle -Rbuild-index-from-fs.lisp < loop.nw | dos2unix > \
+	(no tangle -Rbuild-index-from-fs.lisp < loop.nw > \
 		build-index-from-fs.tmp || \
 		(rm build-index-from-fs.tmp && exit 1)) && \
 	    	mv build-index-from-fs.tmp scripts/build-index-from-fs.lisp
 
+scripts/verify-index.lisp: loop.nw
+	(no tangle -Rverify-index.lisp < loop.nw > \
+		verify-index.tmp || \
+		(rm verify-index.tmp && exit 1)) && \
+		mv verify-index.tmp scripts/verify-index.lisp
+
+scripts/find-duplicates.lisp: loop.nw
+	(no tangle -Rfind-duplicates.lisp < loop.nw > \
+		find-duplicates.tmp || \
+		(rm find-duplicates.tmp && exit 1)) && \
+		mv find-duplicates.tmp scripts/find-duplicates.lisp
+
 scripts/cron-remove-inactive-users.lisp: loop.nw
-	(any tangle -Rcron-remove-inactive-users.lisp < loop.nw | dos2unix > \
+	(no tangle -Rcron-remove-inactive-users.lisp < loop.nw > \
 		cron-remove-inactive-users.tmp || \
 		(rm cron-remove-inactive-users.tmp && exit 1)) && \
 		mv cron-remove-inactive-users.tmp \
 			scripts/cron-remove-inactive-users.lisp
 
 scripts/migrate-add-creation-date.lisp: loop.nw
-	(any tangle -Rmigrate-add-creation-date.lisp < loop.nw | dos2unix > \
+	(no tangle -Rmigrate-add-creation-date.lisp < loop.nw > \
 		migrate-add-creation-date.tmp || \
 		(rm migrate-add-creation-date.tmp && exit 1)) && \
 		mv migrate-add-creation-date.tmp \
 			scripts/migrate-add-creation-date.lisp
 
-run: loop.nw
-	(any tangle -Rrun < loop.nw | dos2unix > run.tmp || \
-		(rm run.tmp && exit 1)) && \
-		mv run.tmp run && \
-		chmod 0755 run
-
 loop.tex: loop.nw
-	any weave -delay -index loop.nw | dos2unix > loop.tex
+	no weave -delay -index loop.nw > loop.tex
 
 loop.pdf: loop.tex
 	latexmk -pdf loop
 
 clean: 
 	rm -f loop loop.exe loop.asd loop.lisp loop.tex loop.pdf \
-	  *.fasl *.db .*~ *~ *.out *.aux *.log *.fls *.fdb_latexmk
+	  *.fasl *.db .*~ *~ *.out *.aux *.log *.fls *.fdb_latexmk \
+	TAGS

README

@@ -21,7 +21,7 @@ install it as a local project.  Go to
 
 and say 
 
-  $ git clone https://git.antartida.xyz/loop/srv loop
+  $ git clone https://git.atlantida.space/root/loop
   $ cd loop
   $ echo /path/to/loop/home > conf-home
   $ make install
@@ -76,7 +76,7 @@ Using another terminal, telnet to your host on port 1024:
 --8<-------------------------------------------------------->8---
 $ telnet localhost 1024
 Trying 127.0.0.1...
-Connected to antartida.xyz.
+Connected to 127.0.0.1.
 Escape character is '^]'.
 200 Welcome! Say ``help'' for a menu.
 quit
@@ -122,22 +122,6 @@ $ crontab -l
 
 We installed LOOP just fine on
 
+  Void GNU/Linux release ``rolling'' with SBCL 2.6.6.void.1
   FreeBSD 14.1, 14.2 with SBCL 2.4.9.
   Debian GNU/Linux 8.11 codename jessie with SBCL 1.2.4.debian.
-
-(*) Systems with installation issues
-
-We installed LOOP on Ubuntu 24.04 (24.01.1 LTS) codename noble with
-SBCL 2.2.9.debian.  We found that CLSQL could not load the shared
-object libsqlite3.so because ``apt install libsqlite3'' installs the
-library at 
-
-  /usr/lib/x86_64-linux-gnu/libsqlite3.so.0.8.6
-
-with a symbolic link to libsqlite3.so.0, but not to libsqlite3.so.
-SBCL is trying to load libsqlite3.so, so a solution is to just tell
-your system to
-
-  ln -s libsqlite3.so.0 libsqlite3.so
-
-at /usr/lib/x86_64-linux-gnu.

groups/local.control.news/1

@@ -1,7 +1,7 @@
-Date: 2024-03-07 21:44:31 GMT-3
+Date: 2024-03-07 21:44:31 GMT-3
-Message-Id: <edjocyeqzqqhnswlbrbo@loop>
+Message-Id: <xdjocyeqzqqhnswlbrbo@loop>
-From: Loop
+From: Loop
-Subject: let there be light
+Subject: let there be light
-Newsgroups: local.control.news
+Newsgroups: local.control.news
-
+
-Administrative news will be posted here by me. -- Loop
+Administrative news will be posted here by me. -- Loop

groups/local.test/1

@@ -1,7 +1,7 @@
-Date: 2024-03-07 21:44:31 GMT-3
+Date: 2024-03-07 21:44:31 GMT-3
-Message-Id: <edjocyeqzqqhnswlbrbo@loop>
+Message-Id: <edjocyeqzqqhnswlbrbo@loop>
-From: Loop
+From: Loop
-Subject: let there be light
+Subject: let there be light
-Newsgroups: local.test
+Newsgroups: local.test
-
+
-A sample group.
+A sample group.

loop.asd

@@ -1,7 +1,8 @@
 ;;; -*- Mode: LISP; Syntax: COMMON-LISP; -*-
 (asdf:defsystem :LOOP
-  :version "a89e088"
+  :version "0.0.1"
   :description "An NNTP server for a circle of friends."
   :depends-on (:lisp-unit :str :uiop :cl-ppcre :local-time :clsql-sqlite3 :clingon
                :filesystem-utils :ironclad/digest/sha256)
-  :components ((:file "loop")))
+  :components ((:file "loop")
+               (:file "version")))

make-release

@@ -8,7 +8,7 @@ usage()
 test $# -lt 2 && usage
 
 tag="$1"; shift
-sed "/a89e088=/ {
+sed "/0.0.1=/ {
  n;
  c\\
 $tag

noweb.sty

@@ -1,989 +0,0 @@
-% noweb.sty -- LaTeX support for noweb
-% DON'T read or edit this file!  Use ...noweb-source/tex/support.nw instead.
-{\obeyspaces\AtBeginDocument{\global\let =\ }} % from texbook, p 381
-\def\nwopt@nomargintag{\let\nwmargintag=\@gobble}
-\def\nwopt@margintag{%
-  \def\nwmargintag##1{\leavevmode\llap{##1\kern\nwmarginglue\kern\codemargin}}}
-\def\nwopt@margintag{%
-  \def\nwmargintag##1{\leavevmode\kern-\codemargin\nwthemargintag{##1}\kern\codemargin}}
-\def\nwthemargintag#1{\llap{#1\kern\nwmarginglue}}
-\nwopt@margintag
-\newdimen\nwmarginglue
-\nwmarginglue=0.3in
-\def\nwtagstyle{\footnotesize\Rm}
-% make \hsize in code sufficient for 88 columns
-\setbox0=\hbox{\tt m}
-\newdimen\codehsize
-\codehsize=91\wd0 % 88 columns wasn't enough; I don't know why
-\newdimen\codemargin
-\codemargin=0pt
-\newdimen\nwdefspace
-\nwdefspace=\codehsize
-% need to use \textwidth in {\LaTeX} to handle styles with
-% non-standard margins (David Bruce).  Don't know why we sometimes
-% wanted \hsize.  27 August 1997.
-%% \advance\nwdefspace by -\hsize\relax
-\ifx\textwidth\undefined
-  \advance\nwdefspace by -\hsize\relax
-\else
-  \advance\nwdefspace by -\textwidth\relax
-\fi
-\chardef\other=12
-\def\setupcode{%
-  \chardef\\=`\\
-  \chardef\{=`\{
-  \chardef\}=`\}
-  \catcode`\$=\other
-  \catcode`\&=\other
-  \catcode`\#=\other
-  \catcode`\%=\other
-  \catcode`\~=\other
-  \catcode`\_=\other
-  \catcode`\^=\other
-  \catcode`\"=\other    % fixes problem with german.sty
-  \obeyspaces\Tt
-}
-%\let\nwlbrace=\{
-%\let\nwrbrace=\}
-\def\nwendquote{\relax\ifhmode\spacefactor=1000 \fi}
-{\catcode`\^^M=\active % make CR an active character
-  \gdef\newlines{\catcode`\^^M=\active % make CR an active character
-         \def^^M{\par\startline}}%
-  \gdef\eatline#1^^M{\relax}%
-}
-%%% DON'T   \gdef^^M{\par\startline}}% in case ^^M appears in a \write
-\def\startline{\noindent\hskip\parindent\ignorespaces}
-\def\nwnewline{\ifvmode\else\hfil\break\leavevmode\hbox{}\fi}
-\def\setupmodname{%
-  \catcode`\$=3
-  \catcode`\&=4
-  \catcode`\#=6
-  \catcode`\%=14
-  \catcode`\~=13
-  \catcode`\_=8
-  \catcode`\^=7
-  \catcode`\ =10
-  \catcode`\^^M=5
-  \let\nwlbrace\lbrace
-  \let\nwrbrace\rbrace
-  \let\{\nwlbrace
-  \let\}\nwrbrace
-  % bad news --- don't know what catcode to give "
-  \Rm}
-\def\LA{\begingroup\maybehbox\bgroup\setupmodname\It$\langle$}
-\def\RA{\/$\rangle$\egroup\endgroup}
-\def\code{\leavevmode\begingroup\setupcode\newlines}
-\def\edoc{\endgroup}
-\let\maybehbox\relax
-\newbox\equivbox
-\setbox\equivbox=\hbox{$\equiv$}
-\newbox\plusequivbox
-\setbox\plusequivbox=\hbox{$\mathord{+}\mathord{\equiv}$}
-% \moddef can't have an argument because there might be \code...\edoc
-\def\moddef{\leavevmode\kern-\codemargin\LA}
-\def\endmoddef{\RA\ifmmode\equiv\else\unhcopy\equivbox\fi
-               \nobreak\hfill\nobreak}
-\def\plusendmoddef{\RA\ifmmode\mathord{+}\mathord{\equiv}\else\unhcopy\plusequivbox\fi
-               \nobreak\hfill\nobreak}
-\def\chunklist{%
-\errhelp{I changed \chunklist to \nowebchunks.  
-I'll try to avoid such incompatible changes in the future.}%
-\errmessage{Use \string\nowebchunks\space instead of \string\chunklist}}
-\def\nowebchunks{\message{<Warning: You need noweave -x to use \string\nowebchunks>}}
-\def\nowebindex{\message{<Warning: You need noweave -index to use \string\nowebindex>}}
-% here is support for the new-style (capitalized) font-changing commands
-% thanks to Dave Love
-\ifx\documentstyle\undefined
-  \let\Rm=\rm \let\It=\it \let\Tt=\tt       % plain
-\else\ifx\selectfont\undefined
-  \let\Rm=\rm \let\It=\it \let\Tt=\tt       % LaTeX OFSS
-\else                                       % LaTeX NFSS
-  \def\Rm{\reset@font\rm}
-  \def\It{\reset@font\it}
-  \def\Tt{\reset@font\tt}
-  \def\Bf{\reset@font\bf}
-\fi\fi
-\ifx\reset@font\undefined \let\reset@font=\relax \fi
-\def\noweboptions#1{%
-  \def\@nwoptionlist{#1}%
-  \@for\@nwoption:=\@nwoptionlist\do{%
-    \@ifundefined{nwopt@\@nwoption}{%
-        \@latexerr{There is no such noweb option as '\@nwoption'}\@eha}{%
-        \csname nwopt@\@nwoption\endcsname}}}
-\codemargin=10pt
-\advance\codehsize by \codemargin       % make room for indentation of code
-\advance\nwdefspace by \codemargin      % and fix adjustment for def/use
-\def\setcodemargin#1{%
-  \advance\codehsize by -\codemargin       % make room for indentation of code
-  \advance\nwdefspace by -\codemargin   % and fix adjustment for def/use
-  \codemargin=#1
-  \advance\codehsize by \codemargin       % make room for indentation of code
-  \advance\nwdefspace by \codemargin    % and fix adjustment for
-                                        % def/use
-}
-\def\nwopt@shift{%
-  \dimen@=-0.8in
-  \if@twoside                 % Values for two-sided printing:
-     \advance\evensidemargin by \dimen@
-  \else                       % Values for one-sided printing:
-     \advance\evensidemargin by \dimen@
-     \advance\oddsidemargin by \dimen@
-  \fi
-%  \advance \marginparwidth -\dimen@
-}
-\let\nwopt@noshift\@empty
-\def\nwbegincode#1{%
-  \begingroup
-    \topsep \nwcodetopsep
-    \@beginparpenalty \@highpenalty
-    \@endparpenalty -\@highpenalty
-  \@begincode }
-\def\nwendcode{\endtrivlist \endgroup \filbreak} % keeps code on 1 page
-
-\newenvironment{webcode}{%
-  \@begincode
-}{%
-  \endtrivlist}
-\def\@begincode{%
-    \trivlist \item[]%
-    \leftskip\@totalleftmargin \advance\leftskip\codemargin
-    \rightskip\hsize \advance\rightskip -\codehsize
-    \parskip\z@ \parindent\z@ \parfillskip\@flushglue
-  \linewidth\codehsize
-    \@@par
-    \def\par{\leavevmode\null \@@par \penalty\nwcodepenalty}%
-    \obeylines
-    \@noligs   \ifx\verbatim@nolig@list\undefined\else
-                 \let\do=\nw@makeother \verbatim@nolig@list \do@noligs\`
-               \fi
-    \setupcode \frenchspacing \@vobeyspaces
-  \nowebsize \setupcode
-  \let\maybehbox\mbox }
-  \newskip\nwcodetopsep \nwcodetopsep = 3pt plus 1.2pt minus 1pt
-  \let\nowebsize=\normalsize
-  \def\nwopt@tinycode{\let\nowebsize=\tiny}
-  \def\nwopt@footnotesizecode{\let\nowebsize=\footnotesize}
-  \def\nwopt@scriptsizecode{\let\nowebsize=\scriptsize}
-  \def\nwopt@smallcode{\let\nowebsize=\small}
-  \def\nwopt@normalsizecode{\let\nowebsize=\normalsize}
-  \def\nwopt@largecode{\let\nowebsize=\large}
-  \def\nwopt@Largecode{\let\nowebsize=\Large}
-  \def\nwopt@LARGEcode{\let\nowebsize=\LARGE}
-  \def\nwopt@hugecode{\let\nowebsize=\huge}
-  \def\nwopt@Hugecode{\let\nowebsize=\Huge}
-\newcount\nwcodepenalty  \nwcodepenalty=\@highpenalty
-\def\nw@makeother#1{\catcode`#1=12 }
-\def\nwbegindocs#1{\ifvmode\noindent\fi}
-\let\nwenddocs=\relax
-\let\nwdocspar=\filbreak
-\def\@nwsemifilbreak#1{\vskip0pt plus#1\penalty-200\vskip0pt plus -#1}
-\newdimen\nwbreakcodespace
-\nwbreakcodespace=0.2in  % by default, leave no more than this on a page
-\def\nwopt@breakcode{%
-  \def\nwdocspar{\@nwsemifilbreak{0.2in}}%
-  \def\nwendcode{\endtrivlist\endgroup} % ditches filbreak
-}
-\raggedbottom
-\def\code{\leavevmode\begingroup\setupcode\@vobeyspaces\obeylines}
-\let\edoc=\endgroup
-\newdimen\@original@textwidth
-\def\ps@noweb{%
-  \@original@textwidth=\textwidth
-  \let\@mkboth\@gobbletwo
-  \def\@oddfoot{}\def\@evenfoot{}%       No feet.
-  \if@twoside         % If two-sided printing.
-    \def\@evenhead{\hbox to \@original@textwidth{%
-           \Rm \thepage\qquad{\Tt\leftmark}\hfil\today}}%        Left heading.
-    \def\@oddhead{\hbox to \@original@textwidth{%
-           \Rm \today\hfil{\Tt\leftmark}\qquad\thepage}}% Right heading.
-  \else               % If one-sided printing.
-    \def\@oddhead{\hbox to \@original@textwidth{%
-           \Rm \today\hfil{\Tt\leftmark}\qquad\thepage}}% Right heading.
-    \let\@evenhead\@oddhead
-  \fi
-  \let\chaptermark\@gobble
-  \let\sectionmark\@gobble
-  \let\subsectionmark\@gobble
-  \let\subsubsectionmark\@gobble
-  \let\paragraphmark\@gobble
-  \let\subparagraphmark\@gobble
-  \def\nwfilename{\begingroup\let\do\@makeother\dospecials
-                \catcode`\{=1 \catcode`\}=2 \nw@filename}
-  \def\nw@filename##1{\endgroup\markboth{##1}{##1}\let\nw@filename=\nw@laterfilename}%
-}
-\def\nw@laterfilename#1{\endgroup\clearpage \markboth{#1}{#1}}
-\let\nwfilename=\@gobble
-\def\nwcodecomment#1{\@@par\penalty\nwcodepenalty
-    \if@firstnwcodecomment
-      \vskip\nwcodecommentsep\penalty\nwcodepenalty\@firstnwcodecommentfalse
-    \fi%
-    \hspace{-\codemargin}{%
-        \rightskip=0pt plus1in
-        \interlinepenalty\nwcodepenalty
-        \let\\\relax\footnotesize\Rm #1\@@par\penalty\nwcodepenalty}}
-\def\@nwalsodefined#1{\nwcodecomment{\@nwlangdepdef\ \nwpageprep\ \@pagesl{#1}.}}
-\def\@nwused#1{\nwcodecomment{\@nwlangdepcud\ \nwpageprep\ \@pagesl{#1}.}}
-\def\@nwnotused#1{\nwcodecomment{\@nwlangdeprtc.}}
-\def\nwoutput#1{\nwcodecomment{\@nwlangdepcwf\ {\Tt \@stripstar#1*\stripped}.}}
-\def\@stripstar#1*#2\stripped{#1}
-\newcommand{\nwprevdefptr}[1]{%
-  \mbox{$\mathord{\triangleleft}\,\mathord{\mbox{\subpageref{#1}}}$}}
-\newcommand{\nwnextdefptr}[1]{%
-  \mbox{$\mathord{\mbox{\subpageref{#1}}}\,\mathord{\triangleright}$}}
-
-\newcommand{\@nwprevnextdefs}[2]{%
-  {\nwtagstyle
-  \ifx\relax#1\else ~~\nwprevdefptr{#1}\fi
-  \ifx\relax#2\else ~~\nwnextdefptr{#2}\fi}}
-\newcommand{\@nwusesondefline}[1]{{\nwtagstyle~~(\@pagenumsl{#1})}}
-\newcommand{\@nwstartdeflinemarkup}{\nobreak\hskip 1.5em plus 1fill\nobreak}
-\newcommand{\@nwenddeflinemarkup}{\nobreak\hskip \nwdefspace minus\nwdefspace\nobreak}
-\def\nwopt@longxref{%
-  \let\nwalsodefined\@nwalsodefined
-  \let\nwused\@nwused
-  \let\nwnotused\@nwnotused
-  \let\nwprevnextdefs\@gobbletwo
-  \let\nwusesondefline\@gobble
-  \let\nwstartdeflinemarkup\relax
-  \let\nwenddeflinemarkup\relax
-}
-\def\nwopt@shortxref{%
-  \let\nwalsodefined\@gobble
-  \let\nwused\@gobble
-  \let\nwnotused\@gobble
-  \let\nwprevnextdefs\@nwprevnextdefs
-  \let\nwusesondefline\@nwusesondefline
-  \let\nwstartdeflinemarkup\@nwstartdeflinemarkup
-  \let\nwenddeflinemarkup\@nwenddeflinemarkup
-}
-\def\nwopt@noxref{%
-  \let\nwalsodefined\@gobble
-  \let\nwused\@gobble
-  \let\nwnotused\@gobble
-  \let\nwprevnextdefs\@gobbletwo
-  \let\nwusesondefline\@gobble
-  \let\nwstartdeflinemarkup\relax
-  \let\nwenddeflinemarkup\relax
-}
-\nwopt@shortxref % to hell with backward compatibility!
-\newskip\nwcodecommentsep \nwcodecommentsep=3pt plus 1pt minus 1pt
-\newif\if@firstnwcodecomment\@firstnwcodecommenttrue
-\newcount\@nwlopage\newcount\@nwhipage  % range lo..hi-1
-\newcount\@nwlosub              % subpage of lo
-\newcount\@nwhisub              % subpage of hi
-\def\@nwfirstpage#1#2#3{% subpage page xref-tag
-  \@nwlopage=#2 \@nwlosub=#1
-  \def\@nwloxreftag{#3}%
-  \advance\@nwpagecount by \@ne
-  \@nwhipage=\@nwlopage\advance\@nwhipage by \@ne }
-\def\@nwnextpage#1#2#3{% subpage page xref-tag
-  \ifnum\@nwhipage=#2 
-    \advance\@nwhipage by \@ne 
-    \advance\@nwpagecount by \@ne
-    \@nwhisub=#1 
-    \def\@nwhixreftag{#3}\else
-  \ifnum#2<\@nwlopage \advance\@nwhipage by \m@ne
-                      \ifnum\@nwhipage=\@nwlopage
-                           \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                                             {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                                              {\@nwloxreftag}}}%
-                      \else
-                        \count@=\@nwhipage \advance\count@ by \m@ne
-                        \ifnum\count@=\@nwlopage % consecutive pages
-                            \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                                             {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                                              {\@nwloxreftag}}%
-                                          \noexpand\noexpand\noexpand\\%
-                                             {{\nwthepagenum{\number\@nwhisub}{\number\@nwhipage}}
-                                              {\@nwhixreftag}}}%
-                        \else \ifnum\@nwlopage<110 \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-                                \count@=\@nwlopage \divide\count@ by 100 \multiply\count@ by 100
-                                \ifnum\count@=\@nwlopage \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-                                  \count@=\@nwlopage \divide\count@ by 100
-                                  \@nwpagetemp=\@nwhipage \divide\@nwpagetemp by 100
-                                  \ifnum\count@=\@nwpagetemp %  lo--least 2 digits of hi
-                                    \multiply\@nwpagetemp by 100
-                                    \advance \@nwhipage by -\@nwpagetemp
-                                    \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-                                  \else \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-                                  \fi
-                                \fi
-                              \fi%
-                        \fi
-                      \fi%
-                      \edef\@tempa{\noexpand\nwix@cons\noexpand\nw@pages{\@tempa}}\@tempa\@nwfirstpage{#1}{#2}{#3}\else
-  \ifnum#2>\@nwhipage \advance\@nwhipage by \m@ne
-                      \ifnum\@nwhipage=\@nwlopage
-                           \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                                             {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                                              {\@nwloxreftag}}}%
-                      \else
-                        \count@=\@nwhipage \advance\count@ by \m@ne
-                        \ifnum\count@=\@nwlopage % consecutive pages
-                            \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                                             {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                                              {\@nwloxreftag}}%
-                                          \noexpand\noexpand\noexpand\\%
-                                             {{\nwthepagenum{\number\@nwhisub}{\number\@nwhipage}}
-                                              {\@nwhixreftag}}}%
-                        \else \ifnum\@nwlopage<110 \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-                                \count@=\@nwlopage \divide\count@ by 100 \multiply\count@ by 100
-                                \ifnum\count@=\@nwlopage \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-                                  \count@=\@nwlopage \divide\count@ by 100
-                                  \@nwpagetemp=\@nwhipage \divide\@nwpagetemp by 100
-                                  \ifnum\count@=\@nwpagetemp %  lo--least 2 digits of hi
-                                    \multiply\@nwpagetemp by 100
-                                    \advance \@nwhipage by -\@nwpagetemp
-                                    \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-                                  \else \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-                                  \fi
-                                \fi
-                              \fi%
-                        \fi
-                      \fi%
-                      \edef\@tempa{\noexpand\nwix@cons\noexpand\nw@pages{\@tempa}}\@tempa\@nwfirstpage{#1}{#2}{#3}\else
-    \@nwlosub=0 \@nwhisub=0
-  \fi\fi\fi
-  }
-\newcount\@nwpagetemp
-\newcount\@nwpagecount
-\def\@nwfirstpagel#1{% label
-  \@ifundefined{r@#1}{\@warning{Reference `#1' on page \thepage \space undefined}%
-                      \nwix@cons\nw@pages{\\{\bf ??}}}{%
-    \edef\@tempa{\noexpand\@nwfirstpage\subpagepair{#1}{#1}}\@tempa}}
-\def\@nwnextpagel#1{% label
-  \@ifundefined{r@#1}{\@warning{Reference `#1' on page \thepage \space undefined}%
-                      \nwix@cons\nw@pages{\\{\bf ??}}}{%
-    \edef\@tempa{\noexpand\@nwnextpage\subpagepair{#1}{#1}}\@tempa}}
-\def\@pagesl#1{%  list of labels
-  \gdef\nw@pages{}\@nwpagecount=0
-  \def\\##1{\@nwfirstpagel{##1}\let\\=\@nwnextpagel}#1%
-  \advance\@nwhipage by \m@ne
-  \ifnum\@nwhipage=\@nwlopage
-       \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                          {\@nwloxreftag}}}%
-  \else
-    \count@=\@nwhipage \advance\count@ by \m@ne
-    \ifnum\count@=\@nwlopage % consecutive pages
-        \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                          {\@nwloxreftag}}%
-                      \noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwhisub}{\number\@nwhipage}}
-                          {\@nwhixreftag}}}%
-    \else \ifnum\@nwlopage<110 \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-            \count@=\@nwlopage \divide\count@ by 100 \multiply\count@ by 100
-            \ifnum\count@=\@nwlopage \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-              \count@=\@nwlopage \divide\count@ by 100
-              \@nwpagetemp=\@nwhipage \divide\@nwpagetemp by 100
-              \ifnum\count@=\@nwpagetemp %  lo--least 2 digits of hi
-                \multiply\@nwpagetemp by 100
-                \advance \@nwhipage by -\@nwpagetemp
-                \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-              \else \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-              \fi
-            \fi
-          \fi%
-    \fi
-  \fi%
-  \edef\@tempa{\noexpand\nwix@cons\noexpand\nw@pages{\@tempa}}\@tempa\def\\##1{\@nwhyperpagenum##1}%
-  \ifnum\@nwpagecount=1 \nwpageword \else \nwpagesword\fi~\commafy{\nw@pages}}
-\def\@nwhyperpagenum#1#2{\nwhyperreference{#2}{#1}}
-
-\def\@pagenumsl#1{%  list of labels -- doesn't include word `pages', commas, or `and'
-  \gdef\nw@pages{}\@nwpagecount=0
-  \def\\##1{\@nwfirstpagel{##1}\let\\=\@nwnextpagel}#1%
-  \advance\@nwhipage by \m@ne
-  \ifnum\@nwhipage=\@nwlopage
-       \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                          {\@nwloxreftag}}}%
-  \else
-    \count@=\@nwhipage \advance\count@ by \m@ne
-    \ifnum\count@=\@nwlopage % consecutive pages
-        \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                          {\@nwloxreftag}}%
-                      \noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwhisub}{\number\@nwhipage}}
-                          {\@nwhixreftag}}}%
-    \else \ifnum\@nwlopage<110 \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-            \count@=\@nwlopage \divide\count@ by 100 \multiply\count@ by 100
-            \ifnum\count@=\@nwlopage \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-              \count@=\@nwlopage \divide\count@ by 100
-              \@nwpagetemp=\@nwhipage \divide\@nwpagetemp by 100
-              \ifnum\count@=\@nwpagetemp %  lo--least 2 digits of hi
-                \multiply\@nwpagetemp by 100
-                \advance \@nwhipage by -\@nwpagetemp
-                \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-              \else \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-              \fi
-            \fi
-          \fi%
-    \fi
-  \fi%
-  \edef\@tempa{\noexpand\nwix@cons\noexpand\nw@pages{\@tempa}}\@tempa%
-  \def\\##1{\@nwhyperpagenum##1\let\\=\@nwpagenumslrest}\nw@pages}
-\def\@nwpagenumslrest#1{~\@nwhyperpagenum#1}
-\def\subpages#1{% list of {{subpage}{page}}
-  \gdef\nw@pages{}\@nwpagecount=0
-  \def\\##1{\edef\@tempa{\noexpand\@nwfirstpage##1{}}\@tempa
-            \def\\####1{\edef\@tempa{\noexpand\@nwnextpage####1}\@tempa}}#1%
-  \advance\@nwhipage by \m@ne
-  \ifnum\@nwhipage=\@nwlopage
-       \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                          {\@nwloxreftag}}}%
-  \else
-    \count@=\@nwhipage \advance\count@ by \m@ne
-    \ifnum\count@=\@nwlopage % consecutive pages
-        \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                          {\@nwloxreftag}}%
-                      \noexpand\noexpand\noexpand\\%
-                         {{\nwthepagenum{\number\@nwhisub}{\number\@nwhipage}}
-                          {\@nwhixreftag}}}%
-    \else \ifnum\@nwlopage<110 \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-            \count@=\@nwlopage \divide\count@ by 100 \multiply\count@ by 100
-            \ifnum\count@=\@nwlopage \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-              \count@=\@nwlopage \divide\count@ by 100
-              \@nwpagetemp=\@nwhipage \divide\@nwpagetemp by 100
-              \ifnum\count@=\@nwpagetemp %  lo--least 2 digits of hi
-                \multiply\@nwpagetemp by 100
-                \advance \@nwhipage by -\@nwpagetemp
-                \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-              \else \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-              \fi
-            \fi
-          \fi%
-    \fi
-  \fi%
-  \edef\@tempa{\noexpand\nwix@cons\noexpand\nw@pages{\@tempa}}\@tempa\def\\##1{\@firstoftwo##1}%
-  \ifnum\@nwpagecount=1 \nwpageword \else \nwpagesword\fi~\commafy{\nw@pages}}
-\def\@nwaddrange{\advance\@nwhipage by \m@ne
-                 \ifnum\@nwhipage=\@nwlopage
-                      \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                                        {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                                         {\@nwloxreftag}}}%
-                 \else
-                   \count@=\@nwhipage \advance\count@ by \m@ne
-                   \ifnum\count@=\@nwlopage % consecutive pages
-                       \edef\@tempa{\noexpand\noexpand\noexpand\\%
-                                        {{\nwthepagenum{\number\@nwlosub}{\number\@nwlopage}}%
-                                         {\@nwloxreftag}}%
-                                     \noexpand\noexpand\noexpand\\%
-                                        {{\nwthepagenum{\number\@nwhisub}{\number\@nwhipage}}
-                                         {\@nwhixreftag}}}%
-                   \else \ifnum\@nwlopage<110 \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-                           \count@=\@nwlopage \divide\count@ by 100 \multiply\count@ by 100
-                           \ifnum\count@=\@nwlopage \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}\else
-                             \count@=\@nwlopage \divide\count@ by 100
-                             \@nwpagetemp=\@nwhipage \divide\@nwpagetemp by 100
-                             \ifnum\count@=\@nwpagetemp %  lo--least 2 digits of hi
-                               \multiply\@nwpagetemp by 100
-                               \advance \@nwhipage by -\@nwpagetemp
-                               \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-                             \else \edef\@tempa{\noexpand\noexpand\noexpand\\{{\number\@nwlopage--\number\@nwhipage}{}}}%
-                             \fi
-                           \fi
-                         \fi%
-                   \fi
-                 \fi%
-                 \edef\@tempa{\noexpand\nwix@cons\noexpand\nw@pages{\@tempa}}\@tempa}
-\def\nwpageword{\@nwlangdepchk}  % chunk, was page
-\def\nwpagesword{\@nwlangdepchks}  % chunk, was page
-\def\nwpageprep{\@nwlangdepin}     % in, was on
-\newcommand\nw@genericref[2]{% what to do, name of ref
-  \expandafter\nw@g@nericref\csname r@#2\endcsname#1{#2}}
-\newcommand\nw@g@nericref[3]{% control sequence, what to do, name
-  \ifx#1\relax
-    \ref{#3}% trigger the standard `undefined ref' mechanisms
-  \else
-    \expandafter#2#1.\\%
-  \fi}
-\def\nw@selectone#1#2#3\\{#1}
-\def\nw@selecttwo#1#2#3\\{#2}
-\def\nw@selectonetwo#1#2#3\\{{#1}{#2}}
-\newcommand{\subpageref}[1]{%
-  \nwhyperreference{#1}{\nw@genericref\@subpageref{#1}}}
-\def\@subpageref#1#2#3\\{%
-  \@ifundefined{2on#2}{#2}{\nwthepagenum{#1}{#2}}}
-\newcommand{\subpagepair}[1]{%  % produces {subpage}{page}
-  \@ifundefined{r@#1}%
-    {{0}{0}}%
-    {\nw@genericref\@subpagepair{#1}}}
-\def\@subpagepair#1#2#3\\{%
-  \@ifundefined{2on#2}{{0}{#2}}{{#1}{#2}}}
-\newcommand{\sublabel}[1]{%
-  \leavevmode % needed to make \@bsphack work
-  \@bsphack
-  \nwblindhyperanchor{#1}%
-  \if@filesw {\let\thepage\relax
-   \def\protect{\noexpand\noexpand\noexpand}%
-   \edef\@tempa{\write\@auxout{\string
-      \newsublabel{#1}{{}{\thepage}}}}%
-   \expandafter}\@tempa
-   \if@nobreak \ifvmode\nobreak\fi\fi\fi\@esphack}
-\newcommand{\nosublabel}[1]{%
-  \@bsphack\if@filesw {\let\thepage\relax
-   \def\protect{\noexpand\noexpand\noexpand}%
-   \edef\@tempa{\write\@auxout{\string
-      \newlabel{#1}{{0}{\thepage}}}}%
-   \expandafter}\@tempa
-   \if@nobreak \ifvmode\nobreak\fi\fi\fi\@esphack}
-\newcommand\newsublabel{%
-  \nw@settrailers
-  \global\let\newsublabel\@newsublabel
-  \@newsublabel}
-\newcommand{\@newsublabel}[2]{%
-  \edef\this@page{\@cdr#2\@nil}%
-  \ifx\this@page\last@page\else
-    \sub@page=\z@
-  \fi
-  \edef\last@page{\this@page}
-  \advance\sub@page by \@ne
-  \ifnum\sub@page=\tw@
-    \global\@namedef{2on\this@page}{}%
-  \fi
-  \pendingsublabel{#1}%
-  \edef\@tempa##1{\noexpand\newlabel{##1}%
-    {{\number\sub@page}{\this@page}\nw@labeltrailers}}%
-  \pending@sublabels
-  \def\pending@sublabels{}}
-\newcommand\nw@settrailers{% -- won't work on first run
-  \@ifpackageloaded{nameref}%
-     {\gdef\nw@labeltrailers{{}{}{}}}%
-     {\gdef\nw@labeltrailers{}}}
-\renewcommand\nw@settrailers{% 
-  \@ifundefined{@secondoffive}%
-     {\gdef\nw@labeltrailers{}}%
-     {\gdef\nw@labeltrailers{{}{}{}}}}
-\newcommand{\nextchunklabel}[1]{%
-  \nwblindhyperanchor{#1}%   % looks slightly bogus --- nr
-  \@bsphack\if@filesw {\let\thepage\relax
-      \edef\@tempa{\write\@auxout{\string\pendingsublabel{#1}}}%
-      \expandafter}\@tempa
-   \if@nobreak \ifvmode\nobreak\fi\fi\fi\@esphack}
-\newcommand\pendingsublabel[1]{%
-  \def\@tempa{\noexpand\@tempa}%
-  \edef\pending@sublabels{\noexpand\@tempa{#1}\pending@sublabels}}
-\def\pending@sublabels{}
-\def\last@page{\relax}
-\newcount\sub@page
-\def\@alphasubpagenum#1#2{#2\ifnum#1=0 \else\@alph{#1}\fi}
-\def\@nosubpagenum#1#2{#2}
-\def\@numsubpagenum#1#2{#2\ifnum#1=0 \else.\@arabic{#1}\fi}
-\def\nwopt@nosubpage{\let\nwthepagenum=\@nosubpagenum\nwopt@nomargintag}
-\def\nwopt@numsubpage{\let\nwthepagenum=\@numsubpagenum}
-\def\nwopt@alphasubpage{\let\nwthepagenum=\@alphasubpagenum}
-\nwopt@alphasubpage
-\newcount\@nwalph@n
-\let\@nwalph@d\@tempcnta
-\let\@nwalph@bound\@tempcntb
-\def\@nwlongalph#1{{%
-  \@nwalph@n=#1\advance\@nwalph@n by-1
-  \@nwalph@bound=26
-  \loop\ifnum\@nwalph@n<\@nwalph@bound\else
-     \advance\@nwalph@n by -\@nwalph@bound
-     \multiply\@nwalph@bound by 26
-  \repeat
-  \loop\ifnum\@nwalph@bound>1
-    \divide\@nwalph@bound by 26
-    \@nwalph@d=\@nwalph@n\divide\@nwalph@d by \@nwalph@bound
-    % d := d * bound ; n -:= d; d := d / bound --- saves a temporary
-    \multiply\@nwalph@d by \@nwalph@bound
-    \advance\@nwalph@n by -\@nwalph@d
-    \divide\@nwalph@d by \@nwalph@bound
-    \advance\@nwalph@d by 1 \@alph{\@nwalph@d}%
-  \repeat
-}}
-\newcount\nw@chunkcount
-\nw@chunkcount=\@ne
-\newcommand{\weblabel}[1]{%
-  \@bsphack
-  \nwblindhyperanchor{#1}%
-  \if@filesw {\let\thepage\relax
-   \def\protect{\noexpand\noexpand\noexpand}%
-   \edef\@tempa{\write\@auxout{\string
-      \newsublabel{#1}{{}{\number\nw@chunkcount}}}}%
-   \expandafter}\@tempa
-   \global\advance\nw@chunkcount by \@ne
-   \if@nobreak \ifvmode\nobreak\fi\fi\fi\@esphack}
-\def\nwopt@webnumbering{%
-  \let\sublabel=\weblabel
-  \def\nwpageword{chunk}\def\nwpagesword{chunks}%
-  \def\nwpageprep{in}}
-% \nwindexdefn{printable name}{identifying label}{label of chunk}
-% \nwindexuse{printable name}{identifying label}{label of chunk}
-
-\def\nwindexdefn#1#2#3{\@auxix{\protect\nwixd}{#2}{#3}}
-\def\nwindexuse#1#2#3{\@auxix{\protect\nwixu}{#2}{#3}}
-
-\def\@auxix#1#2#3{% {marker}{id label}{subpage label}
-   \@bsphack\if@filesw {\let\nwixd\relax\let\nwixu\relax
-   \def\protect{\noexpand\noexpand\noexpand}%
-   \edef\@tempa{\write\@auxout{\string\nwixadd{#1}{#2}{#3}}}%
-   \expandafter}\@tempa
-   \if@nobreak \ifvmode\nobreak\fi\fi\fi\@esphack}
-% \nwixadd{marker}{idlabel}{subpage label}
-\def\nwixadd#1#2#3{%
-  \@ifundefined{nwixl@#2}%
-    {\global\@namedef{nwixl@#2}{#1{#3}}}%
-    {\expandafter\nwix@cons\csname nwixl@#2\endcsname{#1{#3}}}}
-\def\@nwsubscriptident#1#2{\mbox{$\mbox{#1}_{\mathrm{\subpageref{#2}}}$}}
-\def\@nwnosubscriptident#1#2{#1}
-\def\@nwhyperident#1#2{\leavevmode\nwhyperreference{#2}{#1}}
-\def\nwopt@subscriptidents{%
-  \let\nwlinkedidentq\@nwsubscriptident
-  \let\nwlinkedidentc\@nwsubscriptident
-}
-\def\nwopt@nosubscriptidents{%
-  \let\nwlinkedidentq\@nwnosubscriptident
-  \let\nwlinkedidentc\@nwnosubscriptident
-}
-\def\nwopt@hyperidents{%
-  \let\nwlinkedidentq\@nwhyperident
-  \let\nwlinkedidentc\@nwhyperident
-}
-\def\nwopt@nohyperidents{%
-  \let\nwlinkedidentq\@nwnosubscriptident
-  \let\nwlinkedidentc\@nwnosubscriptident
-}
-\def\nwopt@subscriptquotedidents{%
-  \let\nwlinkedidentq\@nwsubscriptident
-}
-\def\nwopt@nosubscriptquotedidents{%
-  \let\nwlinkedidentq\@nwnosubscriptident
-}
-\def\nwopt@hyperquotedidents{%
-  \let\nwlinkedidentq\@nwhyperident
-}
-\def\nwopt@nohyperquotedidents{%
-  \let\nwlinkedidentq\@nwnosubscriptident
-}
-\nwopt@hyperidents
-\newcount\@commacount
-\def\commafy#1{%
-  {\nwix@listcount{#1}\@commacount=\nwix@counter
-   \let\@comma@each=\\%
-   \ifcase\@commacount\let\\=\@comma@each\or\let\\=\@comma@each\or
-     \def\\{\def\\{ \@nwlangdepand\ \@comma@each}\@comma@each}\else
-     \def\\{\def\\{, %
-                   \advance\@commacount by \m@ne
-                   \ifnum\@commacount=1 \@nwlangdepand~\fi\@comma@each}\@comma@each}\fi
-   #1}}
-\def\nwix@cons#1#2{% {list}{\marker{element}}
-  {\toks0=\expandafter{#1}\def\@tempa{#2}\toks2=\expandafter{\@tempa}%
-   \xdef#1{\the\toks0 \the\toks2 }}}
-\def\nwix@uses#1{% {label}
-  \def\nwixu{\\}\let\nwixd\@gobble\@nameuse{nwixl@#1}}
-\def\nwix@defs#1{% {label}
-  \def\nwixd{\\}\let\nwixu\@gobble\@nameuse{nwixl@#1}}
-\newcount\nwix@counter
-\def\nwix@listcount#1{% {list with \\}
-  {\count@=0
-   \def\\##1{\advance\count@ by \@ne }%
-   #1\global\nwix@counter=\count@ }}
-\def\nwix@usecount#1{\nwix@listcount{\nwix@uses{#1}}}
-\def\nwix@defcount#1{\nwix@listcount{\nwix@defs{#1}}}
-\def\nwix@id@defs#1{% index pair
-  {{\Tt \@car#1\@nil}%
-  \def\\##1{\nwix@defs@space\subpageref{##1}}\nwix@defs{\@cdr#1\@nil}}}
-  % useful above to change ~ into something that can break
-% this option is undocumented because I think breakdefs is always right
-\def\nwopt@breakdefs{\def\nwix@defs@space{\penalty200\ }}
-\def\nwopt@nobreakdefs{\def\nwix@defs@space{~}} % old code
-\nwopt@breakdefs
-\def\nwidentuses#1{% list of index pairs
-  \nwcodecomment{\@nwlangdepuss\ \let\\=\nwix@id@defs\commafy{#1}.}}
-\def\nwix@totaluses#1{% list of index pairs
-  {\count@=0
-   \def\\##1{\nwix@usecount{\@cdr##1\@nil}\advance\count@ by\nwix@counter}%
-   #1\global\nwix@counter\count@ }}
-\def\nwix@id@uses#1#2{% {ident}{label}
-  \nwix@usecount{#2}\ifnum\nwix@counter>0
-    {\advance\leftskip by \codemargin
-     \nwcodecomment{{\Tt #1}, \@nwlangdepusd\ \nwpageprep\ \@pagesl{\nwix@uses{#2}}.}}%
-  \else
-    \ifnw@hideunuseddefs\else
-      {\advance\leftskip by \codemargin \nwcodecomment{{\Tt #1}, \@nwlangdepnvu.}}%
-    \fi
-  \fi}
-\def\nwidentdefs#1{% list of index pairs
-  \ifnw@hideunuseddefs\nwix@totaluses{#1}\else\nwix@listcount{#1}\fi
-  \ifnum\nwix@counter>0
-    \nwcodecomment{\@nwlangdepdfs:}%
-    {\def\\##1{\nwix@id@uses ##1}#1}%
-  \fi}
-\newif\ifnw@hideunuseddefs\nw@hideunuseddefsfalse
-\def\nwopt@hideunuseddefs{\nw@hideunuseddefstrue}
-\def\nwopt@noidentxref{%
-  \let\nwidentdefs\@gobble
-  \let\nwidentuses\@gobble}
-\def\nw@underlinedefs{% {list with \nwixd, \nwixu}
-  \let\\=\relax\def\nw@comma{, }
-  \def\nwixd##1{\\\underline{\subpageref{##1}}\let\\\nw@comma}%
-  \def\nwixu##1{\\\subpageref{##1}\let\\\nw@comma}}
-
-\def\nw@indexline#1#2{%
-   {\indent {\Tt #1}: \nw@underlinedefs\@nameuse{nwixl@#2}\par}}
-
-\newenvironment{thenowebindex}{\parindent=-10pt \parskip=\z@ 
-        \advance\leftskip by 10pt 
-        \advance\rightskip by 0pt plus1in\par\@afterindenttrue
-    \def\\##1{\nw@indexline##1}}{}
-\def\nowebindex{%
-  \@ifundefined{nwixs@i}%
-     {\@warning{The \string\nowebindex\space is empty}}%
-     {\begin{thenowebindex}\@nameuse{nwixs@i}\end{thenowebindex}}}
-\def\nowebindex@external{%
-  {\let\nwixadds@c=\@gobble
-   \def\nwixadds@i##1{\nw@indexline##1}%
-   \def\nwixaddsx##1##2{\@nameuse{nwixadds@##1}{##2}}%
-   \begin{thenowebindex}\@input{\jobname.nwi}\end{thenowebindex}}}
-\def\nwixlogsorted#1#2{% list data
-   \@bsphack\if@filesw 
-     \toks0={#2}\immediate\write\@auxout{\string\nwixadds{#1}{\the\toks0}}
-   \if@nobreak \ifvmode\nobreak\fi\fi\fi\@esphack}
-\def\nwixadds#1#2{%
-  \@ifundefined{nwixs@#1}%
-    {\global\@namedef{nwixs@#1}{\\{#2}}}%
-    {\expandafter\nwix@cons\csname nwixs@#1\endcsname{\\{#2}}}}
-\let\nwixaddsx=\@gobbletwo
-\def\nwopt@externalindex{%
-  \ifx\nwixadds\@gobbletwo % already called
-  \else
-    \let\nwixaddsx=\nwixadds \let\nwixadds=\@gobbletwo
-    \let\nowebindex=\nowebindex@external
-    \let\nowebchunks=\nowebchunks@external
-  \fi}
-\def\nowebchunks{%
-  \@ifundefined{nwixs@c}%
-     {\@warning{The are no \string\nowebchunks}}%
-     {\begin{thenowebchunks}\@nameuse{nwixs@c}\end{thenowebchunks}}}
-\def\nowebchunks@external{%
-  {\let\nwixadds@i=\@gobble
-   \def\nwixadds@c##1{\nw@onechunk##1}%
-   \def\nwixaddsx##1##2{\@nameuse{nwixadds@##1}{##2}}%
-   \begin{thenowebchunks}\@input{\jobname.nwi}\end{thenowebchunks}}}
-    \@namedef{r@nw@notdef}{{0}{(\@nwlangdepnvd)}}
-\def\nw@chunkunderlinedefs{% {list of labels with \nwixd, \nwixu}
-  \let\\=\relax\def\nw@comma{, }
-  \def\nwixd##1{\\\underline{\subpageref{##1}}\let\\\nw@comma}%
-  \def\nwixu##1{\\\subpageref{##1}\let\\\nw@comma}}
-\def\nw@onechunk#1#2#3{% {name}{label of first definition}{list with \nwixd, \nwixu}
-  \@ifundefined{r@#2}{}{%
-    \indent\LA #1~{\nwtagstyle\subpageref{#2}}\RA 
-    \if@nwlongchunks{~\nw@chunkunderlinedefs#3}\fi\par}}
-\newenvironment{thenowebchunks}{\vskip3pt
-  \parskip=\z@\parindent=-10pt \advance\leftskip by 10pt
-  \advance\rightskip by 0pt plus10pt \@afterindenttrue
-  \def\\##1{\nw@onechunk##1}}{}
-\newif\if@nwlongchunks
-\@nwlongchunksfalse
-\let\nwopt@longchunks\@nwlongchunkstrue
-\newcommand\@nw@hyper@ref{\hyperreference} % naras
-\newcommand\@nw@hyper@anc{\blindhyperanchor} % naras
-\newcommand\@nw@hyperref@ref[2]{\hyperlink{noweb.#1}{#2}}  % nr
-\newcommand\@nw@hyperref@anc[1]{\hypertarget{noweb.#1}{\relax}}  % nr
-%%\renewcommand\@nw@hyperref@ref[2]{{#2}}  % nr
-%%\renewcommand\@nw@hyperref@anc[1]{}  % nr
-\newcommand\nwhyperreference{%
-  \@ifundefined{hyperlink}
-    {\@ifundefined{hyperreference}
-       {\global\let\nwhyperreference\@gobble}
-       {\global\let\nwhyperreference\@nw@hyper@ref}}
-    {\global\let\nwhyperreference\@nw@hyperref@ref}%
-  \nwhyperreference
-}
-
-\newcommand\nwblindhyperanchor{%
-  \@ifundefined{hyperlink}
-    {\@ifundefined{hyperreference}
-       {\global\let\nwblindhyperanchor\@gobble}
-       {\global\let\nwblindhyperanchor\@nw@hyper@anc}}
-    {\global\let\nwblindhyperanchor\@nw@hyperref@anc}%
-  \nwblindhyperanchor
-}
-\newcommand\nwanchorto{%
-  \begingroup\let\do\@makeother\dospecials
-     \catcode`\{=1 \catcode`\}=2 \nw@anchorto}
-\newcommand\nw@anchorto[1]{\endgroup\def\nw@next{#1}\nw@anchortofin}
-\newcommand\nw@anchortofin[1]{#1\footnote{See URL \texttt{\nw@next}.}}
-\let\nwanchorname\@gobble
-\newif\ifhtml
-\htmlfalse
-\let\nwixident=\relax
-\def\nwbackslash{\char92}
-\def\nwlbrace{\char123}
-\def\nwrbrace{\char125}
-\def\nwopt@english{%
-  \def\@nwlangdepdef{This definition is continued}%
-  \def\@nwlangdepcud{This code is used}%
-  \def\@nwlangdeprtc{Root chunk (not used in this document)}%
-  \def\@nwlangdepcwf{This code is written to file}%
-  \def\@nwlangdepchk{chunk}%
-  \def\@nwlangdepchks{chunks}%
-  \def\@nwlangdepin{in}%
-  \def\@nwlangdepand{and}%
-  \def\@nwlangdepuss{Uses}%
-  \def\@nwlangdepusd{used}%
-  \def\@nwlangdepnvu{never used}%
-  \def\@nwlangdepdfs{Defines}%
-  \def\@nwlangdepnvd{never defined}%
-}
-\let\nwopt@american\nwopt@english
-\def\nwopt@portuges{%
-  \def\@nwlangdepdef{Defini\c{c}\~ao continuada em}%
-  % This definition is continued
-  \def\@nwlangdepcud{C\'odigo usado em}%
-  % This code is used
-  \def\@nwlangdeprtc{Fragmento de topo (sem uso no documento)}%
-  % Root chunk (not used in this document)
-  \def\@nwlangdepcwf{Este c\'odigo foi escrito no ficheiro}%
-  % This code is written to file
-  \def\@nwlangdepchk{fragmento}%
-  % chunk
-  \def\@nwlangdepchks{fragmentos}%
-  % chunks
-  \def\@nwlangdepin{no(s)}%
-  % in
-  \def\@nwlangdepand{e}%
-  % and
-  \def\@nwlangdepuss{Usa}%
-  % Uses
-  \def\@nwlangdepusd{usado}%
-  % used
-  \def\@nwlangdepnvu{nunca usado}%
-  % never used
-  \def\@nwlangdepdfs{Define}%
-  % Defines
-  \def\@nwlangdepnvd{nunca definido}%
-  % never defined
-}
-\def\nwopt@brazil{%
-  \def\@nwlangdepdef{Defini\c{c}\~{a}o continua}%
-  % This definition is continued
-  \def\@nwlangdepcud{C\'odigo usado}%
-  % This code is used
-  \def\@nwlangdeprtc{Trecho raiz, isto é, este trecho come\c{c}a aqui}%
-  % Root chunk (not used in this document)
-  \def\@nwlangdepcwf{C\'odigo escrito no arquivo}%
-  % This code is written to file
-  \def\@nwlangdepchk{trecho}%
-  % chunk
-  \def\@nwlangdepchks{trechos}%
-  % chunks
-  \def\@nwlangdepin{@}%
-  % in
-  \def\@nwlangdepand{e}%
-  % and
-  \def\@nwlangdepuss{Usa}%
-  % Uses
-  \def\@nwlangdepusd{usado}%
-  % used
-  \def\@nwlangdepnvu{nunca usado}%
-  % never used
-  \def\@nwlangdepdfs{Define}%
-  % Defines
-  \def\@nwlangdepnvd{nunca definido}%
-  % never defined
-}
-\def\nwopt@frenchb{%
-  \def\@nwlangdepdef{Suite de la d\'efinition}%
-  % This definition is continued
-  \def\@nwlangdepcud{Ce code est employ\'e}%
-  % This code is used
-  \def\@nwlangdeprtc{Morceau racine (pas employ\'e dans ce document)}%
-  % Root chunk (not used in this document)
-  \def\@nwlangdepcwf{Ce code est \'ecrit dans le fichier}%
-  % This code is written to file
-  \def\@nwlangdepchk{le morceau}%
-  % chunk
-  \def\@nwlangdepchks{les morceaux}%
-  % chunks
-  \def\@nwlangdepin{dans}%
-  % in
-  \def\@nwlangdepand{et}%
-  % and
-  \def\@nwlangdepuss{Utilise}%
-  % Uses
-  \def\@nwlangdepusd{utilis\'{e}}%
-  % used
-  \def\@nwlangdepnvu{jamais employ\'{e}}%
-  % never used
-  \def\@nwlangdepdfs{D\'{e}finit}%
-  % Defines
-  % Cannot use the accent here: \def\@nwlangdepnvd{jamais d\'{e}fini}%
-  \def\@nwlangdepnvd{jamais defini}%
-  % never defined
-}
-\let\nwopt@french\nwopt@frenchb
-\def\nwopt@german{%
-  \def\@nwlangdepdef{Diese Definition wird fortgesetzt}%
-  % This definition is continued
-  \def\@nwlangdepcud{Dieser Code wird benutzt}%
-  % This code is used
-  \def\@nwlangdeprtc{Hauptteil (nicht in diesem Dokument benutzt)}%
-  % Root chunk (not used in this document)
-  %\def\@nwlangdepcwf{Dieser Code schreibt man zum File}%
-  \def\@nwlangdepcwf{Dieser Code geht in Datei}%
-  % This code is written to file
-  \def\@nwlangdepchk{dem Teil}%
-  % chunk
-  %\def\@nwlangdepchks{Teils}%   
-  \def\@nwlangdepchks{den Teilen}%   
-  % chunks
-  \def\@nwlangdepin{in}%
-  % in
-  \def\@nwlangdepand{und}%
-  % and
-  \def\@nwlangdepuss{Benutzt}%
-  % Uses
-  \def\@nwlangdepusd{benutzt}%
-  % used
-  \def\@nwlangdepnvu{nicht benutzt}%
-  % never used
-  \def\@nwlangdepdfs{Definiert}%
-  % Defines
-  \def\@nwlangdepnvd{nicht definiert}%
-  % never defined
-}
-\def\nwopt@german{%
-  \def\@nwlangdepdef{Diese Definition wird fortgesetzt}%
-  % This definition is continued
-  \def\@nwlangdepcud{Dieser Code wird benutzt}%
-  % This code is used
-  \def\@nwlangdeprtc{Hauptteil (nicht in diesem Dokument benutzt)}%
-  % Root chunk (not used in this document)
-  \def\@nwlangdepcwf{Dieser Code schreibt man zum File}%
-  % This code is written to file
-  \def\@nwlangdepchk{Teil}%
-  % chunk
-  \def\@nwlangdepchks{Teils}%
-  % chunks
-  \def\@nwlangdepin{im}%
-  % in
-  \def\@nwlangdepand{und}%
-  % and
-  \def\@nwlangdepuss{Benutzt}%
-  % Uses
-  \def\@nwlangdepusd{benutzt}%
-  % used
-  \def\@nwlangdepnvu{nicht benutzt}%
-  % never used
-  \def\@nwlangdepdfs{Definiert}%
-  % Defines
-  \def\@nwlangdepnvd{nicht definiert}%
-  % never defined
-}
-\let\nwopt@ngerman\nwopt@german
-\ifx\languagename\undefined % default is English
-  \noweboptions{english}
-\else
-  \@ifundefined{nwopt@\languagename}
-     {\noweboptions{english}}
-     {\expandafter\noweboptions\expandafter{\languagename}}
-\fi
-\let\obeyedspace\@xobeysp

scripts/build-index-from-fs.lisp

@@ -5,7 +5,57 @@
     (load quicklisp-init)))
 (ql:quickload :loop :silent t)
 (in-package #:loop)
-(connect-index! "message-id.db")
+(defun index-from-fs! ()
-(drop-create-index!)
+  (loop for path in (in-groups (directory "**/*"))
+        do (let* ((g (str:trim (first (last (pathname-directory path)))))
+                  (i (str:trim (pathname-name path)))
+                  (m (ignore-errors (str:trim (extract-mid (fetch-article g i))))))
+             (when (and m (> (length m) 0))
+               (let* ((hash (index-hash m))
+                      (idx-path (index-path hash)))
+                 (ensure-index-directories! hash)
+                 ;; If index file exists, verify if it shares our inode. If not, force-link it.
+                 (cond
+                   ((probe-file idx-path)
+                    (unless (= (handler-case (sb-posix:stat-ino (sb-posix:stat (uiop:native-namestring path)))
+                                 (sb-posix:syscall-error () 0))
+                               (handler-case (sb-posix:stat-ino (sb-posix:stat (uiop:native-namestring idx-path)))
+                                 (sb-posix:syscall-error () -1)))
+                      ;; Overwrite stale/mismatched index entry with the correct hard link
+                      (ignore-errors (delete-file idx-path))
+                      (sb-posix:link (uiop:native-namestring path) idx-path)
+                      (format t "article ~a/~a force-linked to index ~a~%" g i hash)))
+                   (t
+                    (handler-case
+                        (progn
+                          (sb-posix:link (uiop:native-namestring path) idx-path)
+                          (format t "article ~a/~a indexed by ~a hash ~a~%" g i m hash))
+                      (sb-posix:syscall-error (c)
+                        (if (= (sb-posix:syscall-errno c) sb-posix:eexist)
+                            (format t "article ~a/~a already indexed concurrently.~%" g i)
+                            (error c)))))))))))
+
+(defun index-from-fs!-old ()
+  (loop for path in (in-groups (directory "**/*"))
+        do (let* ((g (str:trim (first (last (pathname-directory path)))))
+                  (i (str:trim (pathname-name path)))
+                  (m (ignore-errors (str:trim (extract-mid (fetch-article g i))))))
+             (when (and m (> (length m) 0))
+               (let* ((hash (index-hash m))
+                      (idx-path (index-path hash)))
+                 (if (probe-file idx-path) ;; article exists in index: unify inodes by re-linking group file to index
+                     (unless (= (get-file-inode path) (get-file-inode idx-path))
+                       (delete-file path)
+                       (sb-posix:link (uiop:native-namestring idx-path) (uiop:native-namestring path))
+                       (format t "article ~a/~a relinked-to-existing-index ~a~%" g i hash))
+                     (progn
+                       (ensure-index-directories! hash)
+                       (handler-case
+                           (progn
+                             (sb-posix:link (uiop:native-namestring path) idx-path)
+                             (format t "article ~a/~a mid ~a hashes-to ~a~%" g i m hash))
+                         (sb-posix:syscall-error (c)
+                           (if (= (sb-posix:syscall-errno c) sb-posix:eexist)
+                               (format t "article ~a/~a indexed due to a cross-post? EEXIST.~%" g i)
+                               (error c)))))))))))
 (index-from-fs!)
-(format t "Index built.~%")

scripts/cron-remove-inactive-users.lisp

@@ -6,6 +6,5 @@
 (ql:quickload :loop :silent t)
 (in-package #:loop)
 (read-accounts!)
-(connect-index! "message-id.db")
 (remove-inactive-users!)
 (write-accounts!)

scripts/find-duplicates.lisp

@@ -0,0 +1,37 @@
+#-quicklisp
+(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
+                                       (user-homedir-pathname))))
+  (when (probe-file quicklisp-init)
+    (load quicklisp-init)))
+(ql:quickload :loop :silent t)
+(in-package #:loop)
+(defun find-duplicate-mids ()
+  "Displays any duplicated message-ids across all group articles."
+  (let ((mid-map (make-hash-table :test 'equal))
+        (found-dupes nil))
+    ;; collect all articles and group by message-id
+    (loop for path in (in-groups (directory "**/*"))
+          do (let* ((dir-list (pathname-directory path))
+                    (g (when dir-list (str:trim (first (last dir-list)))))
+                    (i (pathname-name path))
+                    (i-str (when i (str:trim i))))
+               ;; process numeric article files only
+               (when (and g i-str (parse-integer i-str :junk-allowed t))
+                 (let ((mid (ignore-errors (get-mid-from-path path))))
+                   (when (and mid (> (length (str:trim mid)) 0))
+                     ;; push the (group . article-number) pair into the hash map
+                     (push (cons g i-str) (gethash mid mid-map)))))))
+    ;; remove duplicates
+    (maphash (lambda (mid locations)
+               (when (> (length locations) 1)
+                 (push (cons mid locations) found-dupes)))
+             mid-map)
+    ;; displays
+    (when found-dupes
+      (loop for (mid . locs) in found-dupes
+            do (format t "message-id: ~a~%" mid)
+            ;; sort by group
+            (let ((sorted-locs (sort locs #'string-lessp :key #'car)))
+              (loop for (g . i) in sorted-locs
+                    do (format t "~a/~a~%" g i)))))))
+(find-duplicate-mids)

scripts/verify-index.lisp

@@ -0,0 +1,37 @@
+#-quicklisp
+(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
+                                       (user-homedir-pathname))))
+  (when (probe-file quicklisp-init)
+    (load quicklisp-init)))
+(ql:quickload :loop :silent t)
+(in-package #:loop)
+(defun verify-index-inodes ()
+  "Iterates over all group articles and prints whether they share an inode with index/."
+  (loop for path in (in-groups (directory "**/*"))
+        do (let* ((dir-list (pathname-directory path))
+                  (g (when dir-list (str:trim (first (last dir-list)))))
+                  (i (pathname-name path))
+                  (i-str (when i (str:trim i))))
+             ;; ignore metadata and directories
+             (when (and g i-str (parse-integer i-str :junk-allowed t))
+               (let ((mid (ignore-errors (get-mid-from-path path))))
+                 (if (or (null mid) (zerop (length (str:trim mid))))
+                     (format t "no-mid  ~a/~a couldn't find message-id~%" g i-str)
+                     (let* ((hash (index-hash mid))
+                            (idx-path (index-path hash))
+                            (grp-inode (get-file-inode path))
+                            (idx-inode (get-file-inode idx-path)))
+                       (cond
+                         ((null idx-inode)
+                          (stdout "no-index ~a/~a mid ~a hashes-to ~a~%"
+                                  g i-str mid idx-path))
+                         ((= grp-inode idx-inode)
+                          (stdout "okay ~a/~a shares-inode ~a with ~a~%"
+                                  g i-str grp-inode idx-path))
+                         (t
+                          (stdout "mismatch ~a/~a inode ~a =/= inode ~a hash ~a~%"
+                                  g i-str grp-inode idx-inode idx-path))))))))))
+
+(defun verify-index ()
+  (verify-index-inodes))
+(verify-index)

version.lisp

@@ -0,0 +1,2 @@
+(in-package :loop)
+(defparameter *git-hash* "da757c7")