diff --git a/Makefile b/Makefile index a58abcffe..8eaeb1e53 100644 --- a/Makefile +++ b/Makefile @@ -301,7 +301,7 @@ indent.c: # simple want to examine, run / test winning IOCCC entries. .PHONY: help genpath genfilelist verify_entry_files gen_authors quick_authors \ - gen_location gen_years find_missing_links test entry_index gen_top_html \ + gen_location quick_location gen_years find_missing_links test entry_index gen_top_html \ thanks gen_other_html quick_other_html quick_entry_index \ gen_year_index quick_www www untar_entry_tarball untar_year_tarball \ form_entry_tarball form_year_tarball tar gen_status gen_sitemap \ @@ -338,6 +338,7 @@ help: @echo 'make gen_authors ;: generate the top level authors.html page' @echo 'make quick_authors ;: build authors.html if out of date with author JSON files' @echo 'make gen_location ;: generate the top level location.html page' + @echo 'make quick_location ;: build location.html if out of date with author JSON files' @echo 'make gen_years ;: generate the top level years.html page' @echo 'make entry_index ;: force the build of ALL winner index.html files' @echo 'make gen_other_html ;: build entry HTML files from markdown other than README.md' @@ -491,6 +492,16 @@ gen_location: ${GEN_LOCATION} location.html ${GEN_LOCATION} -v 1 @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' +# build location.html if it is out of date from any author/author_handle.json file +# +# generate the top level location.html page using the +# ${GEN_LOCATION} tool (bin/gen-location.sh). +# +quick_location: ${GEN_LOCATION} location.html + @echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-=' + ${GEN_LOCATION} -v 1 -Q + @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' + # generate the top level years.html page using the ${GEN_YEARS} # tool (bin/gen-years.sh). # @@ -677,7 +688,7 @@ quick_www: ${MAKE} genfilelist ${MAKE} verify_entry_files ${MAKE} quick_authors - ${MAKE} gen_location + ${MAKE} quick_location ${MAKE} gen_years ${MAKE} gen_year_index ${MAKE} gen_top_html diff --git a/bin/gen-location.sh b/bin/gen-location.sh index e943e4e48..0a1ec3de3 100755 --- a/bin/gen-location.sh +++ b/bin/gen-location.sh @@ -84,7 +84,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.4.5 2024-08-05" +export VERSION="1.5 2024-08-17" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -102,7 +102,6 @@ fi export TOPDIR export DOCROOT_SLASH="./" export TAGLINE="bin/$NAME" -export MD2HTML_SH="bin/md2html.sh" export PANDOC_WRAPPER="bin/pandoc-wrapper.sh" export REPO_TOP_URL="https://github.com/ioccc-src/temp-test-ioccc" # GitHub puts individual files under the "blob/master" sub-directory. @@ -111,7 +110,7 @@ export SITE_URL="https://ioccc-src.github.io/temp-test-ioccc" # export NOOP= export DO_NOT_PROCESS= -export EXIT_CODE="0" +QUICK_MODE= # clear options we will add to tools # @@ -121,7 +120,7 @@ declare -ag TOOL_OPTION # usage # export USAGE="usage: $0 [-h] [-v level] [-V] [-d topdir] [-D docroot/] [-n] [-N] - [-t tagline] [-T md2html.sh] [-p tool] + [-t tagline] [-p tool] [-Q] [-w site_url] -h print help message and exit -v level set verbosity level (def level: 0) @@ -139,11 +138,12 @@ export USAGE="usage: $0 [-h] [-v level] [-V] [-d topdir] [-D docroot/] [-n] [-N] -t tagline string to write about the tool that formed the markdown content (def: $TAGLINE) NOTE: 'tagline' may be enclosed within, but may NOT contain an internal single-quote, or double-quote. - -T md2html.sh run 'markdown to html tool' to convert markdown into HTML (def: $MD2HTML_SH) -p tool run 'pandoc wrapper tool' (not pandoc path) during HTML phase number 21 (def: use $PANDOC_WRAPPER) NOTE: The '-p tool' is passed as leading options on tool command lines. + -Q quick mode, do not run $MD2HTML_SH unless markdown is out of date (def: do) + -w site_url Base URL of the website (def: $SITE_URL) NOTE: The '-w site_url' is passed as leading options on tool command lines. @@ -167,7 +167,7 @@ $NAME version: $VERSION" # parse command line # -while getopts :hv:Vd:D:nNt:T:p:w: flag; do +while getopts :hv:Vd:D:nNt:p:Qw: flag; do case "$flag" in h) echo "$USAGE" 1>&2 exit 2 @@ -216,14 +216,12 @@ while getopts :hv:Vd:D:nNt:T:p:w: flag; do TAGLINE="$OPTARG" # -t tagline always added after arg parsing ;; - T) MD2HTML_SH="$OPTARG" - TOOL_OPTION+=("-T") - TOOL_OPTION+=("$MD2HTML_SH") - ;; p) PANDOC_WRAPPER="$OPTARG" TOOL_OPTION+=("-p") TOOL_OPTION+=("$PANDOC_WRAPPER") ;; + Q) QUICK_MODE="-Q" + ;; w) SITE_URL="$OPTARG" TOOL_OPTION+=("-w") TOOL_OPTION+=("$SITE_URL") @@ -348,19 +346,37 @@ export BIN_DIR="bin" # verify that the md2html tool is executable # +export MD2HTML_SH="$BIN_DIR/md2html.sh" if [[ ! -e $MD2HTML_SH ]]; then - echo "$0: ERROR: md2html.sh does not exist: $MD2HTML_SH" 1>&2 + echo "$0: ERROR: bin/md2html.sh does not exist: $MD2HTML_SH" 1>&2 exit 5 fi if [[ ! -f $MD2HTML_SH ]]; then - echo "$0: ERROR: md2html.sh is not a regular file: $MD2HTML_SH" 1>&2 + echo "$0: ERROR: bin/md2html.sh is not a regular file: $MD2HTML_SH" 1>&2 exit 5 fi if [[ ! -x $MD2HTML_SH ]]; then - echo "$0: ERROR: md2html.sh is not an executable file: $MD2HTML_SH" 1>&2 + echo "$0: ERROR: bin/md2html.sh is not an executable file: $MD2HTML_SH" 1>&2 exit 5 fi +# verify jval-wrapper tool is executable +# +export JVAL_WRAPPER="$BIN_DIR/jval-wrapper.sh" +if [[ ! -e $JVAL_WRAPPER ]]; then + echo "$0: ERROR: bin/jval-wrapper.sh file does not exist: $JVAL_WRAPPER" 1>&2 + exit 5 +fi +if [[ ! -f $JVAL_WRAPPER ]]; then + echo "$0: ERROR: bin/jval-wrapper.sh is not a file: $JVAL_WRAPPER" 1>&2 + exit 5 +fi +if [[ ! -x $JVAL_WRAPPER ]]; then + echo "$0: ERROR: cannot find an executable bin/jval-wrapper.sh tool: $JVAL_WRAPPER" 1>&2 + exit 5 +fi +# + # find the location tool # LOCATION_TOOL=$(type -P location) @@ -399,7 +415,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: SITE_URL=$SITE_URL" 1>&2 echo "$0: debug[3]: NOOP=$NOOP" 1>&2 echo "$0: debug[3]: DO_NOT_PROCESS=$DO_NOT_PROCESS" 1>&2 - echo "$0: debug[3]: EXIT_CODE=$EXIT_CODE" 1>&2 + echo "$0: debug[3]: QUICK_MODE=$QUICK_MODE" 1>&2 for index in "${!TOOL_OPTION[@]}"; do echo "$0: debug[3]: TOOL_OPTION[$index]=${TOOL_OPTION[$index]}" 1>&2 done @@ -409,6 +425,8 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: AUTHOR_DIR=$AUTHOR_DIR" 1>&2 echo "$0: debug[3]: BIN_PATH=$BIN_PATH" 1>&2 echo "$0: debug[3]: BIN_DIR=$BIN_DIR" 1>&2 + echo "$0: debug[3]: MD2HTML_SH=$MD2HTML_SH" 1>&2 + echo "$0: debug[3]: JVAL_WRAPPER=$JVAL_WRAPPER" 1>&2 echo "$0: debug[3]: LOCATION_TOOL=$LOCATION_TOOL" 1>&2 echo "$0: debug[3]: LOCATION_HTML=$LOCATION_HTML" 1>&2 fi @@ -422,6 +440,17 @@ if [[ -n $DO_NOT_PROCESS ]]; then exit 0 fi +# case: -Q +# +# Do nothing if authors.html is newer than all author/author_handle.json files. +# +if [[ -n $QUICK_MODE ]]; then + NEWER=$(find "$AUTHOR_DIR" -mindepth 1 -maxdepth 1 -type f -name '*.json' -newer "$LOCATION_HTML" 2>/dev/null) + if [[ -z $NEWER ]]; then + exit 0 + fi +fi + # create a temporary location markdown file # export TMP_LOC_MD=".tmp.$NAME.LOC_MD.$$.tmp" @@ -444,23 +473,178 @@ elif [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: because of -n, temporary location markdown file is not used: $TMP_LOC_MD" 1>&2 fi -# generate the temporary location markdown file +# create a temporary location sort_word handle name set # -# XXX - XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - XXX -# XXX - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - XXX -# XXX - until we have the jnamval command, we must FAKE PARSE the author/author_handle.json file - XXX -# XXX - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - XXX -# XXX - XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - XXX +export TMP_LOC_WORD_HANDLE_NAME_SET=".tmp.$NAME.LOC_AUTHOR_HANDLE.$$.tmp" +if [[ $V_FLAG -ge 3 ]]; then + echo "$0: debug[3]: temporary location sort_word handle name set: $TMP_LOC_WORD_HANDLE_NAME_SET" 1>&2 +fi +if [[ -z $NOOP ]]; then + trap 'rm -f $TMP_LOC_MD $TMP_LOC_WORD_HANDLE_NAME_SET; exit' 0 1 2 3 15 + rm -f "$TMP_LOC_WORD_HANDLE_NAME_SET" + if [[ -e $TMP_LOC_WORD_HANDLE_NAME_SET ]]; then + echo "$0: ERROR: cannot remove temporary location sort_word handle name set: $TMP_LOC_WORD_HANDLE_NAME_SET" 1>&2 + exit 12 + fi + :> "$TMP_LOC_WORD_HANDLE_NAME_SET" + if [[ ! -e $TMP_LOC_WORD_HANDLE_NAME_SET ]]; then + echo "$0: ERROR: cannot create temporary location sort_word handle name set: $TMP_LOC_WORD_HANDLE_NAME_SET" 1>&2 + exit 13 + fi +elif [[ $V_FLAG -ge 3 ]]; then + echo "$0: debug[3]: because of -n, temporary location sort_word handle name set is not used: $TMP_LOC_WORD_HANDLE_NAME_SET" 1>&2 +fi + +# create a temporary location count set # -{ - # process unique location_codes found in author_handle JSON files in dictionary order +export TMP_LOC_COUNT=".tmp.$NAME.LOC_COUNT.$$.tmp" +if [[ $V_FLAG -ge 3 ]]; then + echo "$0: debug[3]: temporary location count set: $TMP_LOC_COUNT" 1>&2 +fi +if [[ -z $NOOP ]]; then + trap 'rm -f $TMP_LOC_MD $TMP_LOC_WORD_HANDLE_NAME_SET $TMP_LOC_COUNT; exit' 0 1 2 3 15 + rm -f "$TMP_LOC_COUNT" + if [[ -e $TMP_LOC_COUNT ]]; then + echo "$0: ERROR: cannot remove temporary location count set: $TMP_LOC_COUNT" 1>&2 + exit 14 + fi + :> "$TMP_LOC_COUNT" + if [[ ! -e $TMP_LOC_COUNT ]]; then + echo "$0: ERROR: cannot create temporary location count set: $TMP_LOC_COUNT" 1>&2 + exit 15 + fi +elif [[ $V_FLAG -ge 3 ]]; then + echo "$0: debug[3]: because of -n, temporary location count set is not used: $TMP_LOC_COUNT" 1>&2 +fi + + +# generate location sort_word author_handle author name set +# +if [[ $V_FLAG -ge 1 ]]; then + echo "$0: debug[1]: about to obtain location codes from author_handle JSON files" 1>&2 +fi +export AUTHOR_COUNT=0 +export AUTHOR_COUNT_CYCLE=20 +find "$AUTHOR_DIR" -mindepth 1 -maxdepth 1 -type f -name '*.json' -print | + while read -r AUTHOR_HANDLE_JSON; do + + # collect location_code + # + LOCATION_CODE=$("$JVAL_WRAPPER" -b -w "$AUTHOR_HANDLE_JSON" '$..location_code') + if [[ -z $LOCATION_CODE ]]; then + echo "$0: ERROR: cannot obtain location_code from: $AUTHOR_HANDLE_JSON" 1>&2 + exit 1 + fi + + # obtain author_handle from author_handle file + # + AUTHOR_HANDLE=$(basename "$AUTHOR_HANDLE_JSON" .json) + if [[ -z $AUTHOR_HANDLE ]]; then + echo "$0: ERROR: cannot obtain author_handle from this filename: $AUTHOR_HANDLE" 1>&2 + exit 16 + fi + + # obtain the author full name + # + FULL_NAME=$("$JVAL_WRAPPER" -b -w "$AUTHOR_HANDLE_JSON" '$..full_name') + if [[ -z $FULL_NAME ]]; then + echo "$0: ERROR: cannot full name from: $AUTHOR_HANDLE" 1>&2 + exit 1 + fi + + # obtain the sort word from the author full name + # + SORT_WORD=$(echo "$FULL_NAME" | sed -e 's/^.* //' | tr '[:upper:]' '[:lower:]' | tr -d -c 'a-z0-9') + if [[ -z $SORT_WORD ]]; then + echo "$0: ERROR: cannot sort word from author full name: $FULL_NAME" 1>&2 + exit 17 + fi + + # print location sort_word author_handle author name + # + echo "$LOCATION_CODE" "$SORT_WORD" "$AUTHOR_HANDLE" "$FULL_NAME" + + # report process every so often if debugging + # + ((++AUTHOR_COUNT)) + if [[ $V_FLAG -ge 1 && $((AUTHOR_COUNT % AUTHOR_COUNT_CYCLE)) -eq 0 ]]; then + echo "$0: debug[1]: cycle $AUTHOR_COUNT finished processing: $AUTHOR_HANDLE_JSON" 1>&2 + fi + +done | LC_ALL=C sort -d -u > "$TMP_LOC_WORD_HANDLE_NAME_SET" +status_codes=("${PIPESTATUS[@]}") +if [[ ${status_codes[*]} =~ [1-9] ]]; then + echo "$0: ERROR: done find | while .. done | LC_ALL=C sort -d -u failed," \ + "error code: ${status_codes[*]}" 1>&2 + exit 18 +fi +# +if [[ ! -s $TMP_LOC_WORD_HANDLE_NAME_SET ]]; then + echo "$0: ERROR: location sort_word author_handle author name set is empty: $TMP_LOC_WORD_HANDLE_NAME_SET" 1>&2 + exit 19 +fi +AUTHOR_COUNT=$(wc -l < "$TMP_LOC_WORD_HANDLE_NAME_SET") +if [[ $V_FLAG -ge 1 ]]; then + echo "$0: debug[1]: cycle $AUTHOR_COUNT" \ + "formed location sort_word author_handle author name set: $TMP_LOC_WORD_HANDLE_NAME_SET" 1>&2 +fi + + +# generate location count set +# +if [[ $V_FLAG -ge 1 ]]; then + echo "$0: debug[1]: about to obtain author counts for each location" 1>&2 +fi +awk '{print $1;}' "$TMP_LOC_WORD_HANDLE_NAME_SET" | LC_ALL=C uniq -c > "$TMP_LOC_COUNT" +status_codes=("${PIPESTATUS[@]}") +if [[ ${status_codes[*]} =~ [1-9] ]]; then + echo "$0: ERROR: done awk .. | LC_ALL=C uniq -c > $TMP_LOC_COUNT failed," \ + "error code: ${status_codes[*]}" 1>&2 + exit 20 +fi +# +if [[ ! -s $TMP_LOC_COUNT ]]; then + echo "$0: ERROR: location count set is empty: $TMP_LOC_COUNT" 1>&2 + exit 21 +fi + + +# form the temporary location.md file +# +if [[ $V_FLAG -ge 1 ]]; then + echo "$0: debug[1]: forming temporary location.md file: $TMP_LOC_MD" 1>&2 +fi +export PREV_LOCATION_CODE="." +export LINE=0 +while read -r LOCATION_CODE SORT_WORD AUTHOR_HANDLE FULL_NAME; do + + # firewall - paranoia # - export PREV_LOCATION_CODE="." - find "$AUTHOR_DIR" -mindepth 1 -maxdepth 1 -type f -name '*.json' -print0 | - xargs -0 grep '"location_code".*:' | - sed -e 's/^.*: "//' -e 's/",*$//' | - LC_ALL=C sort -u | - while read -r LOCATION_CODE; do + ((++LINE)) + if [[ -z $LOCATION_CODE || -z $SORT_WORD || -z $AUTHOR_HANDLE || -z $FULL_NAME ]]; then + echo "$0: ERROR: empty field found for line: $LINE of: $TMP_LOC_WORD_HANDLE_NAME_SET" 1>&2 + echo "$0: Warning: LOCATION_CODE=$LOCATION_CODE" 1>&2 + echo "$0: Warning: SORT_WORD=$SORT_WORD" 1>&2 + echo "$0: Warning: AUTHOR_HANDLE=$AUTHOR_HANDLE" 1>&2 + echo "$0: Warning: FULL_NAME=$FULL_NAME" 1>&2 + exit 22 + fi + + # case: the location has changed + # + if [[ $LOCATION_CODE != "$PREV_LOCATION_CODE" ]]; then + + # case: not the first location + # + # Separate this location from the previous location + # + if [[ $LINE -gt 1 ]]; then + echo + echo '
**PLEASE HELP** us identify proper locations for these authors!
' - echo 'See [FAQ 5.5](faq.html#fix_author) for how you can help!
' + echo 'See the' + echo 'FAQ on "[Help with ZZ](faq.html#zz_help)"' + echo 'for how you can help!
' + fi + + # determine author count for this location + # + AUTHOR_COUNT=$(grep -F " $LOCATION_CODE" "$TMP_LOC_COUNT" | awk '{print $1;}') + if [[ -z $AUTHOR_COUNT || $AUTHOR_COUNT -le 0 ]]; then + echo "$0: ERROR: cannot determine author count" \ + "for location ISO 3166 code: $LOCATION_CODE" 1>&2 + exit 23 fi # output author count for this location @@ -519,52 +700,24 @@ fi echo echo " Author count: **$AUTHOR_COUNT**" echo + fi - # process each author_handle JSON files with this location in sort_word order - # - # For SC2034, we only need to sort on the sort_word. We do not care about its value. - # - # SC2034 (warning): SORT_WORD appears unused. Verify use (or export if used externally). - # https://www.shellcheck.net/wiki/SC2034 - # shellcheck disable=SC2034 - find "$AUTHOR_DIR" -mindepth 1 -maxdepth 1 -type f -name '*.json' -print0 | - xargs -0 grep -l '"location_code".*:.*"'"$LOCATION_CODE"'"' | - while read -r AUTHOR_JSON; do - echo -n "$AUTHOR_JSON " - grep '"sort_word".*:' "$AUTHOR_JSON" | sed -e 's/^.*: "//' -e 's/",*$//' - done | - LC_ALL=C sort -k 2d | - while read -r AUTHOR_HANDLE_PATH SORT_WORD; do + # output the author from the location + # + echo " * $FULL_NAME" + PREV_LOCATION_CODE="$LOCATION_CODE" - # The author_handle foo_bin is found in author/foo_bar.json - # - AUTHOR_HANDLE=$(basename "$AUTHOR_HANDLE_PATH" .json) - export AUTHOR_HANDLE +done < "$TMP_LOC_WORD_HANDLE_NAME_SET" > "$TMP_LOC_MD" - # fetch full_name - # - FULL_NAME=$(grep '"full_name".*:' "$AUTHOR_HANDLE_PATH" | sed -e 's/^.*: "//' -e 's/",*$//') - export FULL_NAME - # output the author from the location - # - echo " * $FULL_NAME" - done - echo '' - echo - PREV_LOCATION_CODE="$LOCATION_CODE" - done +# complete temporary location markdown file +# +{ + echo echo '-
+
Author count: 2
-
+
Author count: 1
-
+
Author count: 1
+
Author count: 2
+
Author count: 1
-
+
Author count: 1
+
-
+
-
+
-
+
-
+
Author count: 1
+
Author count: 1
-
+
-
+
+
-
+
Author count: 1
-
+
Author count: 1
+
-
+
-
+
Author count: 1
-
+
+
Author count: 1
-
+
Author count: 1
-
+
Author count: 4
+
Author count: 2
-
+
-
+
Author count: 98
-
+
Author count: 1
-
+
-See FAQ 5.5 for how you can help! +See the +FAQ on “Help with ZZ” +for how you can help!
Author count: 5
-