diff --git a/Makefile b/Makefile index 830f79cd4..0a146d407 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ SORT_GITIGNORE= bin/sort.gitignore.sh FIND_MISSING_LINKS= bin/find-missing-links.sh CSV2ENTRY= bin/csv2entry.sh ENTRY2CSV= bin/entry2csv.sh - +UNICODE_CHK= bin/unicode-chk.sh ############# @@ -277,7 +277,7 @@ clobber: # Finally: The rules in this section are NOT needed if you # simple want to examine, run / test winning IOCCC entries. -.PHONY: help genpath genfilelist all_jfmt verify_entry_files gen_authors quick_authors \ +.PHONY: help genpath genfilelist all_jfmt verify_entry_files unicode_chk gen_authors quick_authors \ gen_location quick_location gen_years find_missing_links test entry_index gen_top_html \ thanks gen_other_html quick_other_html quick_entry_index \ gen_year_index quick_year_index quick_www www untar_entry_tarball untar_year_tarball \ @@ -313,6 +313,7 @@ help: @echo 'make genfilelist ;: generate YYYY level .filelist' @echo 'make verify_entry_files ;: check to be sure all files in all entries exist' @echo 'make sort_gitignore ;: sort .gitignore files according to rules in bin/sgi.sh' + @echo 'make unicode_chk ;: verify all .entry.json and author_handle.json files' @echo 'make gen_authors ;: generate the top level authors.html page' @echo 'make quick_authors ;: build authors.html if out of date with author JSON files' @echo 'make gen_location ;: generate the top level location.html page' @@ -448,6 +449,11 @@ verify_entry_files: ${ALL_RUN} ${CHK_ENTRY} ${ALL_RUN} -v 1 ${CHK_ENTRY} @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' +unicode_chk: ${UNICODE_CHK} + @echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-=' + ${UNICODE_CHK} -v 1 + @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' + # sort .gitignore files according to rules in bin/sgi.sh # sort_gitignore: ${ALL_RUN} ${SORT_GITIGNORE} @@ -688,6 +694,7 @@ quick_www: @echo '=-=-=-=-= IOCCC complete make genpath =-=-=-=-=' ${MAKE} genfilelist ${MAKE} sort_gitignore + ${MAKE} unicode_chk ${MAKE} verify_entry_files ${MAKE} quick_authors ${MAKE} quick_location @@ -716,6 +723,7 @@ www: @echo '=-=-=-=-= IOCCC complete make genpath =-=-=-=-=' ${MAKE} genfilelist ${MAKE} sort_gitignore + ${MAKE} unicode_chk ${MAKE} verify_entry_files ${MAKE} gen_authors ${MAKE} gen_location diff --git a/bin/unicode-chk.sh b/bin/unicode-chk.sh index b531ffdf4..c4840bf3c 100755 --- a/bin/unicode-chk.sh +++ b/bin/unicode-chk.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# unicode-chk.sh - verify all \\u[[:xdigit:]]{4} in entry and author JSON files +# unicode-chk.sh - verify all \\u[[:xdigit:]]{4} and JSON syntax in entry and author JSON files # # Pass entry and author JSON files thru the following sed script and # flag any \\u[[:xdigit:]]{4} that we might find as problems. @@ -122,7 +122,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.0 2024-09-23" +export VERSION="1.1 2024-09-24" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -143,6 +143,12 @@ export TAGLINE="unspecified" export REPO_TOP_URL="https://github.com/ioccc-src/temp-test-ioccc" # GitHub puts individual files under the "blob/master" sub-directory. export REPO_URL="$REPO_TOP_URL/blob/master" +JPARSE_TOOL=$(type -P jparse) +export JPARSE_TOOL +if [[ -z "$JPARSE_TOOL" ]]; then + echo "$0: FATAL: jparse tool is not installed or not in \$PATH" 1>&2 + exit 5 +fi export SITE_URL="https://ioccc-src.github.io/temp-test-ioccc" # export NOOP= @@ -356,6 +362,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: TAGLINE=$TAGLINE" 1>&2 echo "$0: debug[3]: REPO_TOP_URL=$REPO_TOP_URL" 1>&2 echo "$0: debug[3]: REPO_URL=$REPO_URL" 1>&2 + echo "$0: debug[3]: JPARSE_TOOL=$JPARSE_TOOL" 1>&2 echo "$0: debug[3]: SITE_URL=$SITE_URL" 1>&2 echo "$0: debug[3]: NOOP=$NOOP" 1>&2 echo "$0: debug[3]: DO_NOT_PROCESS=$DO_NOT_PROCESS" 1>&2 @@ -423,6 +430,19 @@ if [[ $# -gt 0 ]]; then exit 8 fi + # verify the JSON of the file + # + if [[ $V_FLAG -ge 1 ]]; then + echo "$0: debug[1]: about to run: $JPARSE_TOOL -q -- $JSON_FILE" 1>&2 + fi + "$JPARSE_TOOL" -q -- "$JSON_FILE" + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: Warning: invalid JSON: $JSON_FILE" 1>&2 + echo "$JSON_FILE" + continue + fi + # scan the author/author_handle.json file for unexpected \\u[[:xdigit:]]{4} strings # if [[ $V_FLAG -ge 1 ]]; then @@ -609,6 +629,19 @@ else continue fi + # verify the JSON of the .entry.json file + # + if [[ $V_FLAG -ge 3 ]]; then + echo "$0: debug[3]: about to run: $JPARSE_TOOL -q -- $YYYY_DIR/.entry.json" 1>&2 + fi + "$JPARSE_TOOL" -q -- "$YYYY_DIR/.entry.json" + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: Warning: invalid JSON: $YYYY_DIR/.entry.json" 1>&2 + echo "$YYYY_DIR/.entry.json" + continue + fi + # scan the .entry.json file for unexpected \\u[[:xdigit:]]{4} strings # if [[ -z $NOOP ]]; then @@ -646,12 +679,25 @@ else # check author/author_handle.json files # if [[ $V_FLAG -ge 1 ]]; then - echo "$0: debug[1]: starting to check author_handle.json files under: $AUTHOR_DIR" 1>&2 + echo "$0: debug[1]: starting to check author_handle.json files under: $TOPDIR/$AUTHOR_DIR" 1>&2 fi find "$AUTHOR_DIR" -mindepth 1 -maxdepth 1 -type f -name '*.json' 2>/dev/null | LC_ALL=C sort -d -u > "$TMP_AUTHOR_HANDLE_INVENTORY" while read -r AUTHOR_HANDLE_JSON; do + # verify the JSON of the .entry.json file + # + if [[ $V_FLAG -ge 3 ]]; then + echo "$0: debug[3]: about to run: $JPARSE_TOOL -q -- $AUTHOR_HANDLE_JSON" 1>&2 + fi + "$JPARSE_TOOL" -q -- "$AUTHOR_HANDLE_JSON" + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: Warning: invalid JSON: $AUTHOR_HANDLE_JSON" 1>&2 + echo "$AUTHOR_HANDLE_JSON" + continue + fi + # scan the author/author_handle.json file for unexpected \\u[[:xdigit:]]{4} strings # if [[ $V_FLAG -ge 5 ]]; then @@ -669,10 +715,9 @@ fi # report any problems # if [[ -s $TMP_JSON_PROBLEM ]]; then - echo "$0: ERROR: found unexpected \\uHexHexHexHex strings in JSON files" 1>&2 - echo "$0: debug[5]: list of JSON files with unexpected \\uHexHexHexHex strings, starts below" 1>&2 + echo "$0: debug[5]: list of JSON file problems, starts below" 1>&2 cat "$TMP_JSON_PROBLEM" 1>&2 - echo "$0: debug[5]: list of JSON files with unexpected \\uHexHexHexHex strings, ends above" 1>&2 + echo "$0: debug[5]: list of JSON file problems, ends above" 1>&2 if [[ $EXIT_CODE -eq 0 ]]; then EXIT_CODE="1" # exit 1 echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2