diff --git a/Makefile b/Makefile index 0a146d407..5ee177e80 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,6 @@ SORT_GITIGNORE= bin/sort.gitignore.sh FIND_MISSING_LINKS= bin/find-missing-links.sh CSV2ENTRY= bin/csv2entry.sh ENTRY2CSV= bin/entry2csv.sh -UNICODE_CHK= bin/unicode-chk.sh ############# @@ -277,7 +276,7 @@ clobber: # Finally: The rules in this section are NOT needed if you # simple want to examine, run / test winning IOCCC entries. -.PHONY: help genpath genfilelist all_jfmt verify_entry_files unicode_chk gen_authors quick_authors \ +.PHONY: help genpath genfilelist all_jfmt verify_entry_files gen_authors quick_authors \ gen_location quick_location gen_years find_missing_links test entry_index gen_top_html \ thanks gen_other_html quick_other_html quick_entry_index \ gen_year_index quick_year_index quick_www www untar_entry_tarball untar_year_tarball \ @@ -313,7 +312,6 @@ help: @echo 'make genfilelist ;: generate YYYY level .filelist' @echo 'make verify_entry_files ;: check to be sure all files in all entries exist' @echo 'make sort_gitignore ;: sort .gitignore files according to rules in bin/sgi.sh' - @echo 'make unicode_chk ;: verify all .entry.json and author_handle.json files' @echo 'make gen_authors ;: generate the top level authors.html page' @echo 'make quick_authors ;: build authors.html if out of date with author JSON files' @echo 'make gen_location ;: generate the top level location.html page' @@ -449,11 +447,6 @@ verify_entry_files: ${ALL_RUN} ${CHK_ENTRY} ${ALL_RUN} -v 1 ${CHK_ENTRY} @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' -unicode_chk: ${UNICODE_CHK} - @echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-=' - ${UNICODE_CHK} -v 1 - @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' - # sort .gitignore files according to rules in bin/sgi.sh # sort_gitignore: ${ALL_RUN} ${SORT_GITIGNORE} @@ -464,14 +457,14 @@ sort_gitignore: ${ALL_RUN} ${SORT_GITIGNORE} # generate the top level authors.html page using the # ${GEN_AUTHORS} tool (bin/gen-authors.sh). # -gen_authors: ${GEN_AUTHORS} authors.html +gen_authors: ${GEN_AUTHORS} @echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-=' ${GEN_AUTHORS} -v 1 @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' # build authors.html if it is out of date from any author/author_handle.json file # -quick_authors: ${GEN_AUTHORS} authors.html +quick_authors: ${GEN_AUTHORS} @echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-=' ${GEN_AUTHORS} -v 1 -Q @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' @@ -479,7 +472,7 @@ quick_authors: ${GEN_AUTHORS} authors.html # generate the top level location.html page using the # ${GEN_LOCATION} tool (bin/gen-location.sh). # -gen_location: ${GEN_LOCATION} location.html +gen_location: ${GEN_LOCATION} @echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-=' ${GEN_LOCATION} -v 1 @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' @@ -489,7 +482,7 @@ gen_location: ${GEN_LOCATION} location.html # generate the top level location.html page using the # ${GEN_LOCATION} tool (bin/gen-location.sh). # -quick_location: ${GEN_LOCATION} location.html +quick_location: ${GEN_LOCATION} @echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-=' ${GEN_LOCATION} -v 1 -Q @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' @@ -497,7 +490,7 @@ quick_location: ${GEN_LOCATION} location.html # generate the top level years.html page using the ${GEN_YEARS} # tool (bin/gen-years.sh). # -gen_years: ${GEN_YEARS} years.html +gen_years: ${GEN_YEARS} @echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-=' ${GEN_YEARS} -v 1 @echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-=' @@ -694,7 +687,6 @@ quick_www: @echo '=-=-=-=-= IOCCC complete make genpath =-=-=-=-=' ${MAKE} genfilelist ${MAKE} sort_gitignore - ${MAKE} unicode_chk ${MAKE} verify_entry_files ${MAKE} quick_authors ${MAKE} quick_location @@ -723,7 +715,6 @@ www: @echo '=-=-=-=-= IOCCC complete make genpath =-=-=-=-=' ${MAKE} genfilelist ${MAKE} sort_gitignore - ${MAKE} unicode_chk ${MAKE} verify_entry_files ${MAKE} gen_authors ${MAKE} gen_location diff --git a/bin/README.md b/bin/README.md index 243bd72ca..aed88199a 100644 --- a/bin/README.md +++ b/bin/README.md @@ -599,7 +599,7 @@ The [jfmt-wrapper.sh](%%REPO_URL%%/bin/jfmt-wrapper.sh) tool is a wrapper tool for `jfmt(1)`, a tool that will format a JSON # file into a canonical style. -NOTE: As of 2024 July 28 the `jfmt(1)` tool has not been written, +NOTE: As of 2024 Oct 08 the `jfmt(1)` tool has not been written, so [jfmt-wrapper.sh](%%REPO_URL%%/bin/jfmt-wrapper.sh) uses the `JSONPath.sh(1)` tool from the recently forked and modified [JSONPath.sh tool](https://github.com/lcn2/JSONPath.sh). @@ -620,7 +620,7 @@ tool for `jprint(1)`, a tool that will format JSON in a canonical style to stdout. NOTE: `jprint(1)` is a planned tool based on `jfmt(1)`. -As of 2024 July 28 that tool has not been written, so +As of 2024 Oct 08 that tool has not been written, so [jprint-wrapper.sh](%%REPO_URL%%/bin/jprint-wrapper.sh) uses the `JSONPath.sh(1)` tool from the recently forked and modified [JSONPath.sh tool](https://github.com/lcn2/JSONPath.sh). @@ -638,6 +638,55 @@ Or use in a pipe: ``` +
+### [jval-wrapper.sh](%%REPO_URL%%/bin/jval-wrapper.sh) +
+ +The [jval-wrapper.sh](%%REPO_URL%%/bin/jval-wrapper.sh) tool is a wrapper +tool for `jval(1)`, a tool that will output a JSON value using the +XPath for JSON mechanism. For information in XPath for JSON see: + +- [https://goessner.net/articles/JsonPath/](https://goessner.net/articles/JsonPath/) +- [https://jsonpath.com](https://jsonpath.com) +- [https://github.com/lcn2/JSONPath.sh?tab=readme-ov-file#jsonpath-patterns-and-extensions](https://github.com/lcn2/JSONPath.sh?tab=readme-ov-file#jsonpath-patterns-and-extensions) + +NOTE: As of 2024 Oct 08 the `jval(1)` tool has not been written, +so [jval-wrapper.sh](%%REPO_URL%%/bin/jval-wrapper.sh) uses the +`jsp(1)` tool from [//github.com/kjozsa/jsp](//github.com/kjozsa/jsp) +or if not found, the `JSONPath.sh(1)` tool from the recently forked and modified +[JSONPath.sh tool](https://github.com/lcn2/JSONPath.sh). + +Usage: + +``` + bin/jval-wrapper.sh [-b] [-q] [-T] file.json [pattern] + + -b print only values (def: print both JSON member and JSON value) + -q remove enclosing double quotes (def: keep any enclosing double quotes) + -T trim all newlines (def: do not) + + file.json JSON file to process, - ==> read stdin + pattern Xpath for JSON query pattern +``` + +For example: + + +``` + bin/jval-wrapper.sh 1984/mullender/.entry.json '$.award' + bin/jval-wrapper.sh 1984/mullender/.entry.json '$..author_handle' + + bin/jval-wrapper.sh 1984/mullender/.entry.json '$.manifest[2]' + bin/jval-wrapper.sh -q 1984/mullender/.entry.json '$.manifest[2]' + + bin/jval-wrapper.sh -b 1984/mullender/.entry.json '$.manifest[2].entry_text' + bin/jval-wrapper.sh -b -q 1984/mullender/.entry.json '$.manifest[2].inventory_order' + + bin/jval-wrapper.sh author/Anton_Algmyr.json '$.full_name' + bin/jval-wrapper.sh -q -T author/Anton_Algmyr.json '$.full_name' | jstrdecode - +``` + +
### [md2html.cfg](%%REPO_URL%%/bin/md2html.cfg)
diff --git a/bin/csv2entry.sh b/bin/csv2entry.sh index c7aecba50..de4b0fca5 100755 --- a/bin/csv2entry.sh +++ b/bin/csv2entry.sh @@ -143,7 +143,7 @@ shopt -s lastpipe # explicitly run the last command of the pipe line in the curr # set variables referenced in the usage message # -export VERSION="1.4 2024-09-27" +export VERSION="1.5 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -514,36 +514,19 @@ if [[ ! -x $JVAL_WRAPPER ]]; then fi -# verify that the bin/unicode-chk.sh tool is executable -# -export UNICODE_CHK="$BIN_DIR/unicode-chk.sh" -if [[ ! -e $UNICODE_CHK ]]; then - echo "$0: ERROR: bin/unicode-chk.sh does not exist: $UNICODE_CHK" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_CHK ]]; then - echo "$0: ERROR: bin/unicode-chk.sh is not a regular file: $UNICODE_CHK" 1>&2 - exit 5 -fi -if [[ ! -x $UNICODE_CHK ]]; then - echo "$0: ERROR: bin/unicode-chk.sh is not an executable file: $UNICODE_CHK" 1>&2 - exit 5 -fi - - # verify that the bin/combine_author_handle.sh tool is executable # export COMBINE_AUTHOR="$BIN_DIR/combine_author_handle.sh" if [[ ! -e $COMBINE_AUTHOR ]]; then - echo "$0: ERROR: bin/unicode-chk.sh does not exist: $COMBINE_AUTHOR" 1>&2 + echo "$0: ERROR: combine_author_handle.sh does not exist: $COMBINE_AUTHOR" 1>&2 exit 5 fi if [[ ! -f $COMBINE_AUTHOR ]]; then - echo "$0: ERROR: bin/unicode-chk.sh is not a regular file: $COMBINE_AUTHOR" 1>&2 + echo "$0: ERROR: combine_author_handle.sh is not a regular file: $COMBINE_AUTHOR" 1>&2 exit 5 fi if [[ ! -x $COMBINE_AUTHOR ]]; then - echo "$0: ERROR: bin/unicode-chk.sh is not an executable file: $COMBINE_AUTHOR" 1>&2 + echo "$0: ERROR: combine_author_handle.sh is not an executable file: $COMBINE_AUTHOR" 1>&2 exit 5 fi @@ -574,7 +557,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: BIN_PATH=$BIN_DIR" 1>&2 echo "$0: debug[3]: BIN_DIR=$BIN_DIR" 1>&2 echo "$0: debug[3]: JVAL_WRAPPER=$JVAL_WRAPPER" 1>&2 - echo "$0: debug[3]: UNICODE_CHK=$UNICODE_CHK" 1>&2 echo "$0: debug[3]: COMBINE_AUTHOR=$COMBINE_AUTHOR" 1>&2 echo "$0: debug[3]: DOT_ENTRY_JSON_BASENAME=$DOT_ENTRY_JSON_BASENAME" 1>&2 echo "$0: debug[3]: ENTRY_JSON_FORMAT_VERSION=$ENTRY_JSON_FORMAT_VERSION" 1>&2 @@ -1566,18 +1548,6 @@ sed -e '/^#/d' -e 's/,/ /g' "$MANIFEST_CSV" | printf "}\n" } >> "$TMP_ENTRY_JSON" - # check the JSON file - # - if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to check: $PREV_ENTRY_JSON" 1>&2 - fi - "$UNICODE_CHK" "$TMP_ENTRY_JSON" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: ERROR: JSON file failed check: $TMP_ENTRY_JSON" 1>&2 - exit 1 - fi - # update entry .entry.json if different # if [[ -z $NOOP ]]; then diff --git a/bin/entry2csv.sh b/bin/entry2csv.sh index 3a7db7e8f..94e391e78 100755 --- a/bin/entry2csv.sh +++ b/bin/entry2csv.sh @@ -118,7 +118,7 @@ shopt -s lastpipe # explicitly run the last command of the pipe line in the curr # set variables referenced in the usage message # -export VERSION="1.4 2024-09-27" +export VERSION="1.5 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -135,6 +135,17 @@ if [[ $status -eq 0 ]]; then fi export TOPDIR # +JSTRDECODE=$(type -P jstrdecode) +export JSTRDECODE +if [[ -z $JSTRDECODE ]]; then + echo "$0: FATAL: jstrdecode is not installed or not in \$PATH" 1>&2 + echo "$0: notice: to install jstrdecode:" 1>&2 + echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2 + echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2 + echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2 + exit 5 +fi +# export NOOP= export DO_NOT_PROCESS= export EXIT_CODE="0" @@ -186,12 +197,12 @@ function output_award # PATTERN='$..award' if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN'" 1>&2 + echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE -" 1>&2 fi - "$JVAL_WRAPPER" -b -q "$ENTRY_JSON_PATH" "$PATTERN" | sed -f "$UNICODE_FIX_SED" + "$JVAL_WRAPPER" -b -q -T "$ENTRY_JSON_PATH" "$PATTERN" | "$JSTRDECODE" - status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] ]]; then - echo "$0: ERROR: in output_award: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN' failed," \ + echo "$0: ERROR: in output_award: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE - failed," \ "error codes: ${status_codes[*]}" 1>&2 return 5 fi @@ -346,14 +357,14 @@ function output_abstract # PATTERN='$..abstract' if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN'" 1>&2 + echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE -" 1>&2 fi export ABSTRACT - ABSTRACT=$("$JVAL_WRAPPER" -b -q "$ENTRY_JSON_PATH" "$PATTERN") - status="$?" - if [[ $status -ne 0 || -z $ABSTRACT ]]; then - echo "$0: ERROR: in output_abstract: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN' failed," \ - "error code: $status" 1>&2 + ABSTRACT=$("$JVAL_WRAPPER" -b -q -T "$ENTRY_JSON_PATH" "$PATTERN" | "$JSTRDECODE" -) + status_codes=("${PIPESTATUS[@]}") + if [[ ${status_codes[*]} =~ [1-9] || -z $ABSTRACT ]]; then + echo "$0: ERROR: in output_abstract: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE - failed," \ + "error codes: ${status_codes[*]}" 1>&2 return 5 fi if [[ ${#ABSTRACT} -ge 65 ]]; then @@ -364,13 +375,7 @@ function output_abstract echo "$0: ERROR: in output_abstract: abstract contains ; or & or , in $ENTRY_JSON_PATH" 1>&2 return 7 fi - sed -f "$UNICODE_FIX_SED" -e 's/[;&,]//g' <<< "$ABSTRACT" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: ERROR: sed -f $UNICODE_FIX_SED -e ... failed," \ - "error code: $status" 1>&2 - return 8 - fi + echo "${ABSTRACT//[;&,]/}" return 0 } @@ -623,23 +628,6 @@ if [[ ! -r $MANIFEST_ENTRY_CSV_ENTRY_AWK ]]; then fi -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # print running info if verbose # # If -v 3 or higher, print exported variables in order that they were exported. @@ -650,6 +638,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2 echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2 echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2 + echo "$0: debug[3]: JSTRDECODE=$JSTRDECODE" 1>&2 echo "$0: debug[3]: NOOP=$NOOP" 1>&2 echo "$0: debug[3]: DO_NOT_PROCESS=$DO_NOT_PROCESS" 1>&2 echo "$0: debug[3]: EXIT_CODE=$EXIT_CODE" 1>&2 @@ -665,7 +654,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: TOP_FILE=$TOP_FILE" 1>&2 echo "$0: debug[3]: BIN_PATH=$BIN_DIR" 1>&2 echo "$0: debug[3]: BIN_DIR=$BIN_DIR" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 echo "$0: debug[3]: JVAL_WRAPPER=$JVAL_WRAPPER" 1>&2 echo "$0: debug[3]: MANIFEST_ENTRY_CSV_ENTRY_AWK=$MANIFEST_ENTRY_CSV_ENTRY_AWK" 1>&2 fi diff --git a/bin/gen-authors.sh b/bin/gen-authors.sh index ca9ab6d4b..337344492 100755 --- a/bin/gen-authors.sh +++ b/bin/gen-authors.sh @@ -88,7 +88,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.13 2024-09-23" +export VERSION="1.14 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -104,6 +104,18 @@ if [[ $status -eq 0 ]]; then TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) fi export TOPDIR +# +JSTRDECODE=$(type -P jstrdecode) +export JSTRDECODE +if [[ -z $JSTRDECODE ]]; then + echo "$0: FATAL: jstrdecode is not installed or not in \$PATH" 1>&2 + echo "$0: notice: to install jstrdecode:" 1>&2 + echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2 + echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2 + echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2 + exit 5 +fi +# export DOCROOT_SLASH="./" export TAGLINE="bin/$NAME" export MD2HTML_SH="bin/md2html.sh" @@ -240,13 +252,13 @@ function output_award # PATTERN='$..award' if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN'" 1>&2 + echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE -" 1>&2 fi - "$JVAL_WRAPPER" -b -q "$ENTRY_JSON_PATH" "$PATTERN" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: ERROR: in output_award: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN' failed," \ - "error code: $status" 1>&2 + "$JVAL_WRAPPER" -b -q -T "$ENTRY_JSON_PATH" "$PATTERN" | "$JSTRDECODE" - + status_codes=("${PIPESTATUS[@]}") + if [[ ${status_codes[*]} =~ [1-9] ]]; then + echo "$0: ERROR: in output_award: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE - failed," \ + "error codes: ${status_codes[*]}" 1>&2 return 5 fi return 0 @@ -489,32 +501,15 @@ export AUTHORS_HTML="authors.html" # export COMBINE_AUTHOR="$BIN_DIR/combine_author_handle.sh" if [[ ! -e $COMBINE_AUTHOR ]]; then - echo "$0: ERROR: bin/unicode-chk.sh does not exist: $COMBINE_AUTHOR" 1>&2 + echo "$0: ERROR: combine_author_handle.sh does not exist: $COMBINE_AUTHOR" 1>&2 exit 5 fi if [[ ! -f $COMBINE_AUTHOR ]]; then - echo "$0: ERROR: bin/unicode-chk.sh is not a regular file: $COMBINE_AUTHOR" 1>&2 + echo "$0: ERROR: combine_author_handle.sh is not a regular file: $COMBINE_AUTHOR" 1>&2 exit 5 fi if [[ ! -x $COMBINE_AUTHOR ]]; then - echo "$0: ERROR: bin/unicode-chk.sh is not an executable file: $COMBINE_AUTHOR" 1>&2 - exit 5 -fi - - -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 + echo "$0: ERROR: combine_author_handle.sh is not an executable file: $COMBINE_AUTHOR" 1>&2 exit 5 fi @@ -529,6 +524,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2 echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2 echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2 + echo "$0: debug[3]: JSTRDECODE=$JSTRDECODE" 1>&2 echo "$0: debug[3]: DOCROOT_SLASH=$DOCROOT_SLASH" 1>&2 echo "$0: debug[3]: TAGLINE=$TAGLINE" 1>&2 echo "$0: debug[3]: MD2HTML_SH=$MD2HTML_SH" 1>&2 @@ -556,7 +552,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: LOCATION_TOOL=$LOCATION_TOOL" 1>&2 echo "$0: debug[3]: AUTHORS_HTML=$AUTHORS_HTML" 1>&2 echo "$0: debug[3]: COMBINE_AUTHOR=$COMBINE_AUTHOR" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 fi @@ -738,7 +733,6 @@ EOF fi export AUTHOR_DATA AUTHOR_DATA=$("$JVAL_WRAPPER" -q -- "$filename" | - sed -f "$UNICODE_FIX_SED" | LC_ALL=C sort -d) status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] ]]; then @@ -767,7 +761,7 @@ EOF # case "$NAME" in full_name) - FULL_NAME="$VALUE" + FULL_NAME=$("$JSTRDECODE" "$VALUE") ;; location_code) LOCATION_CODE="$VALUE" @@ -788,7 +782,7 @@ EOF AUTHOR_GITHUB="$VALUE" ;; affiliation) - AUTHOR_AFFILIATION="$VALUE" + AUTHOR_AFFILIATION=$("$JSTRDECODE" "$VALUE") ;; author_handle) AUTHOR_HANDLE="$VALUE" @@ -955,7 +949,7 @@ EOF # collect the award for this entry id # - AWARD=$(output_award "$ENTRY_JSON" | sed -f "$UNICODE_FIX_SED") + AWARD=$(output_award "$ENTRY_JSON") export AWARD if [[ -z $AWARD ]]; then echo "$0: ERROR: award not found in .entry.json: $ENTRY_JSON" 1>&2 diff --git a/bin/gen-location.sh b/bin/gen-location.sh index ce60b4b36..65dd5369b 100755 --- a/bin/gen-location.sh +++ b/bin/gen-location.sh @@ -85,7 +85,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.7 2024-09-23" +export VERSION="1.8 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -101,6 +101,18 @@ if [[ $status -eq 0 ]]; then TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) fi export TOPDIR +# +JSTRDECODE=$(type -P jstrdecode) +export JSTRDECODE +if [[ -z $JSTRDECODE ]]; then + echo "$0: FATAL: jstrdecode is not installed or not in \$PATH" 1>&2 + echo "$0: notice: to install jstrdecode:" 1>&2 + echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2 + echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2 + echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2 + exit 5 +fi +# export DOCROOT_SLASH="./" export TAGLINE="bin/$NAME" export PANDOC_WRAPPER="bin/pandoc-wrapper.sh" @@ -397,23 +409,6 @@ fi export LOCATION_HTML="location.html" -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # print running info if verbose # # If -v 3 or higher, print exported variables in order that they were exported. @@ -424,6 +419,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2 echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2 echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2 + echo "$0: debug[3]: JSTRDECODE=$JSTRDECODE" 1>&2 echo "$0: debug[3]: DOCROOT_SLASH=$DOCROOT_SLASH" 1>&2 echo "$0: debug[3]: TAGLINE=$TAGLINE" 1>&2 echo "$0: debug[3]: MD2HTML_SH=$MD2HTML_SH" 1>&2 @@ -447,7 +443,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: JVAL_WRAPPER=$JVAL_WRAPPER" 1>&2 echo "$0: debug[3]: LOCATION_TOOL=$LOCATION_TOOL" 1>&2 echo "$0: debug[3]: LOCATION_HTML=$LOCATION_HTML" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 fi # -N stops early before any processing is performed @@ -574,7 +569,7 @@ find "$AUTHOR_DIR" -mindepth 1 -maxdepth 1 -type f -name '*.json' -print | # obtain the author full name # PATTERN='$..full_name' - FULL_NAME=$("$JVAL_WRAPPER" -b -q "$AUTHOR_HANDLE_JSON" "$PATTERN" | sed -f "$UNICODE_FIX_SED") + FULL_NAME=$("$JVAL_WRAPPER" -b -q -T "$AUTHOR_HANDLE_JSON" "$PATTERN" | "$JSTRDECODE" -) status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] || -z $FULL_NAME ]]; then echo "$0: ERROR: cannot full name from: $AUTHOR_HANDLE" 1>&2 diff --git a/bin/gen-years.sh b/bin/gen-years.sh index c36345806..89e510d7f 100755 --- a/bin/gen-years.sh +++ b/bin/gen-years.sh @@ -88,7 +88,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.5 2024-09-23" +export VERSION="1.6 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -104,6 +104,18 @@ if [[ $status -eq 0 ]]; then TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) fi export TOPDIR +# +JSTRDECODE=$(type -P jstrdecode) +export JSTRDECODE +if [[ -z $JSTRDECODE ]]; then + echo "$0: FATAL: jstrdecode is not installed or not in \$PATH" 1>&2 + echo "$0: notice: to install jstrdecode:" 1>&2 + echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2 + echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2 + echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2 + exit 5 +fi +# TAC_TOOL=$(type -P tac) export TAC_TOOL if [[ ! -x $TAC_TOOL ]]; then @@ -214,9 +226,9 @@ function output_award # obtain the award string # PATTERN='$..award' - AWARD_STRING=$("$JVAL_WRAPPER" -b -q "$ENTRY_JSON_PATH" "$PATTERN") - status="$?" - if [[ $status -ne 0 || -z $AWARD_STRING ]]; then + AWARD_STRING=$("$JVAL_WRAPPER" -b -q -T "$ENTRY_JSON_PATH" "$PATTERN" | "$JSTRDECODE" -) + status_codes=("${PIPESTATUS[@]}") + if [[ ${status_codes[*]} =~ [1-9] || -z $AWARD_STRING ]]; then echo "$0: ERROR: in output_award: no award found in .entry.json file: $ENTRY_JSON_PATH" 1>&2 return 5 fi @@ -552,23 +564,6 @@ fi export YEARS_HTML="years.html" -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # print running info if verbose # # If -v 3 or higher, print exported variables in order that they were exported. @@ -579,6 +574,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2 echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2 echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2 + echo "$0: debug[3]: JSTRDECODE=$JSTRDECODE" 1>&2 echo "$0: debug[3]: TAC_TOOL=$TAC_TOOL" 1>&2 echo "$0: debug[3]: FMT_TOOL=$FMT_TOOL" 1>&2 echo "$0: debug[3]: DOCROOT_SLASH=$DOCROOT_SLASH" 1>&2 @@ -601,7 +597,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: JVAL_WRAPPER=$JVAL_WRAPPER" 1>&2 echo "$0: debug[3]: TOP_FILE=$TOP_FILE" 1>&2 echo "$0: debug[3]: YEARS_HTML=$YEARS_HTML" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 fi @@ -868,9 +863,9 @@ for YYYY in $("$TAC_TOOL" "$TOP_FILE"); do # determine the award for this entry # - AWARD=$(output_award "$ENTRY_JSON" | sed -f "$UNICODE_FIX_SED") - status_codes=("${PIPESTATUS[@]}") - if [[ ${status_codes[*]} =~ [1-9] || -z $AWARD ]]; then + AWARD=$(output_award "$ENTRY_JSON") + status="$?" + if [[ $status -ne 0 || -z $AWARD ]]; then echo "$0: ERROR: cannot find award in .entry.json: $ENTRY_JSON" 1>&2 EXIT_CODE="7" # exit 7 echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 diff --git a/bin/index.html b/bin/index.html index ff6bf99ed..03249d077 100644 --- a/bin/index.html +++ b/bin/index.html @@ -735,7 +735,7 @@ do that, that is not a problem.

The jfmt-wrapper.sh tool is a wrapper tool for jfmt(1), a tool that will format a JSON # file into a canonical style.

-

NOTE: As of 2024 July 28 the jfmt(1) tool has not been written, +

NOTE: As of 2024 Oct 08 the jfmt(1) tool has not been written, so jfmt-wrapper.sh uses the JSONPath.sh(1) tool from the recently forked and modified JSONPath.sh tool.

@@ -748,7 +748,7 @@ so jprint-wrapper.sh uses the JSONPath.sh(1) tool from the recently forked and modified JSONPath.sh tool.

@@ -756,6 +756,43 @@ the JSONPath.sh(1) tool from the recently forked and modified
    bin/jprint-wrapper.sh .entry.json

Or use in a pipe:

    cat .info.json | bin/jprint-wrapper.sh
+
+

jval-wrapper.sh

+
+

The jval-wrapper.sh tool is a wrapper +tool for jval(1), a tool that will output a JSON value using the +XPath for JSON mechanism. For information in XPath for JSON see:

+ +

NOTE: As of 2024 Oct 08 the jval(1) tool has not been written, +so jval-wrapper.sh uses the +jsp(1) tool from //github.com/kjozsa/jsp +or if not found, the JSONPath.sh(1) tool from the recently forked and modified +JSONPath.sh tool.

+

Usage:

+
    bin/jval-wrapper.sh [-b] [-q] [-T] file.json [pattern]
+
+    -b        print only values (def: print both JSON member and JSON value)
+    -q        remove enclosing double quotes (def: keep any enclosing double quotes)
+    -T        trim all newlines (def: do not)
+
+    file.json JSON file to process, - ==> read stdin
+    pattern   Xpath for JSON query pattern
+

For example:

+
    bin/jval-wrapper.sh 1984/mullender/.entry.json '$.award'
+    bin/jval-wrapper.sh 1984/mullender/.entry.json '$..author_handle'
+
+    bin/jval-wrapper.sh 1984/mullender/.entry.json '$.manifest[2]'
+    bin/jval-wrapper.sh -q 1984/mullender/.entry.json '$.manifest[2]'
+
+    bin/jval-wrapper.sh -b 1984/mullender/.entry.json '$.manifest[2].entry_text'
+    bin/jval-wrapper.sh -b -q 1984/mullender/.entry.json '$.manifest[2].inventory_order'
+
+    bin/jval-wrapper.sh author/Anton_Algmyr.json '$.full_name'
+    bin/jval-wrapper.sh -q -T author/Anton_Algmyr.json '$.full_name' | jstrdecode -

md2html.cfg

diff --git a/bin/jfmt-wrapper.sh b/bin/jfmt-wrapper.sh index 9dd44a71b..ff5b52038 100755 --- a/bin/jfmt-wrapper.sh +++ b/bin/jfmt-wrapper.sh @@ -110,7 +110,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.2 2024-09-23" +export VERSION="1.3 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -326,23 +326,6 @@ fi export BIN_DIR="bin" -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # validate the XPath for JSON tool # case "$XPATHJSON_USE" in @@ -446,7 +429,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: CD_FAILED=$CD_FAILED" 1>&2 echo "$0: debug[3]: BIN_PATH=$BIN_PATH" 1>&2 echo "$0: debug[3]: BIN_DIR=$BIN_DIR" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 fi @@ -495,7 +477,7 @@ case "$XPATHJSON_USE" in echo "$0: debug[3]: about to: $JSP_TOOL --indent 4 --format --no-color < $JSON_FILE | sed ..." 1>&2 fi "$JSP_TOOL" --indent 4 --format --no-color < "$JSON_FILE" | - sed -f "$UNICODE_FIX_SED" -e 's/\(\S\): /\1 : /' > "$TMP_JSON_FILE" + sed -e 's/\(\S\): /\1 : /' > "$TMP_JSON_FILE" status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] ]]; then echo "$0: ERROR: $JSP_TOOL --indent 4 --format --no-color < $JSON_FILE | sed ... failed", \ diff --git a/bin/jprint-wrapper.sh b/bin/jprint-wrapper.sh index c3d2514de..b4a5b42de 100755 --- a/bin/jprint-wrapper.sh +++ b/bin/jprint-wrapper.sh @@ -110,7 +110,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.3 2024-09-23" +export VERSION="1.4 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -126,6 +126,7 @@ if [[ $status -eq 0 ]]; then TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) fi export TOPDIR +# export REPO_TOP_URL="https://github.com/ioccc-src/temp-test-ioccc" # GitHub puts individual files under the "blob/master" sub-directory. export REPO_URL="$REPO_TOP_URL/blob/master" @@ -326,23 +327,6 @@ fi export BIN_DIR="bin" -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # validate the XPath for JSON tool # case "$XPATHJSON_USE" in @@ -446,7 +430,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: CD_FAILED=$CD_FAILED" 1>&2 echo "$0: debug[3]: BIN_PATH=$BIN_PATH" 1>&2 echo "$0: debug[3]: BIN_DIR=$BIN_DIR" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 fi @@ -485,7 +468,7 @@ case "$XPATHJSON_USE" in echo "$0: debug[3]: about to: $JSP_TOOL --indent 4 --format --no-color | sed ..." 1>&2 fi "$JSP_TOOL" --indent 4 --format --no-color | - sed -f "$UNICODE_FIX_SED" -e 's/\(\S\): /\1 : /' + sed -e 's/\(\S\): /\1 : /' status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] ]]; then echo "$0: ERROR: $JSP_TOOL --indent 4 --format --no-color | sed ... failed", \ @@ -500,7 +483,7 @@ case "$XPATHJSON_USE" in echo "$0: debug[3]: about to: $JSP_TOOL --indent 4 --format --no-color < $JSON_FILE | sed ..." 1>&2 fi "$JSP_TOOL" --indent 4 --format --no-color < "$JSON_FILE" | - sed -f "$UNICODE_FIX_SED" -e 's/\(\S\): /\1 : /' + sed -e 's/\(\S\): /\1 : /' status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] ]]; then echo "$0: ERROR: $JSP_TOOL --indent 4 --format --no-color < $JSON_FILE | sed ... failed", \ diff --git a/bin/jval-wrapper.sh b/bin/jval-wrapper.sh index d3e8c367a..75deba9f2 100755 --- a/bin/jval-wrapper.sh +++ b/bin/jval-wrapper.sh @@ -1,25 +1,13 @@ #!/usr/bin/env bash # -# jval-wrapper.sh - print JSON values to stdout +# jval-wrapper.sh - print JSON values to stdout using the XPath for JSON model # # We form a wrapper for the `jval(1)`, a tool that allow the printing of JSON values. # -# We propose that the grammar used by JSONPath.sh(1): -# -# https://github.com/lcn2/JSONPath.sh -# -# becomes the model for how `jval(1)` prints JSON values. Consider, for, example: -# -# JSONPath.sh -w -f 1984/mullender/.entry.json '$author_set..author_handle' -# -# and in particular: -# -# JSONPath.sh -w -b -f 1984/mullender/.entry.json '$author_set..author_handle' -# -# as a model of printing the `author_handle` values from the JSON `author_set` array. -# -# See the following URL for more information. +# See the following on XPath for JSON model: # +# https://goessner.net/articles/JsonPath/ +# https://jsonpath.com # https://github.com/lcn2/JSONPath.sh?tab=readme-ov-file#jsonpath-patterns-and-extensions # # Copyright (c) 2024 by Landon Curt Noll. All Rights Reserved. @@ -106,7 +94,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.1 2024-09-23" +export VERSION="1.2 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -122,6 +110,7 @@ if [[ $status -eq 0 ]]; then TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) fi export TOPDIR +# export REPO_TOP_URL="https://github.com/ioccc-src/temp-test-ioccc" # GitHub puts individual files under the "blob/master" sub-directory. export REPO_URL="$REPO_TOP_URL/blob/master" @@ -150,11 +139,12 @@ export PATTERN="" export PRINT_ONLY_VALUES="" export REMOVE_QUOTES="" export QUICK_CHECK="" +export TRIM_NEWLINE="" # set usage message # -export USAGE="usage: $0 [-h] [-v level] [-V] [-n] [-N] [-b] [-q] [-t tool] [-Q] file.json [pattern] +export USAGE="usage: $0 [-h] [-v level] [-V] [-n] [-N] [-b] [-q] [-T] [-t tool] [-Q] file.json [pattern] -h print help message and exit -v level set verbosity level (def level: 0) @@ -165,12 +155,13 @@ export USAGE="usage: $0 [-h] [-v level] [-V] [-n] [-N] [-b] [-q] [-t tool] [-Q] -b print only values (def: print both JSON member and JSON value) -q remove enclosing double quotes (def: keep any enclosing double quotes) + -T trim all newlines (def: do not) -t tool tool to use: jsp or JSONPath.sh (def: try jsp, otherwise try JSONPath.sh) -Q quick check tool using trivial input (def: do not) file.json JSON file to read, - ==> read stdin - pattern JSONPath query (def: $PATTERN} + pattern Xpath for JSON query pattern (def: $PATTERN} Exit codes: 0 all OK @@ -193,7 +184,7 @@ export DO_NOT_PROCESS= # parse command line # -while getopts :hv:VnNbqt:Q flag; do +while getopts :hv:VnNbqTt:Q flag; do case "$flag" in h) echo "$USAGE" 1>&2 exit 2 @@ -211,6 +202,8 @@ while getopts :hv:VnNbqt:Q flag; do ;; q) REMOVE_QUOTES="true" ;; + T) TRIM_NEWLINE="true" + ;; t) # validate -t tool case "$OPTARG" in jsp) XPATHJSON_USE="jsp" ;; @@ -339,23 +332,6 @@ fi export BIN_DIR="bin" -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # validate the XPath for JSON tool # case "$XPATHJSON_USE" in @@ -455,6 +431,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: PRINT_ONLY_VALUES=$PRINT_ONLY_VALUES" 1>&2 echo "$0: debug[3]: REMOVE_QUOTES=$REMOVE_QUOTES" 1>&2 echo "$0: debug[3]: QUICK_CHECK=$QUICK_CHECK" 1>&2 + echo "$0: debug[3]: TRIM_NEWLINE=$TRIM_NEWLINE" 1>&2 echo "$0: debug[3]: NOOP=$NOOP" 1>&2 echo "$0: debug[3]: DO_NOT_PROCESS=$DO_NOT_PROCESS" 1>&2 echo "$0: debug[3]: JSON_FILE=$JSON_FILE" 1>&2 @@ -462,7 +439,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: CD_FAILED=$CD_FAILED" 1>&2 echo "$0: debug[3]: BIN_PATH=$BIN_PATH" 1>&2 echo "$0: debug[3]: BIN_DIR=$BIN_DIR" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 fi @@ -502,30 +478,72 @@ jsp) # if [[ -z $JSON_FILE ]]; then - if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: about to: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN'" 1>&2 - fi - "$JSP_TOOL" --no-color --format --indent 0 -- "$PATTERN" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: ERROR: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' failed", \ - "error code: $status" 1>&2 - exit 1 + if [[ -z $TRIM_NEWLINE ]]; then + + # case: jsp w/o -T from standard input + # + if [[ $V_FLAG -ge 3 ]]; then + echo "$0: debug[3]: about to: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN'" 1>&2 + fi + "$JSP_TOOL" --no-color --format --indent 0 -- "$PATTERN" + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: ERROR: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' failed", \ + "error code: $status" 1>&2 + exit 1 + fi + + else + + # case: jsp -T from standard input + # + if [[ $V_FLAG -ge 3 ]]; then + NL='\\n' + echo "$0: debug[3]: about to: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' | tr -d $NL" 1>&2 + fi + "$JSP_TOOL" --no-color --format --indent 0 -- "$PATTERN" | tr -d '\n' + status_codes=("${PIPESTATUS[@]}") + if [[ ${status_codes[*]} =~ [1-9] ]]; then + echo "$0: ERROR: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' failed", \ + "error codes: ${status_codes[*]}" 1>&2 + exit 1 + fi fi # case: jsp processing from a file # else - if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: about to: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' < $JSON_FILE" 1>&2 - fi - "$JSP_TOOL" --no-color --format --indent 0 -- "$PATTERN" < "$JSON_FILE" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: ERROR: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' < $JSON_FILE failed", \ - "error code: $status" 1>&2 - exit 1 + if [[ -z $TRIM_NEWLINE ]]; then + + # case: jsp w/o -T from file + # + if [[ $V_FLAG -ge 3 ]]; then + echo "$0: debug[3]: about to: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' < $JSON_FILE" 1>&2 + fi + "$JSP_TOOL" --no-color --format --indent 0 -- "$PATTERN" < "$JSON_FILE" + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: ERROR: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' < $JSON_FILE failed", \ + "error code: $status" 1>&2 + exit 1 + fi + + else + + # case: jsp -T from file + # + if [[ $V_FLAG -ge 3 ]]; then + NL='\\n' + echo "$0: debug[3]: about to: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' < $JSON_FILE | tr -d $NL" 1>&2 + fi + "$JSP_TOOL" --no-color --format --indent 0 -- "$PATTERN" < "$JSON_FILE" | tr -d '\n' + status_codes=("${PIPESTATUS[@]}") + if [[ ${status_codes[*]} =~ [1-9] ]]; then + echo "$0: ERROR: $JSP_TOOL --no-color --format --indent 0 -- '$PATTERN' < $JSON_FILE failed", \ + "error codes: ${status_codes[*]}" 1>&2 + exit 1 + fi fi # case: print only values from jsp output @@ -621,32 +639,76 @@ JSONPath.sh) # if [[ -z $JSON_FILE ]]; then - if [[ $V_FLAG -ge 1 ]]; then - echo "$0: debug[1]: about to: $JSONPATH_SH -w -u -- '$PATTERN'" 1>&2 - fi - "$JSONPATH_SH" -w -u -- "$PATTERN" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: ERROR: $JSONPATH_SH -w -u -- '$PATTERN' failed", \ - "error code: $status" 1>&2 - exit 1 + if [[ -z $TRIM_NEWLINE ]]; then + + # case: JSONPath.sh w/o -T from standard input + # + if [[ $V_FLAG -ge 1 ]]; then + echo "$0: debug[1]: about to: $JSONPATH_SH -w -u -- '$PATTERN'" 1>&2 + fi + "$JSONPATH_SH" -w -u -- "$PATTERN" + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: ERROR: $JSONPATH_SH -w -u -- '$PATTERN' failed", \ + "error code: $status" 1>&2 + exit 1 + fi + + else + + # case: JSONPath.sh -T from standard input + # + if [[ $V_FLAG -ge 1 ]]; then + NL='\\n' + echo "$0: debug[1]: about to: $JSONPATH_SH -w -u -- '$PATTERN' | tr -d $NL" 1>&2 + fi + "$JSP_TOOL" --no-color --format --indent 0 -- "$PATTERN" | tr -d '\n' + status_codes=("${PIPESTATUS[@]}") + if [[ $status -ne 0 ]]; then + echo "$0: ERROR: $JSONPATH_SH -w -u -- '$PATTERN' failed", \ + "error codes: ${status_codes[*]}" 1>&2 + exit 1 + fi fi # case: JSONPath.sh processing from a file # else - if [[ $V_FLAG -ge 1 ]]; then - echo "$0: debug[1]: about to: $JSONPATH_SH -w -u -f $JSON_FILE -- '$PATTERN'" 1>&2 - fi - "$JSONPATH_SH" -w -u -f "$JSON_FILE" -- "$PATTERN" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: ERROR: $JSONPATH_SH -w -u -f $JSON_FILE -- '$PATTERN' failed", \ - "error code: $status" 1>&2 - exit 1 + if [[ -z $TRIM_NEWLINE ]]; then + + # case: JSONPath.sh w/o -T from file + # + if [[ $V_FLAG -ge 1 ]]; then + echo "$0: debug[1]: about to: $JSONPATH_SH -w -u -f $JSON_FILE -- '$PATTERN'" 1>&2 + fi + "$JSONPATH_SH" -w -u -f "$JSON_FILE" -- "$PATTERN" + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: ERROR: $JSONPATH_SH -w -u -f $JSON_FILE -- '$PATTERN' failed", \ + "error code: $status" 1>&2 + exit 1 + fi + + else + + # case: JSONPath.sh -T from file + # + if [[ $V_FLAG -ge 1 ]]; then + NL='\\n' + echo "$0: debug[1]: about to: $JSONPATH_SH -w -u -f $JSON_FILE -- '$PATTERN' | tr -d $NL" 1>&2 + fi + "$JSONPATH_SH" -w -u -f "$JSON_FILE" -- "$PATTERN" | tr -d '\n' + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: ERROR: $JSONPATH_SH -w -u -f $JSON_FILE -- '$PATTERN' failed", \ + "error code: $status" 1>&2 + exit 1 + fi + fi + # case: print only values from JSONPath.sh output # fi | if [[ -n $PRINT_ONLY_VALUES ]]; then diff --git a/bin/output-index-author.sh b/bin/output-index-author.sh index dd478bb78..1521030ee 100755 --- a/bin/output-index-author.sh +++ b/bin/output-index-author.sh @@ -104,7 +104,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.9 2024-09-23" +export VERSION="1.10 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -120,6 +120,18 @@ if [[ $status -eq 0 ]]; then TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) fi export TOPDIR +# +JSTRDECODE=$(type -P jstrdecode) +export JSTRDECODE +if [[ -z $JSTRDECODE ]]; then + echo "$0: FATAL: jstrdecode is not installed or not in \$PATH" 1>&2 + echo "$0: notice: to install jstrdecode:" 1>&2 + echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2 + echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2 + echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2 + exit 5 +fi +# export DOCROOT_SLASH="../../" export REPO_TOP_URL="https://github.com/ioccc-src/temp-test-ioccc" # GitHub puts individual files under the "blob/master" sub-directory. @@ -312,7 +324,7 @@ function output_full_name # extract Full Name from the author/author_handle.json file # PATTERN='$..full_name' - "$JVAL_WRAPPER" -b -q "$AUTHOR_HANDLE_JSON_PATH" "$PATTERN" | sed -f "$UNICODE_FIX_SED" + "$JVAL_WRAPPER" -b -q -T "$AUTHOR_HANDLE_JSON_PATH" "$PATTERN" | "$JSTRDECODE" - status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] ]]; then echo "$0: ERROR: in output_full_name: $JVAL_WRAPPER -b -q $AUTHOR_HANDLE_JSON_PATH '$PATTERN' failed," \ @@ -646,23 +658,6 @@ if [[ ! -x $LOCATION_TOOL ]]; then fi -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # parameter debugging # if [[ $V_FLAG -ge 3 ]]; then @@ -671,6 +666,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2 echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2 echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2 + echo "$0: debug[3]: JSTRDECODE=$JSTRDECODE" 1>&2 echo "$0: debug[3]: DOCROOT_SLASH=$DOCROOT_SLASH" 1>&2 echo "$0: debug[3]: REPO_TOP_URL=$REPO_TOP_URL" 1>&2 echo "$0: debug[3]: REPO_URL=$REPO_URL" 1>&2 @@ -698,7 +694,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: DOT_PATH_CONTENT=$DOT_PATH_CONTENT" 1>&2 echo "$0: debug[3]: ENTRY_JSON=$ENTRY_JSON" 1>&2 echo "$0: debug[3]: LOCATION_TOOL=$LOCATION_TOOL" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 fi diff --git a/bin/output-year-index.sh b/bin/output-year-index.sh index 89a332ebb..54b1369af 100755 --- a/bin/output-year-index.sh +++ b/bin/output-year-index.sh @@ -113,7 +113,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.5 2024-09-23" +export VERSION="1.6 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -129,6 +129,18 @@ if [[ $status -eq 0 ]]; then TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) fi export TOPDIR +# +JSTRDECODE=$(type -P jstrdecode) +export JSTRDECODE +if [[ -z $JSTRDECODE ]]; then + echo "$0: FATAL: jstrdecode is not installed or not in \$PATH" 1>&2 + echo "$0: notice: to install jstrdecode:" 1>&2 + echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2 + echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2 + echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2 + exit 5 +fi +# export DOCROOT_SLASH="../../" export REPO_TOP_URL="https://github.com/ioccc-src/temp-test-ioccc" # GitHub puts individual files under the "blob/master" sub-directory. @@ -216,13 +228,13 @@ function output_award # PATTERN='$..award' if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN'" 1>&2 + echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE -" 1>&2 fi - "$JVAL_WRAPPER" -b -q "$ENTRY_JSON_PATH" "$PATTERN" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: ERROR: in output_award: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN' failed," \ - "error code: $status" 1>&2 + "$JVAL_WRAPPER" -b -q -T "$ENTRY_JSON_PATH" "$PATTERN" | "$JSTRDECODE" - + status_codes=("${PIPESTATUS[@]}") + if [[ ${status_codes[*]} =~ [1-9] ]]; then + echo "$0: ERROR: in output_award: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE - failed," \ + "error codes: ${status_codes[*]}" 1>&2 return 5 fi return 0 @@ -534,23 +546,6 @@ if [[ -z $ORDINAL ]]; then fi -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # parameter debugging # if [[ $V_FLAG -ge 3 ]]; then @@ -559,6 +554,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2 echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2 echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2 + echo "$0: debug[3]: JSTRDECODE=$JSTRDECODE" 1>&2 echo "$0: debug[3]: DOCROOT_SLASH=$DOCROOT_SLASH" 1>&2 echo "$0: debug[3]: REPO_TOP_URL=$REPO_TOP_URL" 1>&2 echo "$0: debug[3]: REPO_URL=$REPO_URL" 1>&2 @@ -572,7 +568,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: YEAR_FILE=$YEAR_FILE" 1>&2 echo "$0: debug[3]: LINE_NUM=$LINE_NUM" 1>&2 echo "$0: debug[3]: ORDINAL=$ORDINAL" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 fi @@ -697,7 +692,7 @@ for YYYY_DIR in $(< "$YEAR_FILE"); do # determine the award for this entry # - AWARD=$(output_award "$ENTRY_JSON" | sed -f "$UNICODE_FIX_SED") + AWARD=$(output_award "$ENTRY_JSON") status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] || -z $AWARD ]]; then echo "$0: ERROR: cannot find award in .entry.json: $ENTRY_JSON" 1>&2 diff --git a/bin/subst.entry-index.sh b/bin/subst.entry-index.sh index 16da9a8f8..b93974be8 100755 --- a/bin/subst.entry-index.sh +++ b/bin/subst.entry-index.sh @@ -106,7 +106,7 @@ shopt -s globstar # enable ** to match all files and zero or more directories an # set variables referenced in the usage message # -export VERSION="1.7 2024-09-27" +export VERSION="1.8 2024-10-08" NAME=$(basename "$0") export NAME export V_FLAG=0 @@ -122,6 +122,18 @@ if [[ $status -eq 0 ]]; then TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) fi export TOPDIR +# +JSTRDECODE=$(type -P jstrdecode) +export JSTRDECODE +if [[ -z $JSTRDECODE ]]; then + echo "$0: FATAL: jstrdecode is not installed or not in \$PATH" 1>&2 + echo "$0: notice: to install jstrdecode:" 1>&2 + echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2 + echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2 + echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2 + exit 5 +fi +# export DOCROOT_SLASH="../../" export REPO_TOP_URL="https://github.com/ioccc-src/temp-test-ioccc" # GitHub puts individual files under the "blob/master" sub-directory. @@ -206,13 +218,12 @@ function output_award # PATTERN='$..award' if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN' | sed ..." 1>&2 + echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE -" 1>&2 fi - "$JVAL_WRAPPER" -b -q "$ENTRY_JSON_PATH" "$PATTERN" | sed -f "$UNICODE_FIX_SED" - status="$?" + "$JVAL_WRAPPER" -b -q -T "$ENTRY_JSON_PATH" "$PATTERN" | "$JSTRDECODE" - status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] ]]; then - echo "$0: ERROR: in output_award: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN | sed ... failed," \ + echo "$0: ERROR: in output_award: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE - failed," \ "error codes: ${status_codes[*]}" 1>&2 return 5 fi @@ -258,13 +269,12 @@ function output_abstract # PATTERN='$..abstract' if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN' | sed ..." 1>&2 + echo "$0: debug[5]: about to run: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN' | $JSTRDECODE -" 1>&2 fi - "$JVAL_WRAPPER" -b -q "$ENTRY_JSON_PATH" "$PATTERN" | sed -f "$UNICODE_FIX_SED" - status="$?" + "$JVAL_WRAPPER" -b -q -T "$ENTRY_JSON_PATH" "$PATTERN" | "$JSTRDECODE" - status_codes=("${PIPESTATUS[@]}") if [[ ${status_codes[*]} =~ [1-9] ]]; then - echo "$0: ERROR: in output_abstract: $JVAL_WRAPPER -b -q -- $ENTRY_JSON_PATH '$PATTERN | sed ... failed," \ + echo "$0: ERROR: in output_abstract: $JVAL_WRAPPER -b -q -T -- $ENTRY_JSON_PATH '$PATTERN | $JSTRDECODE - failed," \ "error codes: ${status_codes[*]}" 1>&2 return 5 fi @@ -445,23 +455,6 @@ if [[ ! -x $JVAL_WRAPPER ]]; then fi -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - # verify we have our awk script # export ENTRY_NAVBAR_AWK="$BIN_DIR/subst.entry-navbar.awk" @@ -581,6 +574,7 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2 echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2 echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2 + echo "$0: debug[3]: JSTRDECODE=$JSTRDECODE" 1>&2 echo "$0: debug[3]: DOCROOT_SLASH=$DOCROOT_SLASH" 1>&2 echo "$0: debug[3]: REPO_TOP_URL=$REPO_TOP_URL" 1>&2 echo "$0: debug[3]: REPO_URL=$REPO_URL" 1>&2 @@ -597,7 +591,6 @@ if [[ $V_FLAG -ge 3 ]]; then echo "$0: debug[3]: BIN_PATH=$BIN_PATH" 1>&2 echo "$0: debug[3]: BIN_DIR=$BIN_DIR" 1>&2 echo "$0: debug[3]: JVAL_WRAPPER=$JVAL_WRAPPER" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 echo "$0: debug[3]: ENTRY_JSON=$ENTRY_JSON" 1>&2 echo "$0: debug[3]: YEAR_DIR=$YEAR_DIR" 1>&2 echo "$0: debug[3]: ENTRY_DIR=$ENTRY_DIR" 1>&2 diff --git a/bin/unicode-chk.sh b/bin/unicode-chk.sh deleted file mode 100755 index d0ec87a0c..000000000 --- a/bin/unicode-chk.sh +++ /dev/null @@ -1,735 +0,0 @@ -#!/usr/bin/env bash -# -# unicode-chk.sh - verify all \\u[[:xdigit:]]{4} and JSON syntax in entry and author JSON files -# -# Pass entry and author JSON files thru the following sed script and -# flag any \\u[[:xdigit:]]{4} that we might find as problems. -# -# bin/unicode-fix.sed -# -# XXX - XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - XXX -# XXX - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - XXX -# XXX - until we have fixed jstrdecode, we must sed out \u[0-0A-fa-f]{4} from JSON - XXX -# XXX - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - XXX -# XXX - XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - XXX -# -# Until this bug is the jparse repo is fixed, we need to avoid -# using \\u[[:xdigit:]]{4} within JSON encoded strings. -# -# https://github.com/xexyl/jparse/issues/13 -# -# The above mentioned bug results in certain tools that are based on the -# jparse repo: -# -# https://github.com/xexyl/jparse -# -# including code from the mkiocccentry repo's jparse sub-directory: -# -# https://github.com/ioccc-src/mkiocccentry/tree/master/jparse -# -# We need to avoid using \\u[[:xdigit:]]{4} within JSON encoded strings. -# Sorry (tm Canada :-)). -# -# The following sed script converts known problematic \\u[[:xdigit:]]{4} -# combinations into their RAW character form: -# -# bin/unicode-fix.sed -# -# Use of that sed script serves as a work-a-round to avoid to avoid any -# problematic \\u[[:xdigit:]]{4} combinations in JSON encoded strings. -# -# Copyright (c) 2024 by Landon Curt Noll. All Rights Reserved. -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby granted, -# provided that the above copyright, this permission notice and text -# this comment, and the disclaimer below appear in all of the following: -# -# supporting documentation -# source copies -# source works derived from this source -# binaries derived from this source or from derived source -# -# LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO -# EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF -# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. -# -# chongo (Landon Curt Noll, http://www.isthe.com/chongo/index.html) /\oo/\ -# -# Share and enjoy! :-) - - -# firewall - run only with a bash that is version 5.1.8 or later -# -# The "/usr/bin/env bash" command must result in using a bash that -# is version 5.1.8 or later. -# -# We could relax this version and insist on version 4.2 or later. Versions -# of bash between 4.2 and 5.1.7 might work. However, to be safe, we will require -# bash version 5.1.8 or later. -# -# WHY 5.1.8 and not 4.2? This safely is done because macOS Homebrew bash we -# often use is "version 5.2.26(1)-release" or later, and the RHEL Linux bash we -# use often use is "version 5.1.8(1)-release" or later. These versions are what -# we initially tested. We recommend you either upgrade bash or install a newer -# version of bash and adjust your $PATH so that "/usr/bin/env bash" finds a bash -# that is version 5.1.8 or later. -# -# NOTE: The macOS shipped, as of 2024 March 15, a version of bash is something like -# bash "version 3.2.57(1)-release". That macOS shipped version of bash -# will NOT work. For users of macOS we recommend you install Homebrew, -# (see https://brew.sh), and then run "brew install bash" which will -# typically install it into /opt/homebrew/bin/bash, and then arrange your $PATH -# so that "/usr/bin/env bash" finds "/opt/homebrew/bin" (or whatever the -# Homebrew bash is). -# -# NOTE: And while MacPorts might work, we noticed a number of subtle differences -# with some of their ported tools to suggest you might be better off -# with installing Homebrew (see https://brew.sh). No disrespect is intended -# to the MacPorts team as they do a commendable job. Nevertheless we ran -# into enough differences with MacPorts environments to suggest you -# might find a better experience with this tool under Homebrew instead. -# -if [[ -z ${BASH_VERSINFO[0]} || - ${BASH_VERSINFO[0]} -lt 5 || - ${BASH_VERSINFO[0]} -eq 5 && ${BASH_VERSINFO[1]} -lt 1 || - ${BASH_VERSINFO[0]} -eq 5 && ${BASH_VERSINFO[1]} -eq 1 && ${BASH_VERSINFO[2]} -lt 8 ]]; then - echo "$0: ERROR: bash version needs to be >= 5.1.8: $BASH_VERSION" 1>&2 - echo "$0: Warning: bash version >= 4.2 might work but 5.1.8 was the minimum we tested" 1>&2 - echo "$0: Notice: For macOS users: install Homebrew (see https://brew.sh), then run" \ - ""brew install bash" and then modify your \$PATH so that \"#!/usr/bin/env bash\"" \ - "finds the Homebrew installed (usually /opt/homebrew/bin/bash) version of bash" 1>&2 - exit 4 -fi - - -# setup bash file matching -# -# We must declare arrays with -ag or -Ag, and we need loops to "export" modified variables. -# This requires a bash with a version 4.2 or later. See the larger comment above about bash versions. -# -shopt -s nullglob # enable expanded to nothing rather than remaining unexpanded -shopt -u failglob # disable error message if no matches are found -shopt -u dotglob # disable matching files starting with . -shopt -u nocaseglob # disable strict case matching -shopt -u extglob # enable extended globbing patterns -shopt -s globstar # enable ** to match all files and zero or more directories and subdirectories - - -# set variables referenced in the usage message -# -export VERSION="1.1 2024-09-24" -NAME=$(basename "$0") -export NAME -export V_FLAG=0 -GIT_TOOL=$(type -P git) -export GIT_TOOL -if [[ -z "$GIT_TOOL" ]]; then - echo "$0: FATAL: git tool is not installed or not in \$PATH" 1>&2 - exit 5 -fi -"$GIT_TOOL" rev-parse --is-inside-work-tree >/dev/null 2>&1 -status="$?" -if [[ $status -eq 0 ]]; then - TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel) -fi -export TOPDIR -export DOCROOT_SLASH="../../" -export TAGLINE="unspecified" -export REPO_TOP_URL="https://github.com/ioccc-src/temp-test-ioccc" -# GitHub puts individual files under the "blob/master" sub-directory. -export REPO_URL="$REPO_TOP_URL/blob/master" -JPARSE_TOOL=$(type -P jparse) -export JPARSE_TOOL -if [[ -z "$JPARSE_TOOL" ]]; then - echo "$0: FATAL: jparse tool is not installed or not in \$PATH" 1>&2 - exit 5 -fi -export SITE_URL="https://ioccc-src.github.io/temp-test-ioccc" -# -export NOOP= -export DO_NOT_PROCESS= -export EXIT_CODE="0" - - -# usage -# -export USAGE="usage: $0 [-h] [-v level] [-V] [-d topdir] [-D docroot/] [-n] [-N] [file.json ..] - - -h print help message and exit - -v level set verbosity level (def level: 0) - -V print version string and exit - - -d topdir set topdir (def: $TOPDIR) - -D docroot/ set the document root path followed by slash (def: $DOCROOT_SLASH) - NOTE: The '-D docroot/' is passed as leading options on tool command lines. - NOTE: 'docroot' must end in a slash - - -n go thru the actions, but do not update any files (def: do the action) - -N do not process anything, just parse arguments (def: process something) - - [file.json ...] check file.json files (def: check all entry and author JSON files) - -Exit codes: - 0 all OK - 1 found unexpected \\uHexHexHexHex strings in JSON files - 2 -h and help string printed or -V and version string printed - 3 command line error - 4 bash version is too old - 5 some internal tool is not found or not an executable file - 6 problems found with or in the topdir or topdir/YYYY directory - 7 problems found with or in the entry topdir/YYYY/dir directory - 8 file.json not a readable file - >= 10 internal error - -$NAME version: $VERSION" - - -# parse command line -# -while getopts :hv:Vd:D:j:nN flag; do - case "$flag" in - h) echo "$USAGE" 1>&2 - exit 2 - ;; - v) V_FLAG="$OPTARG" - ;; - V) echo "$VERSION" - exit 2 - ;; - d) TOPDIR="$OPTARG" - ;; - D) # parse -D docroot/ - case "$OPTARG" in - */) ;; - *) echo "$0: ERROR: in -D docroot/, the docroot must end in /" 1>&2 - echo 1>&2 - echo "$USAGE" 1>&2 - exit 3 - ;; - esac - DOCROOT_SLASH="$OPTARG" - ;; - n) NOOP="-n" - ;; - N) DO_NOT_PROCESS="-N" - ;; - \?) echo "$0: ERROR: invalid option: -$OPTARG" 1>&2 - echo 1>&2 - echo "$USAGE" 1>&2 - exit 3 - ;; - :) echo "$0: ERROR: option -$OPTARG requires an argument" 1>&2 - echo 1>&2 - echo "$USAGE" 1>&2 - exit 3 - ;; - *) echo "$0: ERROR: unexpected value from getopts: $flag" 1>&2 - echo 1>&2 - echo "$USAGE" 1>&2 - exit 3 - ;; - esac -done -# -# remove the options -# -shift $(( OPTIND - 1 )); -# -if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: file argument count: $#" 1>&2 -fi - - -# verify that we have a topdir directory -# -REPO_NAME=$(basename "$REPO_TOP_URL") -export REPO_NAME -if [[ -z $TOPDIR ]]; then - echo "$0: ERROR: cannot find top of git repo directory" 1>&2 - echo "$0: Notice: if needed: $GIT_TOOL clone $REPO_TOP_URL; cd $REPO_NAME" 1>&2 - exit 6 -fi -if [[ ! -e $TOPDIR ]]; then - echo "$0: ERROR: TOPDIR does not exist: $TOPDIR" 1>&2 - echo "$0: Notice: if needed: $GIT_TOOL clone $REPO_TOP_URL; cd $REPO_NAME" 1>&2 - exit 6 -fi -if [[ ! -d $TOPDIR ]]; then - echo "$0: ERROR: TOPDIR is not a directory: $TOPDIR" 1>&2 - echo "$0: Notice: if needed: $GIT_TOOL clone $REPO_TOP_URL; cd $REPO_NAME" 1>&2 - exit 6 -fi - - -# cd to topdir -# -if [[ ! -e $TOPDIR ]]; then - echo "$0: ERROR: cannot cd to non-existent path: $TOPDIR" 1>&2 - exit 6 -fi -if [[ ! -d $TOPDIR ]]; then - echo "$0: ERROR: cannot cd to a non-directory: $TOPDIR" 1>&2 - exit 6 -fi -export CD_FAILED -if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to: cd $TOPDIR" 1>&2 -fi -cd "$TOPDIR" || CD_FAILED="true" -if [[ -n $CD_FAILED ]]; then - echo "$0: ERROR: cd $TOPDIR failed" 1>&2 - exit 6 -fi -if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: now in directory: $(/bin/pwd)" 1>&2 -fi - - -# verify that we have an author subdirectory -# -export AUTHOR_PATH="$TOPDIR/author" -if [[ ! -d $AUTHOR_PATH ]]; then - echo "$0: ERROR: author is not a directory under topdir: $AUTHOR_PATH" 1>&2 - exit 6 -fi -export AUTHOR_DIR="author" - - -# verify that we have a bin subdirectory -# -export BIN_PATH="$TOPDIR/bin" -if [[ ! -d $BIN_PATH ]]; then - echo "$0: ERROR: bin is not a directory under topdir: $BIN_PATH" 1>&2 - exit 6 -fi -export BIN_DIR="bin" - - -# verify that the bin/unicode-fix.sed tool is executable -# -export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed" -if [[ ! -e $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -f $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi -if [[ ! -r $UNICODE_FIX_SED ]]; then - echo "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2 - exit 5 -fi - - -# verify we have a non-empty readable .top file -# -export TOP_FILE=".top" -if [[ ! -e $TOP_FILE ]]; then - echo "$0: ERROR: .top does not exist: $TOP_FILE" 1>&2 - exit 6 -fi -if [[ ! -f $TOP_FILE ]]; then - echo "$0: ERROR: .top is not a regular file: $TOP_FILE" 1>&2 - exit 6 -fi -if [[ ! -r $TOP_FILE ]]; then - echo "$0: ERROR: .top is not an readable file: $TOP_FILE" 1>&2 - exit 6 -fi -if [[ ! -s $TOP_FILE ]]; then - echo "$0: ERROR: .top is not a non-empty readable file: $TOP_FILE" 1>&2 - exit 6 -fi - - -# print running info if verbose -# -# If -v 3 or higher, print exported variables in order that they were exported. -# -if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: VERSION=$VERSION" 1>&2 - echo "$0: debug[3]: NAME=$NAME" 1>&2 - echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2 - echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2 - echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2 - echo "$0: debug[3]: DOCROOT_SLASH=$DOCROOT_SLASH" 1>&2 - echo "$0: debug[3]: TAGLINE=$TAGLINE" 1>&2 - echo "$0: debug[3]: REPO_TOP_URL=$REPO_TOP_URL" 1>&2 - echo "$0: debug[3]: REPO_URL=$REPO_URL" 1>&2 - echo "$0: debug[3]: JPARSE_TOOL=$JPARSE_TOOL" 1>&2 - echo "$0: debug[3]: SITE_URL=$SITE_URL" 1>&2 - echo "$0: debug[3]: NOOP=$NOOP" 1>&2 - echo "$0: debug[3]: DO_NOT_PROCESS=$DO_NOT_PROCESS" 1>&2 - echo "$0: debug[3]: EXIT_CODE=$EXIT_CODE" 1>&2 - echo "$0: debug[3]: REPO_NAME=$REPO_NAME" 1>&2 - echo "$0: debug[3]: CD_FAILED=$CD_FAILED" 1>&2 - echo "$0: debug[3]: AUTHOR_PATH=$AUTHOR_PATH" 1>&2 - echo "$0: debug[3]: AUTHOR_DIR=$AUTHOR_DIR" 1>&2 - echo "$0: debug[3]: BIN_PATH=$BIN_PATH" 1>&2 - echo "$0: debug[3]: BIN_DIR=$BIN_DIR" 1>&2 - echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2 - echo "$0: debug[3]: TOP_FILE=$TOP_FILE" 1>&2 -fi - - -# -N stops early before any processing is performed -# -if [[ -n $DO_NOT_PROCESS ]]; then - if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: arguments parsed, -N given, exiting 0" 1>&2 - fi - exit 0 -fi - - -# create a JSON problem file -# -export TMP_JSON_PROBLEM=".tmp.$NAME.JSON_PROBLEM.$$.tmp" -if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: JSON problem inventory file: $TMP_JSON_PROBLEM" 1>&2 -fi -trap 'rm -f $TMP_AUTHOR_HANDLE_INVENTORY; exit' 0 1 2 3 15 -rm -f "$TMP_JSON_PROBLEM" -if [[ -e $TMP_JSON_PROBLEM ]]; then - echo "$0: ERROR: cannot remove JSON problem inventory file: $TMP_JSON_PROBLEM" 1>&2 - exit 10 -fi -: > "$TMP_JSON_PROBLEM" -if [[ ! -e $TMP_JSON_PROBLEM ]]; then - echo "$0: ERROR: cannot create JSON problem inventory file: $TMP_JSON_PROBLEM" 1>&2 - exit 11 -fi - - -# case: check only file given as args -# -if [[ $# -gt 0 ]]; then - - # check specific JSON files - # - for JSON_FILE in "$@"; do - - # just be a readable file - # - if [[ ! -e $JSON_FILE ]]; then - echo "$0: ERROR: does not exist: $JSON_FILE" 1>&2 - exit 8 - fi - if [[ ! -f $JSON_FILE ]]; then - echo "$0: ERROR: is not a regular file: $JSON_FILE" 1>&2 - exit 8 - fi - if [[ ! -r $JSON_FILE ]]; then - echo "$0: ERROR: is not an readable file: $JSON_FILE" 1>&2 - exit 8 - fi - - # verify the JSON of the file - # - if [[ $V_FLAG -ge 1 ]]; then - echo "$0: debug[1]: about to run: $JPARSE_TOOL -q -- $JSON_FILE" 1>&2 - fi - "$JPARSE_TOOL" -q -- "$JSON_FILE" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: Warning: invalid JSON: $JSON_FILE" 1>&2 - echo "$JSON_FILE" - continue - fi - - # scan the author/author_handle.json file for unexpected \\u[[:xdigit:]]{4} strings - # - if [[ $V_FLAG -ge 1 ]]; then - echo "$0: debug[1]: about to run: sed -f $UNICODE_FIX_SED -- $JSON_FILE 2>&1 | grep -E ..." 1>&2 - fi - FOUND_UHEX=$(sed -f "$UNICODE_FIX_SED" -- "$JSON_FILE" 2>&1 | grep -E '\\u[[:xdigit:]]{4}') - if [[ -n $FOUND_UHEX ]]; then - echo "$JSON_FILE" - fi - - done >> "$TMP_JSON_PROBLEM" - - -# case: check all .entry.json files and author JSON files -# -else - - # check all .entry.json files - # - if [[ $V_FLAG -ge 1 ]]; then - echo "$0: debug[1]: starting to check .entry.json files under: $TOPDIR" 1>&2 - fi - export YYYY - for YYYY in $(< "$TOP_FILE"); do - - # debug YYYY - # - if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: starting to check year: $YYYY" 1>&2 - fi - - # verify that YYYY is a readable directory - # - if [[ ! -e $YYYY ]]; then - echo "$0: ERROR: YYYY does not exist: $YYYY" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -d $YYYY ]]; then - echo "$0: ERROR: YYYY is not a directory: $YYYY" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -r $YYYY ]]; then - echo "$0: ERROR: YYYY is not an readable directory: $YYYY" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - - # verify that YYYY has a non-empty readable .year file - # - export YEAR_FILE="$YYYY/.year" - if [[ ! -e $YEAR_FILE ]]; then - echo "$0: ERROR: YYYY/.year does not exist: $YEAR_FILE" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -f $YEAR_FILE ]]; then - echo "$0: ERROR: YYYY/.year is not a regular file: $YEAR_FILE" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -r $YEAR_FILE ]]; then - echo "$0: ERROR: YYYY/.year is not an readable file: $YEAR_FILE" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -s $YEAR_FILE ]]; then - echo "$0: ERROR: YYYY/.year is not a non-empty readable file: $YEAR_FILE" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - - # process each entry directory under YYYY - # - export YYYY_DIR - for YYYY_DIR in $(< "$YEAR_FILE"); do - - # debug YYYY - # - if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: starting to check year/dir: $YYYY_DIR" 1>&2 - fi - - # parse YYYY_DIR - # - if [[ ! -d $YYYY_DIR ]]; then - echo "$0: ERROR: YYYY_DIR is not a directory: $YYYY_DIR" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -w $YYYY_DIR ]]; then - echo "$0: ERROR: YYYY_DIR is not a writable directory: $YYYY_DIR" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - export YEAR_DIR=${YYYY_DIR%%/*} - if [[ -z $YEAR_DIR ]]; then - echo "$0: ERROR: YYYY_DIR not in YYYY/dir form: $YYYY_DIR" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - export ENTRY_DIR=${YYYY_DIR#*/} - if [[ -z $ENTRY_DIR ]]; then - echo "$0: ERROR: YYYY_DIR not in $YEAR_DIR/dir form: $YYYY_DIR" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ $ENTRY_DIR = */* ]]; then - echo "$0: ERROR: YYYY_DIR: $YYYY_DIR dir contains a /: $ENTRY_DIR" 1>&2 - EXIT_CODE="6" # exit 6 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - - # verify that YYYY_DIR is a writable directory - # - if [[ ! -e $YYYY_DIR ]]; then - echo "$0: ERROR: YYYY_DIR does not exist: $YYYY_DIR" 1>&2 - EXIT_CODE="7" # exit 7 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -d $YYYY_DIR ]]; then - echo "$0: ERROR: YYYY_DIR is not a directory: $YYYY_DIR" 1>&2 - EXIT_CODE="7" # exit 7 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -w $YYYY_DIR ]]; then - echo "$0: ERROR: YYYY_DIR is not an writable directory: $YYYY_DIR" 1>&2 - EXIT_CODE="7" # exit 7 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - - # verify YYYY/dir/.path - # - export DOT_PATH="$YYYY_DIR/.path" - if [[ ! -s $DOT_PATH ]]; then - echo "$0: ERROR: not a non-empty file: $DOT_PATH" 1>&2 - EXIT_CODE="7" # exit 7 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - DOT_PATH_CONTENT=$(< "$DOT_PATH") - if [[ $YYYY_DIR != "$DOT_PATH_CONTENT" ]]; then - echo "$0: ERROR: arg: $YYYY_DIR does not match $DOT_PATH contents: $DOT_PATH_CONTENT" 1>&2 - EXIT_CODE="7" # exit 7 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - - # verify the YYYY/dir/.entry.json - # - export ENTRY_JSON="$YYYY_DIR/.entry.json" - if [[ ! -e $ENTRY_JSON ]]; then - echo "$0: ERROR: .entry.json does not exist: $ENTRY_JSON" 1>&2 - EXIT_CODE="7" # exit 7 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -f $ENTRY_JSON ]]; then - echo "$0: ERROR: .entry.json is not a file: $ENTRY_JSON" 1>&2 - EXIT_CODE="7" # exit 7 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - if [[ ! -r $ENTRY_JSON ]]; then - echo "$0: ERROR: .entry.json is not a readable file: $ENTRY_JSON" 1>&2 - EXIT_CODE="7" # exit 7 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - continue - fi - - # verify the JSON of the .entry.json file - # - if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: about to run: $JPARSE_TOOL -q -- $YYYY_DIR/.entry.json" 1>&2 - fi - "$JPARSE_TOOL" -q -- "$YYYY_DIR/.entry.json" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: Warning: invalid JSON: $YYYY_DIR/.entry.json" 1>&2 - echo "$YYYY_DIR/.entry.json" - continue - fi - - # scan the .entry.json file for unexpected \\u[[:xdigit:]]{4} strings - # - if [[ -z $NOOP ]]; then - if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to run: sed -f $UNICODE_FIX_SED -- $YYYY_DIR/.entry.json 2>&1 | grep -E ..." 1>&2 - fi - FOUND_UHEX=$(sed -f "$UNICODE_FIX_SED" -- "$YYYY_DIR/.entry.json" 2>&1 | grep -E '\\u[[:xdigit:]]{4}' 2>&1) - export FOUND_UHEX - if [[ -n $FOUND_UHEX ]]; then - echo "$YYYY_DIR/.entry.json" - fi - fi - done - done >> "$TMP_JSON_PROBLEM" - - - # create a temporary author_handle inventory file - # - export TMP_AUTHOR_HANDLE_INVENTORY=".tmp.$NAME.AUTHOR_HANDLE_INVENTORY.$$.tmp" - if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: temporary author_handle inventory file: $TMP_AUTHOR_HANDLE_INVENTORY" 1>&2 - fi - trap 'rm -f $TMP_AUTHOR_HANDLE_INVENTORY $TMP_AUTHOR_HANDLE_INVENTORY; exit' 0 1 2 3 15 - rm -f "$TMP_AUTHOR_HANDLE_INVENTORY" - if [[ -e $TMP_AUTHOR_HANDLE_INVENTORY ]]; then - echo "$0: ERROR: cannot remove temporary author_handle inventory file: $TMP_AUTHOR_HANDLE_INVENTORY" 1>&2 - exit 12 - fi - : > "$TMP_AUTHOR_HANDLE_INVENTORY" - if [[ ! -e $TMP_AUTHOR_HANDLE_INVENTORY ]]; then - echo "$0: ERROR: cannot create temporary author_handle inventory file: $TMP_AUTHOR_HANDLE_INVENTORY" 1>&2 - exit 13 - fi - - # check author/author_handle.json files - # - if [[ $V_FLAG -ge 1 ]]; then - echo "$0: debug[1]: starting to check author_handle.json files under: $TOPDIR/$AUTHOR_DIR" 1>&2 - fi - find "$AUTHOR_DIR" -mindepth 1 -maxdepth 1 -type f -name '*.json' 2>/dev/null | - LC_ALL=C sort -d -u > "$TMP_AUTHOR_HANDLE_INVENTORY" - while read -r AUTHOR_HANDLE_JSON; do - - # verify the JSON of the .entry.json file - # - if [[ $V_FLAG -ge 3 ]]; then - echo "$0: debug[3]: about to run: $JPARSE_TOOL -q -- $AUTHOR_HANDLE_JSON" 1>&2 - fi - "$JPARSE_TOOL" -q -- "$AUTHOR_HANDLE_JSON" - status="$?" - if [[ $status -ne 0 ]]; then - echo "$0: Warning: invalid JSON: $AUTHOR_HANDLE_JSON" 1>&2 - echo "$AUTHOR_HANDLE_JSON" - continue - fi - - # scan the author/author_handle.json file for unexpected \\u[[:xdigit:]]{4} strings - # - if [[ $V_FLAG -ge 5 ]]; then - echo "$0: debug[5]: about to run: sed -f $UNICODE_FIX_SED -- $AUTHOR_HANDLE_JSON 2>&1 | grep -E ..." 1>&2 - fi - FOUND_UHEX=$(sed -f "$UNICODE_FIX_SED" -- "$AUTHOR_HANDLE_JSON" 2>&1 | grep -E '\\u[[:xdigit:]]{4}') - if [[ -n $FOUND_UHEX ]]; then - echo "$AUTHOR_HANDLE_JSON" - fi - - done < "$TMP_AUTHOR_HANDLE_INVENTORY" >> "$TMP_JSON_PROBLEM" -fi - - -# report any problems -# -if [[ -s $TMP_JSON_PROBLEM ]]; then - echo "$0: debug[5]: list of JSON file problems, starts below" 1>&2 - cat "$TMP_JSON_PROBLEM" 1>&2 - echo "$0: debug[5]: list of JSON file problems, ends above" 1>&2 - if [[ $EXIT_CODE -eq 0 ]]; then - EXIT_CODE="1" # exit 1 - echo "$0: Warning: EXIT_CODE set to: $EXIT_CODE" 1>&2 - fi -fi - - -# All Done!!! All Done!!! -- Jessica Noll, Age 2 -# -if [[ $EXIT_CODE -ne 0 ]]; then - echo "$0: Warning: about to exit non-zero: $EXIT_CODE" 1>&2 -elif [[ $V_FLAG -ge 1 ]]; then - echo "$0: debug[1]: about to exit $EXIT_CODE" 1>&2 -fi -exit "$EXIT_CODE" diff --git a/bin/unicode-fix.sed b/bin/unicode-fix.sed deleted file mode 100644 index fa2dd2c38..000000000 --- a/bin/unicode-fix.sed +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env sed -# -# bin/unicode-fix.sed - workaround to pre-translate certain JSON /u codes -# -# Until the jstrdecode(1) is fixed so that it properly converts 1 or 2 consecutive -# JSON string \uHexHexHexHex values into the proper binary symbol, we pre-translate -# JSON string \uHexHexHexHex values used in this web site as a workaround to -# avoid the jstrdecode(1) bug. We must do this as it's a more complicated bug -# and we wish to move along with the IOCCC. -# -# XXX - XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - XXX -# XXX - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - XXX -# XXX - until we have fixed jstrdecode, we must FAKE PARSE IOCCC JSON files - XXX -# XXX - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - GROSS HACK - XXX -# XXX - XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - XXX -# -# Until this bug is the jparse repo is fixed, we need to avoid -# using \\u[[:xdigit:]]{4} within JSON encoded strings. -# -# https://github.com/xexyl/jparse/issues/13 -# -# The above mentioned bug results in certain tools that are based on the -# jparse repo: -# -# https://github.com/xexyl/jparse -# -# including code from the mkiocccentry repo's jparse sub-directory: -# -# https://github.com/ioccc-src/mkiocccentry/tree/master/jparse -# -# We need to avoid using \\u[[:xdigit:]]{4} within JSON encoded strings. -# Sorry (tm Canada :-)). -# -# The following sed script converts known problematic \\u[[:xdigit:]]{4} -# combinations into their RAW character form: -# -# bin/unicode-fix.sed -# -# Use of that sed script serves as a workaround to avoid to avoid any -# problematic \\u[[:xdigit:]]{4} combinations in JSON encoded strings. -# -# For information on the jstrdecode(1) bug, see: -# -# https://github.com/xexyl/jparse/issues/13 -# -# usage: -# -# cmd | sed -f bin/unicode-fix.sed -# -# sed -f bin/unicode-fix.sed file > output -# -# sed -i -f bin/unicode-fix.sed file -# -# VERSION="1.0 2024-09-23" -# -s/\\u00c4/Ä/g -s/\\u00e1/á/g -s/\\u00eb/ë/g -s/\\u00fd/ý/g -s/\\u0159/ř/g -s/\\u2019/’/g