use jsp tool, workaround JSON Unicode bugs, misc fixes

If the [jsp tool](https://github.com/kjozsa/jsp) is installed, it will be used in favor of the [JSONPath.sh tool](https://github.com/lcn2/JSONPath.sh). Updated `bin/jfmt-wrapper.sh`, `bin/jprint-wrapper.sh`, and `bin/jval-wrapper.sh` to use jsp if found. These 3 tools may be given `-t tool` where tool is either "_jsp_" or "_JSONPath.sh_" for force the use of the tool. They also use `-Q` to perform a quick check tool using trivial input. The `bin/jval-wrapper.sh` tool had been simplified. The `-w`, `-i` and `-u` options are gone. Instead it has: ``` -b print only values (def: print both JSON member and JSON value) -q remove enclosing double quotes (def: keep any enclosing double quotes) ``` Add `make all_jfmt` to Makefile to canonically format all entry and author handle JSON files. A sed script `bin/unicode-fix.sed` has been added that converts JSON string "_\u[0-9A-Fa-f]{4}_" encoding that may arise from known JSON data. This is done as a work-a-round for [Unicode / UTF-8 issues](https://github.com/xexyl/jparse/issues/13) with `jstrdecode(1)`. The `bin/unicode-chk.sh` tool, if given on file arguments, scans all `.entry.json` and `author/author_handle.json` JSON files to be sure that, after `bin/unicode-fix.sed` processing, no JSON string "_\u[0-9A-Fa-f]{4}_" encodings remain. Removed all `-j jparse` args from commands. Instead the `jparse(1)` tool is looked for along $PATH. Added `combine_author_handle.sh` tool to transform all of the `author/author_handle.json` files into a temporary as a single JSON file so that `bin/jval-wrapper.sh` tool may be used to extract all "_sort_word_" or all "_entry_id_" values. Changed `bin/all-jfmt.sh` VERSION from "1.0 2024-08-31" to "1.1 2024-09-24". Changed `bin/csv2entry.sh` VERSION from "1.2.3 2024-08-30" to "1.3 2024-09-23". Changed `bin/cvt-submission.sh` VERSION from "1.1 2024-08-31" to "1.2 2024-09-23". Changed `bin/entry2csv.sh` VERSION from "1.2.1 2024-08-17" to "1.3 2024-09-23". Changed `bin/gen-authors.sh` VERSION from "1.12 2024-08-18" to "1.13 2024-09-23". Changed `bin/gen-location.sh` VERSION from "1.6 2024-08-18" to "1.7 2024-09-23". Changed `bin/gen-status.sh` VERSION from "1.6 2024-08-17" to "1.7 2024-09-23". Changed `bin/gen-years.sh` VERSION from "1.4 2024-08-17" to "1.5 2024-09-23". Changed `bin/jfmt-wrapper.sh` VERSION from "1.1 2024-08-20" to "1.2 2024-09-23". Changed `bin/jprint-wrapper.sh` VERSION from "1.1 2024-08-20" to "1.3 2024-09-23". Changed `bin/jval-wrapper.sh` VERSION from "1.0.4 2024-08-20" to "1.1 2024-09-23". Changed `bin/manifest.entry.json.awk` VERSION from "1.1.2 2024-07-28" to "1.2 2024-09-17". Changed `bin/output-index-author.sh` VERSION from "1.8 2024-08-17" to "1.9 2024-09-23". Changed `bin/output-year-index.sh` VERSION from "1.4.4 2024-08-05" to "1.5 2024-09-23". Changed `bin/subst.entry-index.sh` VERSION from "1.5 2024-08-13" to "1.6 2024-09-23". Added `bin/combine_author_handle.sh` VERSION "1.0 2024-09-23". Added `bin/unicode-chk.sh` VERSION "1.0 2024-09-23". Added `bin/unicode-fix.sed` VERSION "1.0 2024-09-23". Perform `make www` to perform the above and test under macOS. FYI: On our MacBook Pro, `make www` took: ``` make www 631.79s user 264.95s system 94% cpu 15:53.71 total ``` FYI: On our MacBook Pro, `make quick_www` took: ``` make quick_www 81.43s user 58.26s system 102% cpu 2:16.02 total ```
2024-09-24 04:04:27 -07:00
1 parent e84d6eb095
commit 499be5983d
@@ -104,7 +104,7 @@ shopt -s globstar	# enable ** to match all files and zero or more directories an

 # set variables referenced in the usage message
 #
-export VERSION="1.8 2024-08-17"
+export VERSION="1.9 2024-09-23"
 NAME=$(basename "$0")
 export NAME
 export V_FLAG=0
@@ -188,6 +188,7 @@ export DO_NOT_PROCESS=
 function output_author_handles
 {
    local ENTRY_JSON_PATH;	# the .entry.json path
+    local PATTERN;		# XPath for JSON pattern

    # parse args
    #
@@ -211,10 +212,11 @@ function output_author_handles

    # extract author handles from .entry.json
    #
-    "$JVAL_WRAPPER" -w -b "$ENTRY_JSON_PATH" '$..author_handle'
+    PATTERN='$..author_handle'
+    "$JVAL_WRAPPER" -b -q "$ENTRY_JSON_PATH" "$PATTERN"
    status="$?"
    if [[ $status -ne 0 ]]; then
-	echo  "$0: ERROR: $JVAL_WRAPPER -w -b $ENTRY_JSON_PATH \$..author_handle failed," \
+	echo  "$0: ERROR: in output_author_handles: $JVAL_WRAPPER -b -q $ENTRY_JSON_PATH '$PATTERN' failed," \
 	      "error: $status" 1>&2
 	return 5
    fi
@@ -236,6 +238,7 @@ function output_author_handles
 function output_entry_ids
 {
    local AUTHOR_HANDLE_JSON_PATH;	# the .entry.json path
+    local PATTERN;		# XPath for JSON pattern

    # parse args
    #
@@ -259,10 +262,11 @@ function output_entry_ids

    # extract entry ids from the author/author_handle.json file
    #
-    "$JVAL_WRAPPER" -w -b "$AUTHOR_HANDLE_JSON_PATH" '$..entry_id'
+    PATTERN='$..entry_id'
+    "$JVAL_WRAPPER" -b -q "$AUTHOR_HANDLE_JSON_PATH" "$PATTERN"
    status="$?"
    if [[ $status -ne 0 ]]; then
-	echo  "$0: ERROR: $JVAL_WRAPPER -w -b $AUTHOR_HANDLE_JSON_PATH \$..entry_id failed," \
+	echo  "$0: ERROR: in output_entry_ids: $JVAL_WRAPPER -b -q $AUTHOR_HANDLE_JSON_PATH '$PATTERN' failed," \
 	      "error: $status" 1>&2
 	return 5
    fi
@@ -283,6 +287,7 @@ function output_entry_ids
 function output_full_name
 {
    local AUTHOR_HANDLE_JSON_PATH;	# the .entry.json path
+    local PATTERN;		# XPath for JSON pattern

    # parse args
    #
@@ -306,11 +311,12 @@ function output_full_name

    # extract Full Name from the author/author_handle.json file
    #
-    "$JVAL_WRAPPER" -w -b "$AUTHOR_HANDLE_JSON_PATH" '$..full_name'
-    status="$?"
-    if [[ $status -ne 0 ]]; then
-	echo  "$0: ERROR: $JVAL_WRAPPER -w -b $AUTHOR_HANDLE_JSON_PATH \$..full_name failed," \
-	      "error: $status" 1>&2
+    PATTERN='$..full_name'
+    "$JVAL_WRAPPER" -b -q "$AUTHOR_HANDLE_JSON_PATH" "$PATTERN" | sed -f "$UNICODE_FIX_SED"
+    status_codes=("${PIPESTATUS[@]}")
+    if [[ ${status_codes[*]} =~ [1-9] ]]; then
+	echo  "$0: ERROR: in output_full_name: $JVAL_WRAPPER -b -q $AUTHOR_HANDLE_JSON_PATH '$PATTERN' failed," \
+	      "error codes: ${status_codes[*]}" 1>&2
 	return 5
    fi
    return 0
@@ -331,6 +337,7 @@ function output_full_name
 function output_location_code
 {
    local AUTHOR_HANDLE_JSON_PATH;	# the .entry.json path
+    local PATTERN;		# XPath for JSON pattern

    # parse args
    #
@@ -354,10 +361,11 @@ function output_location_code

    # extract Location Code from the author/author_handle.json file
    #
-    "$JVAL_WRAPPER" -w -b "$AUTHOR_HANDLE_JSON_PATH" '$..location_code'
+    PATTERN='$..location_code'
+    "$JVAL_WRAPPER" -b -q "$AUTHOR_HANDLE_JSON_PATH" "$PATTERN"
    status="$?"
    if [[ $status -ne 0 ]]; then
-	echo  "$0: ERROR: $JVAL_WRAPPER -w -b $ENTRY_JSON_PATH \$..location_code failed," \
+	echo  "$0: ERROR: in output_location_code: $JVAL_WRAPPER -b -q $ENTRY_JSON_PATH '$PATTERN' failed," \
 	      "error: $status" 1>&2
 	return 5
    fi
@@ -638,6 +646,23 @@ if [[ ! -x $LOCATION_TOOL ]]; then
 fi


+# verify that the bin/unicode-fix.sed tool is executable
+#
+export UNICODE_FIX_SED="$BIN_DIR/unicode-fix.sed"
+if [[ ! -e $UNICODE_FIX_SED ]]; then
+    echo  "$0: ERROR: bin/unicode-fix.sed does not exist: $UNICODE_FIX_SED" 1>&2
+    exit 5
+fi
+if [[ ! -f $UNICODE_FIX_SED ]]; then
+    echo  "$0: ERROR: bin/unicode-fix.sed is not a regular file: $UNICODE_FIX_SED" 1>&2
+    exit 5
+fi
+if [[ ! -r $UNICODE_FIX_SED ]]; then
+    echo  "$0: ERROR: bin/unicode-fix.sed is not an readable file: $UNICODE_FIX_SED" 1>&2
+    exit 5
+fi
+
+
 # parameter debugging
 #
 if [[ $V_FLAG -ge 3 ]]; then
@@ -673,6 +698,7 @@ if [[ $V_FLAG -ge 3 ]]; then
    echo "$0: debug[3]: DOT_PATH_CONTENT=$DOT_PATH_CONTENT" 1>&2
    echo "$0: debug[3]: ENTRY_JSON=$ENTRY_JSON" 1>&2
    echo "$0: debug[3]: LOCATION_TOOL=$LOCATION_TOOL" 1>&2
+    echo "$0: debug[3]: UNICODE_FIX_SED=$UNICODE_FIX_SED" 1>&2
 fi