Add bin/find-invalid-json.sh to validate JSON
This script is run from 'make test' too so that if someone changes a JSON file and it is invalid the GitHub workflow will report an error. This script is based on bin/find-missing-links.sh. It can be cleaned up with exit codes (and consistency checks need to be made) but other issues have higher priorities so I will leave this the way it is for now. Requires verge to be installed and the jparse version to be 1.2.0 or greater.
This commit is contained in:
Cody Boone Ferguson committed
1 parent
12b8cfd108
commit
b36b7a9a4c
2 files changed
+251
-1
No files matched your search
@@ -67,6 +67,7 @@ GEN_STATUS= bin/gen-status.sh
|
||||
GEN_SITEMAP= bin/gen-sitemap.sh
|
||||
SORT_GITIGNORE= bin/sort.gitignore.sh
|
||||
FIND_MISSING_LINKS= bin/find-missing-links.sh
|
||||
FIND_INVALID_JSON= bin/find-invalid-json.sh
|
||||
CSV2ENTRY= bin/csv2entry.sh
|
||||
ENTRY2CSV= bin/entry2csv.sh
|
||||
|
||||
@@ -278,7 +279,7 @@ clobber:
|
||||
|
||||
.PHONY: help genpath genfilelist all_jfmt verify_entry_files gen_authors quick_authors \
|
||||
gen_location quick_location gen_years find_missing_links test entry_index gen_top_html \
|
||||
thanks gen_other_html quick_other_html quick_entry_index \
|
||||
thanks gen_other_html quick_other_html quick_entry_index find_invalid_json \
|
||||
gen_year_index quick_year_index quick_www www untar_entry_tarball untar_year_tarball \
|
||||
form_entry_tarball form_year_tarball tar gen_status gen_sitemap \
|
||||
sitemap timestamp update csv2entry entry2csv
|
||||
@@ -326,6 +327,7 @@ help:
|
||||
@echo 'make gen_next ;: generate the HTML files in next/'
|
||||
@echo 'make quick_entry_index ;: build winner index.html files that might be out of date'
|
||||
@echo 'make find_missing_links ;: find markdown links to missing local files'
|
||||
@echo 'make find_invalid_json ;; find invalid JSON files'
|
||||
@echo
|
||||
@echo 'make test ;: summary of mostly harmless tests'
|
||||
@echo
|
||||
@@ -565,6 +567,14 @@ find_missing_links:
|
||||
${FIND_MISSING_LINKS} -v 1
|
||||
@echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-='
|
||||
|
||||
# find invalid JSON files
|
||||
#
|
||||
find_invalid_json:
|
||||
@echo '=-=-=-=-= IOCCC begin make $@ =-=-=-=-='
|
||||
${FIND_INVALID_JSON} -v 1
|
||||
@echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-='
|
||||
|
||||
|
||||
# convert author_wins.csv, manifest.csv and year_prize.csv CSV files to
|
||||
# .entry.json files.
|
||||
#
|
||||
@@ -598,6 +608,7 @@ test:
|
||||
${MAKE} genfilelist
|
||||
${MAKE} verify_entry_files
|
||||
${MAKE} find_missing_links
|
||||
${MAKE} find_invalid_json
|
||||
@echo '=-=-=-=-= IOCCC complete make $@ =-=-=-=-='
|
||||
|
||||
|
||||
|
||||
Executable
+239
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# find-invalid-json.sh - find invalid JSON files
|
||||
#
|
||||
# The JSON parser jparse was co-developed in 2022 by:
|
||||
#
|
||||
# @xexyl
|
||||
# https://xexyl.net Cody Boone Ferguson
|
||||
# https://ioccc.xexyl.net
|
||||
# and:
|
||||
# chongo (Landon Curt Noll, http://www.isthe.com/chongo/index.html) /\oo/\
|
||||
#
|
||||
# This script was written in 2024 by Cody Boone Ferguson.
|
||||
#
|
||||
# "Because sometimes even the IOCCC Judges need some help." :-)
|
||||
#
|
||||
# "Share and Enjoy!"
|
||||
# -- Sirius Cybernetics Corporation Complaints Division, JSON spec department. :-)
|
||||
|
||||
VERGE=$(type -P verge)
|
||||
export VERGE
|
||||
if [[ -z $VERGE ]]; then
|
||||
echo "$0: FATAL: verge is not installed or not in \$PATH" 1>&2
|
||||
echo "$0: notice: to install verge:" 1>&2
|
||||
echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2
|
||||
echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2
|
||||
echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2
|
||||
exit 5
|
||||
fi
|
||||
JPARSE=$(type -P jparse)
|
||||
export JPARSE
|
||||
if [[ -z $JPARSE ]]; then
|
||||
echo "$0: FATAL: jparse is not installed or not in \$PATH" 1>&2
|
||||
echo "$0: notice: to install jparse:" 1>&2
|
||||
echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2
|
||||
echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2
|
||||
echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2
|
||||
exit 5
|
||||
fi
|
||||
export MIN_JPARSE_VERSION="1.2.0"
|
||||
JPARSE_VERSION=$("$JPARSE" -V | head -1 | awk '{print $3;}')
|
||||
if ! "$VERGE" "$JPARSE_VERSION" "$MIN_JPARSE_VERSION"; then
|
||||
echo "$0: FATAL: jparse version: $JPARSE_VERSION < minimum version: $MIN_JPARSE_VERSION" 1>&2
|
||||
echo "$0: notice: consider updating jparse from mkiocccentry repo" 1>&2
|
||||
echo "$0: notice: run: git clone https://github.com/ioccc-src/mkiocccentry.git" 1>&2
|
||||
echo "$0: notice: then: cd mkiocccentry && make clobber all" 1>&2
|
||||
echo "$0: notice: then: cd jparse && sudo make install clobber" 1>&2
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# set variables referenced in the usage message
|
||||
#
|
||||
export VERSION="1.0.0 2024-10-14"
|
||||
NAME=$(basename "$0")
|
||||
export NAME
|
||||
export V_FLAG=0
|
||||
GIT_TOOL=$(type -P git)
|
||||
export GIT_TOOL
|
||||
if [[ -z "$GIT_TOOL" ]]; then
|
||||
echo "$0: FATAL: git tool is not installed or not in \$PATH" 1>&2
|
||||
exit 5
|
||||
fi
|
||||
"$GIT_TOOL" rev-parse --is-inside-work-tree >/dev/null 2>&1
|
||||
status="$?"
|
||||
if [[ $status -eq 0 ]]; then
|
||||
TOPDIR=$("$GIT_TOOL" rev-parse --show-toplevel)
|
||||
fi
|
||||
export TOPDIR
|
||||
|
||||
|
||||
# usage
|
||||
#
|
||||
export USAGE="usage: $0 [-h] [-v level] [-V] [-d topdir]
|
||||
|
||||
-h print help message and exit
|
||||
-v level set verbosity level (def level: 0)
|
||||
-V print version string and exit
|
||||
|
||||
-d topdir set topdir (def: $TOPDIR)
|
||||
|
||||
Exit codes:
|
||||
0 all OK
|
||||
1 invalid JSON file(s) found
|
||||
2 -h and help string printed or -V and version string printed
|
||||
3 command line error
|
||||
4 problems found with or in the topdir or topdir/YYYY directory
|
||||
5 problems found with or in the entry topdir/YYYY/dir directory
|
||||
>= 10 internal error
|
||||
|
||||
$NAME version: $VERSION"
|
||||
|
||||
# parse command line
|
||||
#
|
||||
while getopts :hv:Vd:nN flag; do
|
||||
case "$flag" in
|
||||
h) echo "$USAGE" 1>&2
|
||||
exit 2
|
||||
;;
|
||||
v) V_FLAG="$OPTARG"
|
||||
;;
|
||||
V) echo "$VERSION"
|
||||
exit 2
|
||||
;;
|
||||
d) TOPDIR="$OPTARG"
|
||||
;;
|
||||
\?) echo "$0: ERROR: invalid option: -$OPTARG" 1>&2
|
||||
echo 1>&2
|
||||
echo "$USAGE" 1>&2
|
||||
exit 3
|
||||
;;
|
||||
:) echo "$0: ERROR: option -$OPTARG requires an argument" 1>&2
|
||||
echo 1>&2
|
||||
echo "$USAGE" 1>&2
|
||||
exit 3
|
||||
;;
|
||||
*) echo "$0: ERROR: unexpected value from getopts: $flag" 1>&2
|
||||
echo 1>&2
|
||||
echo "$USAGE" 1>&2
|
||||
exit 3
|
||||
;;
|
||||
esac
|
||||
done
|
||||
#
|
||||
# remove the options
|
||||
#
|
||||
shift $(( OPTIND - 1 ));
|
||||
|
||||
#
|
||||
# verify arg count and parse args
|
||||
#
|
||||
export TOOL
|
||||
case "$#" in
|
||||
0) ;;
|
||||
*) echo "$0: ERROR: expected 0 args, found: $#" 1>&2
|
||||
echo "$USAGE" 1>&2
|
||||
exit 3
|
||||
;;
|
||||
esac
|
||||
#
|
||||
if [[ $V_FLAG -ge 1 ]]; then
|
||||
echo "$0: debug[1]: start scan for invalid JSON files" 1>&2
|
||||
fi
|
||||
|
||||
|
||||
# print running info if verbose
|
||||
#
|
||||
# If -v 3 or higher, print exported variables in order that they were exported.
|
||||
#
|
||||
if [[ $V_FLAG -ge 3 ]]; then
|
||||
echo "$0: debug[3]: VERSION=$VERSION" 1>&2
|
||||
echo "$0: debug[3]: NAME=$NAME" 1>&2
|
||||
echo "$0: debug[3]: V_FLAG=$V_FLAG" 1>&2
|
||||
echo "$0: debug[3]: GIT_TOOL=$GIT_TOOL" 1>&2
|
||||
echo "$0: debug[3]: TOPDIR=$TOPDIR" 1>&2
|
||||
fi
|
||||
|
||||
|
||||
# cd to topdir
|
||||
#
|
||||
if [[ ! -e $TOPDIR ]]; then
|
||||
echo "$0: ERROR: cannot cd to non-existent path: $TOPDIR" 1>&2
|
||||
exit 4
|
||||
fi
|
||||
if [[ ! -d $TOPDIR ]]; then
|
||||
echo "$0: ERROR: cannot cd to a non-directory: $TOPDIR" 1>&2
|
||||
exit 4
|
||||
fi
|
||||
export CD_FAILED
|
||||
if [[ $V_FLAG -ge 5 ]]; then
|
||||
echo "$0: debug[5]: about to: cd $TOPDIR" 1>&2
|
||||
fi
|
||||
cd "$TOPDIR" || CD_FAILED="true"
|
||||
if [[ -n $CD_FAILED ]]; then
|
||||
echo "$0: ERROR: cd $TOPDIR failed" 1>&2
|
||||
exit 5
|
||||
fi
|
||||
if [[ $V_FLAG -ge 3 ]]; then
|
||||
echo "$0: debug[3]: now in directory: $(/bin/pwd)" 1>&2
|
||||
fi
|
||||
|
||||
|
||||
# create a temporary invalid JSON error message file
|
||||
#
|
||||
export TMP_INVALID_JSON=".tmp.$NAME.INVALID_JSON.$$.tmp"
|
||||
if [[ $V_FLAG -ge 3 ]]; then
|
||||
echo "$0: debug[3]: temporary invalid JSON error message file: $TMP_INVALID_JSON" 1>&2
|
||||
fi
|
||||
if [[ -z $NOOP ]]; then
|
||||
trap 'rm -f $TMP_INVALID_JSON; exit' 0 1 2 3 15
|
||||
rm -f "$TMP_INVALID_JSON"
|
||||
if [[ -e $TMP_INVALID_JSON ]]; then
|
||||
echo "$0: ERROR: cannot remove temporary invalid JSON error message file: $TMP_INVALID_JSON" 1>&2
|
||||
exit 12
|
||||
fi
|
||||
:> "$TMP_INVALID_JSON"
|
||||
if [[ ! -e $TMP_INVALID_JSON ]]; then
|
||||
echo "$0: ERROR: cannot create temporary invalid JSON error message file: $TMP_INVALID_JSON" 1>&2
|
||||
exit 13
|
||||
fi
|
||||
elif [[ $V_FLAG -ge 3 ]]; then
|
||||
echo "$0: debug[3]: because of -n, temporary invalid JSON error message file is not used: $TMP_INVALID_JSON" 1>&2
|
||||
fi
|
||||
|
||||
|
||||
# collect JSON files
|
||||
find . -name '*.json' -print |
|
||||
# get name of each JSON file
|
||||
while read -r JSON_FILE; do
|
||||
# try parsing JSON file
|
||||
if ! "$JPARSE" "$JSON_FILE"; then
|
||||
echo "$JSON_FILE is invalid JSON"
|
||||
fi
|
||||
# this writes the invalid JSON files to the temporary error file
|
||||
done > "$TMP_INVALID_JSON"
|
||||
|
||||
# count the invalid JSON files
|
||||
#
|
||||
INVALID_JSON_COUNT=$(wc -l < "$TMP_INVALID_JSON" | tr -d ' ')
|
||||
export INVALID_JSON_COUNT
|
||||
((INVALID_JSON_COUNT=INVALID_JSON_COUNT/4))
|
||||
|
||||
|
||||
# report on any invalid JSON files
|
||||
#
|
||||
if [[ -s $TMP_INVALID_JSON ]]; then
|
||||
sed -e "s;^$TOPDIR/;;" "$TMP_INVALID_JSON" 1>&2
|
||||
fi
|
||||
|
||||
|
||||
# All Done!!! All Done!!! -- Jessica Noll, Age 2
|
||||
#
|
||||
if [[ -s $TMP_INVALID_JSON ]]; then
|
||||
echo "$0: ERROR: $INVALID_JSON_COUNT invalid JSON file(s) found" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ $V_FLAG -ge 1 ]]; then
|
||||
echo "$0: debug[1]: $INVALID_JSON_COUNT invalid JSON file(s) found" 1>&2
|
||||
fi
|
||||
exit 0
|
||||
Reference in new issue
Block a user