Cody Boone Ferguson
205fa25ced
This is a HUGE commit and this task is not done. There are still some
changes that might need to be discussed and since so much is done here I
think it's better to let this go through and then make sure everything
is okay. When the remaining references are discussed they can also be
changed.
I did NOT include changes in the tmp/OLD_manifest directory.
Some of the changes in this commit (it might be exhaustive but it might
not be - not sure):
- In the Makefiles the WINNERS was renamed AUTHORS.
- References to winners.html were changed to authors.html.
- The FAQ named anchor 'fix_a_winner' was renamed 'fix_an_entry'.
- The JSON fields (or whatever they're called) 'winner_id' and
'winner_JSON_format_version' were changed so that 'winner' is 'entry'.
- In scripts for entries 'IOCCC winner' was changed to 'IOCCC entry'.
- 'winners_text' is now 'entries_text'. It might be better to be
'entry_text' as it's per entry but this was not noticed at first. Since
this is easy to fix I am holding off on this for the meantime.
- The script tmp/update-manifest.sh now has the commit log messages
changed to the correct terms as well.
The index.html files were updated as well with any of the above
changes.
There are some references that should NOT be updated. Two examples
caught (and maybe more have slipped through that will have to be changed
back): in 1984/anonymous/README.md the author wrote back to Landon after
being told about the famous tattoo and they used the words:
.. my entry will be the first IOCCC winner ..
but since this comes from the author it should remain so.
The other one is that winners.html should still refer to winners.html as
it informs people that it's been moved to authors.html (it does redirect
too but even so if it was changed it would defeat the purpose of the
message).
With paths like authors.html I tried to copy paste so that hopefully
there are no typos. It is possible that I did not end up doing this with
all but these can be fixed when discovered.
430 lines
12 KiB
Bash
Executable File
430 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# fix_csv - fix the exported CSV files, form path_list.* files
|
|
#
|
|
# When Apple numbers exports a CSV file from manifest.numbers,
|
|
# and exports a CSV file from missing-manifest.numbers,
|
|
# the CSV file lines are separated by carriage returns (\015)
|
|
# and is not terminated by a newline. This script will "correct"
|
|
# these issues.
|
|
#
|
|
# XXX - This is a temporary utility that will be removed when
|
|
# the .entry.json files are built
|
|
|
|
# setup
|
|
#
|
|
export MANIFEST_CSV="manifest.csv"
|
|
export TMP_CSV
|
|
|
|
if [[ ! -f $MANIFEST_CSV ]]; then
|
|
echo "$0: ERROR: MANIFEST_CSV is missing: $MANIFEST_CSV" 1>&2
|
|
exit 10
|
|
fi
|
|
if [[ ! -s $MANIFEST_CSV ]]; then
|
|
echo "$0: ERROR: MANIFEST_CSV is empty: $MANIFEST_CSV" 1>&2
|
|
exit 11
|
|
fi
|
|
|
|
# setup temp manifest CSV file
|
|
#
|
|
TMP_CSV="tmp.$$.$MANIFEST_CSV"
|
|
rm -f "$TMP_CSV"
|
|
if [[ -e $TMP_CSV ]]; then
|
|
echo "$0: ERROR: TMP_CSV cannot be removed: $TMP_CSV" 1>&2
|
|
exit 12
|
|
fi
|
|
|
|
# remove carriage returns in manifest CSV file
|
|
#
|
|
trap 'rm -f $TMP_CSV; exit' 0 1 2 3 15
|
|
tr -d '\015' < "$MANIFEST_CSV" > "$TMP_CSV"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
printf "$0: ERROR: TMP_CSV: tr -d '\\\015' < %s > %s failed, error: %s\n" "$MANIFEST_CSV" "$TMP_CSV" "$status" 1>&2
|
|
exit 13
|
|
fi
|
|
if [[ ! -s $TMP_CSV ]]; then
|
|
printf "$0: ERROR: TMP_CSV: tr -d '\\\015' < %s > %s produced an empty file" "$MANIFEST_CSV" "$TMP_CSV" 1>&2
|
|
exit 14
|
|
fi
|
|
|
|
# sanity check manifest CSV
|
|
#
|
|
|
|
# check for double whitespace
|
|
#
|
|
trap 'rm -f $TMP_CSV $TMP_CHECK; exit' 0 1 2 3 15
|
|
TMP_CHECK="tmp.$$.check"
|
|
rm -f "$TMP_CHECK"
|
|
grep -E '[[:space:]][[:space:]]' "$TMP_CSV" > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found multiple consecutive whitespace in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: consecutive whitespace lines from $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: consecutive whitespace lines from $MANIFEST_CSV ends above" 1>&2
|
|
exit 15
|
|
fi
|
|
|
|
# check for leading whitespace
|
|
#
|
|
rm -f "$TMP_CHECK"
|
|
grep -E ',[[:space:]]|^[[:space:]]' "$TMP_CSV" > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found leading whitespace in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: leading whitespace lines from $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: leading whitespace lines from $MANIFEST_CSV ends above" 1>&2
|
|
exit 16
|
|
fi
|
|
|
|
# check for trailing whitespace
|
|
#
|
|
rm -f "$TMP_CHECK"
|
|
grep -E '[[:space:]],|[[:space:]]$' "$TMP_CSV" > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found trailing whitespace in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: trailing whitespace lines from $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: trailing whitespace lines from $MANIFEST_CSV ends above" 1>&2
|
|
exit 17
|
|
fi
|
|
|
|
# check for duplicate year,dir,file_path
|
|
#
|
|
rm -f "$TMP_CHECK"
|
|
sed -e 's/^\([^,]*,[^,]*,[^,]*\).*/\1/' "$TMP_CSV" | sort | uniq -d > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found duplicated year,dir,file_path in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: duplicated year,dir,file_path in $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: duplicated year,dir,file_path in $MANIFEST_CSV ends above" 1>&2
|
|
exit 18
|
|
fi
|
|
|
|
# check for ,TRUE,
|
|
#
|
|
rm -f "$TMP_CHECK"
|
|
grep -E ',TRUE,|,TRUE$' "$TMP_CSV" > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found ,TRUE, instead of ,true, in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: lines with ,TRUE, $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: lines with ,TRUE, $MANIFEST_CSV ends above" 1>&2
|
|
exit 19
|
|
fi
|
|
|
|
# check for ,FALSE,
|
|
#
|
|
rm -f "$TMP_CHECK"
|
|
grep -E ',FALSE,|,FALSE$' "$TMP_CSV" > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found ,FALSE, instead of ,false, in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: lines with ,FALSE, $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: lines with ,FALSE, $MANIFEST_CSV ends above" 1>&2
|
|
exit 20
|
|
fi
|
|
|
|
# check for ,null,
|
|
#
|
|
rm -f "$TMP_CHECK"
|
|
grep -E ',null,|.,null$' "$TMP_CSV" > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found ,null, in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: lines with ,null, $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: lines with ,null, $MANIFEST_CSV ends above" 1>&2
|
|
exit 21
|
|
fi
|
|
rm -f "$TMP_CHECK"
|
|
|
|
# check for ,NULL,
|
|
#
|
|
rm -f "$TMP_CHECK"
|
|
grep -E ',NULL,|,NULL$' "$TMP_CSV" > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found ,NULL, in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: lines with ,NULL, $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: lines with ,NULL, $MANIFEST_CSV ends above" 1>&2
|
|
exit 22
|
|
fi
|
|
rm -f "$TMP_CHECK"
|
|
|
|
# fix manifest CSV
|
|
#
|
|
|
|
# sort manifest CSV file
|
|
#
|
|
# sort by: year dir file_path inventory_order everything_else
|
|
#
|
|
sort -t, -k1,1 -k2d,2 -k4g,4 -k3d,3 -k5d,8 "$TMP_CSV" -o "$TMP_CSV"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
printf "$0: ERROR: sort -t, -k1,1 -k2d,2 -k4n,4 -k3d,3 -k5d,8 %s -o %s failed, error: %s\n" "$TMP_CSV" "$TMP_CSV" "$status" 1>&2
|
|
exit 23
|
|
fi
|
|
|
|
# append newline if last line of manifest CSV file if not empty
|
|
#
|
|
LAST_CHAR=$(tail -c 1 "$TMP_CSV")
|
|
if [[ -n $LAST_CHAR ]]; then
|
|
echo "" >> "$TMP_CSV"
|
|
fi
|
|
|
|
# check for empty cells
|
|
#
|
|
rm -f "$TMP_CHECK"
|
|
grep -E ',,|,$' "$TMP_CSV" > "$TMP_CHECK"
|
|
if [[ -s $TMP_CHECK ]]; then
|
|
echo "$0: ERROR: found ,empty cell in $MANIFEST_CSV" 1>&2
|
|
echo "$0: Warning: lines with empty cell $MANIFEST_CSV starts below" 1>&2
|
|
cat "$TMP_CHECK" 1>&2
|
|
echo "$0: Warning: lines with empty cell $MANIFEST_CSV ends above" 1>&2
|
|
exit 24
|
|
fi
|
|
rm -f "$TMP_CHECK"
|
|
|
|
# replace manifest CSV file if different
|
|
#
|
|
if ! cmp -s "$TMP_CSV" "$MANIFEST_CSV"; then
|
|
|
|
# replace CSV file with a better formatted version
|
|
#
|
|
echo "$0: updating $MANIFEST_CSV"
|
|
mv -f -v "$TMP_CSV" "$MANIFEST_CSV"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "$0: ERROR: mv -f -v $TMP_CSV $MANIFEST_CSV failed, error: $status" 1>&2
|
|
exit 25
|
|
fi
|
|
|
|
else
|
|
|
|
# CSV file is OK, remove the tmp file
|
|
#
|
|
rm -f "$TMP_CSV"
|
|
if [[ -e $TMP_CSV ]]; then
|
|
echo "$0: ERROR: TMP_CSV cannot be removed for cleanup: $TMP_CSV" 1>&2
|
|
exit 26
|
|
fi
|
|
fi
|
|
|
|
# setup the full manifest path list
|
|
#
|
|
TMP_CSV="tmp.$$.$MANIFEST_CSV"
|
|
sed -e '/^# -/d' "$MANIFEST_CSV" |
|
|
sort -u |
|
|
sort -t, -k1,1 -k2d,2 -k4g,4 -k3d,3 -k5d,10 > "$TMP_CSV"
|
|
if [[ ! -s $TMP_CSV ]]; then
|
|
echo "$0: ERROR: sorted $MANIFEST_CSV is empty" 1>&2
|
|
exit 27
|
|
fi
|
|
if ! cmp -s "$TMP_CSV" "$MANIFEST_CSV"; then
|
|
|
|
# replace full CSV file if different
|
|
#
|
|
echo "$0: updating $MANIFEST_CSV"
|
|
mv -f -v "$TMP_CSV" "$MANIFEST_CSV"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "$0: ERROR: mv -f -v $TMP_CSV $MANIFEST_CSV failed, error: $status" 1>&2
|
|
exit 28
|
|
fi
|
|
|
|
else
|
|
|
|
# fill CSV file is OK, remove the tmp file
|
|
#
|
|
rm -f "$TMP_CSV"
|
|
if [[ -e $TMP_CSV ]]; then
|
|
echo "$0: ERROR: TMP_CSV cannot be removed for cleanup: $TMP_CSV" 1>&2
|
|
exit 29
|
|
fi
|
|
fi
|
|
|
|
# build required file path list
|
|
#
|
|
export REQUIRED_PATH_LIST="path_list.required.txt"
|
|
export TMP_FILE="tmp.$$.$REQUIRED_PATH_LIST"
|
|
trap 'rm -f $TMP_FILE; exit' 0 1 2 3 15
|
|
awk -F, '$1 !~ "^#" && ( $5 == "true" || $8 == "original source" ) { print $1 "/" $2 "/" $3; }' "$MANIFEST_CSV" | sort -t/ > "$TMP_FILE"
|
|
if [[ ! -s $TMP_FILE ]]; then
|
|
echo "$0: ERROR: TMP_FILE, for required file list, is empty: $TMP_CSV" 1>&2
|
|
exit 30
|
|
fi
|
|
if ! cmp -s "$TMP_FILE" "$REQUIRED_PATH_LIST"; then
|
|
|
|
# replace required file
|
|
#
|
|
echo "$0: updating $REQUIRED_PATH_LIST"
|
|
mv -f -v "$TMP_FILE" "$REQUIRED_PATH_LIST"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "$0: ERROR: mv -f -v $TMP_FILE $REQUIRED_PATH_LIST failed, error: $status" 1>&2
|
|
exit 31
|
|
fi
|
|
|
|
else
|
|
|
|
# required file list is OK, remove the tmp file
|
|
#
|
|
rm -f "$TMP_FILE"
|
|
if [[ -e $TMP_CSV ]]; then
|
|
echo "$0: ERROR: TMP_FILE cannot be removed for cleanup: $TMP_FILE" 1>&2
|
|
exit 32
|
|
fi
|
|
fi
|
|
|
|
# build the built file path list
|
|
#
|
|
export BUILT_PATH_LIST="path_list.built.txt"
|
|
export TMP_FILE="tmp.$$.$BUILT_PATH_LIST"
|
|
trap 'rm -f $TMP_FILE; exit' 0 1 2 3 15
|
|
awk -F, '$1 !~ "^#" && $5 == "false" && $8 != "original source" { print $1 "/" $2 "/" $3; }' "$MANIFEST_CSV" | sort -t/ > "$TMP_FILE"
|
|
if [[ ! -s $TMP_FILE ]]; then
|
|
echo "$0: ERROR: TMP_FILE, for built file list, is empty: $TMP_CSV" 1>&2
|
|
exit 33
|
|
fi
|
|
if ! cmp -s "$TMP_FILE" "$BUILT_PATH_LIST"; then
|
|
|
|
# replace built file
|
|
#
|
|
echo "$0: updating $BUILT_PATH_LIST"
|
|
mv -f -v "$TMP_FILE" "$BUILT_PATH_LIST"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "$0: ERROR: mv -f -v $TMP_FILE $BUILT_PATH_LIST failed, error: $status" 1>&2
|
|
exit 34
|
|
fi
|
|
|
|
else
|
|
|
|
# built file list is OK, remove the tmp file
|
|
#
|
|
rm -f "$TMP_FILE"
|
|
if [[ -e $TMP_CSV ]]; then
|
|
echo "$0: ERROR: TMP_FILE cannot be removed for cleanup: $TMP_FILE" 1>&2
|
|
exit 35
|
|
fi
|
|
fi
|
|
|
|
# build manifest file list
|
|
#
|
|
export MANIFEST_LIST="path_list.manifest.txt"
|
|
export TMP_FILE="tmp.$$.$BUILT_PATH_LIST"
|
|
trap 'rm -f $TMP_FILE; exit' 0 1 2 3 15
|
|
cat "$REQUIRED_PATH_LIST" "$BUILT_PATH_LIST" | sort -t/ > "$TMP_FILE"
|
|
if [[ ! -s $TMP_FILE ]]; then
|
|
echo "$0: ERROR: TMP_FILE, for manifest file list, is empty: $TMP_CSV" 1>&2
|
|
exit 36
|
|
fi
|
|
if ! cmp -s "$TMP_FILE" "$MANIFEST_LIST"; then
|
|
|
|
# replace manifest file
|
|
#
|
|
echo "$0: updating $MANIFEST_LIST"
|
|
mv -f -v "$TMP_FILE" "$MANIFEST_LIST"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "$0: ERROR: mv -f -v $TMP_FILE $MANIFEST_LIST failed, error: $status" 1>&2
|
|
exit 37
|
|
fi
|
|
|
|
else
|
|
|
|
# manifest file list is OK, remove the tmp file
|
|
#
|
|
rm -f "$TMP_FILE"
|
|
if [[ -e $TMP_CSV ]]; then
|
|
echo "$0: ERROR: TMP_FILE cannot be removed for cleanup: $TMP_FILE" 1>&2
|
|
exit 38
|
|
fi
|
|
fi
|
|
|
|
# fix other CSV files
|
|
#
|
|
# We will fix author.csv, author_wins.csv, and year_prize.csv
|
|
#
|
|
for csv in author.csv author_wins.csv year_prize.csv; do
|
|
|
|
# verify that a CSV file exists
|
|
#
|
|
if [[ ! -f $csv ]]; then
|
|
echo "$0: ERROR: CSV is missing: $csv" 1>&2
|
|
exit 39
|
|
fi
|
|
if [[ ! -s $csv ]]; then
|
|
echo "$0: ERROR: CSV is empty: $csv" 1>&2
|
|
exit 40
|
|
fi
|
|
|
|
# setup temp CSV file
|
|
#
|
|
TMP_CSV="tmp.$$.$csv"
|
|
rm -f "$TMP_CSV"
|
|
if [[ -e $TMP_CSV ]]; then
|
|
echo "$0: ERROR: TMP_CSV cannot be removed: $TMP_CSV" 1>&2
|
|
exit 41
|
|
fi
|
|
|
|
# remove carriage returns in CSV file
|
|
#
|
|
trap 'rm -f $TMP_CSV; exit' 0 1 2 3 15
|
|
tr -d '\015' < "$csv" > "$TMP_CSV"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
printf "$0: ERROR: TMP_CSV: tr -d '\\\015' < %s > %s failed, error: %s\n" "$csv" "$TMP_CSV" "$status" 1>&2
|
|
exit 42
|
|
fi
|
|
if [[ ! -s $TMP_CSV ]]; then
|
|
printf "$0: ERROR: TMP_CSV: tr -d '\\\015' < %s > %s produced an empty file" "$csv" "$TMP_CSV" 1>&2
|
|
exit 43
|
|
fi
|
|
|
|
# sort CSV file
|
|
#
|
|
sort -t, -k1d,1 -k2d,2 "$TMP_CSV" -o "$TMP_CSV"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
printf "$0: ERROR: sort -t, -k1d,1 -k2d,2 %s -o %s failed, error: %s\n" "$TMP_CSV" "$TMP_CSV" "$status" 1>&2
|
|
exit 44
|
|
fi
|
|
|
|
# append newline if last line of CSV file if not empty
|
|
#
|
|
LAST_CHAR=$(tail -c 1 "$TMP_CSV")
|
|
if [[ -n $LAST_CHAR ]]; then
|
|
echo "" >> "$TMP_CSV"
|
|
fi
|
|
|
|
# replace CSV file if different
|
|
#
|
|
if ! cmp -s "$TMP_CSV" "$csv"; then
|
|
|
|
# replace CSV file with a better formatted version
|
|
#
|
|
echo "$0: updating $csv"
|
|
mv -f -v "$TMP_CSV" "$csv"
|
|
status="$?"
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "$0: ERROR: mv -f -v $TMP_CSV $csv failed, error: $status" 1>&2
|
|
exit 45
|
|
fi
|
|
|
|
else
|
|
|
|
# CSV file is OK, remove the tmp file
|
|
#
|
|
rm -f "$TMP_CSV"
|
|
if [[ -e $TMP_CSV ]]; then
|
|
echo "$0: ERROR: TMP_CSV cannot be removed for cleanup: $TMP_CSV" 1>&2
|
|
exit 46
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# All Done!!! -- Jessica Noll, Age 2
|
|
#
|
|
exit 0
|