From 5f8b1a88414debe2ef8f3a93be07d221ece64ed3 Mon Sep 17 00:00:00 2001 From: Landon Curt Noll Date: Wed, 1 Nov 2023 01:43:57 -0700 Subject: [PATCH] rename fix_manifest_csv.sh -> fix_csv.sh, fix other CSV files The fix_csv.sh now fixes the other 3 CSV files: author.csv author_wins.csv year_prize.csv --- tmp/README.md | 59 ++++++++--------- tmp/author.csv | 10 +-- tmp/author_wins.csv | 4 +- tmp/{fix_manifest_csv.sh => fix_csv.sh} | 84 ++++++++++++++++++++++++- 4 files changed, 115 insertions(+), 42 deletions(-) rename tmp/{fix_manifest_csv.sh => fix_csv.sh} (74%) diff --git a/tmp/README.md b/tmp/README.md index e799df115..79f4ee534 100644 --- a/tmp/README.md +++ b/tmp/README.md @@ -210,15 +210,7 @@ via the [macOS](https://www.apple.com/macos/) 3. Click ((Export)) * 3a. If needed click ((Replace)) -4. open a terminal window - -5. cd to the `tmp` directory - -6. tr -d '\015' < author_wins.csv > tmp.csv - -7. echo >> tmp.csv - -8. mv -f tmp.csv author_wins.csv +4. Execute ./[fix_csv.sh](fix_csv.sh) ### [author.numbers](author.numbers) @@ -279,15 +271,7 @@ via the [macOS](https://www.apple.com/macos/) 3. Click ((Export)) * 3a. If needed click ((Replace)) -4. open a terminal window - -5. cd to the `tmp` directory - -6. tr -d '\015' < author.csv > tmp.csv - -7. echo >> tmp.csv - -8. mv -f tmp.csv author.csv +4. Execute ./[fix_csv.sh](fix_csv.sh) ### [author_handle.txt](author_handle.txt) @@ -361,10 +345,25 @@ Generate a fake index.html for a given winner. Generate a fake .inventory.html for a given winner. -### [fix_manifest_csv.sh](fix_manifest_csv.sh) +### [fix_csv.sh](fix_csv.sh) -This tool will fix the [manifest.csv](manifest.csv) by - removing carriage returns and appending a newline if it was missing. +This tool will fix CSV files exported by +[macOS](https://www.apple.com/macos) +[Numbers](https://www.apple.com/numbers/) by removing carriage +returns and appending a newline if it was missing. + +Fixing involves these actions: + +```sh +tr -d '\015' < input.csv > tmp.csv +echo >> tmp.csv +mv -f tmp.csv input.csv +``` + +This tool also forms [path_list.built.txt](path_list.built.txt), +[path_list.manifest.txt](path_list.manifest.txt), and +[path_list.required.txt](path_list.required.txt) from +[manifest.csv](manifest.csv). ### [gen_author_json.sh](gen_author_json.sh) @@ -514,7 +513,7 @@ via the [macOS](https://www.apple.com/macos/) 3. Click ((Export)) * 3a. If needed click ((Replace)) -4. execute ./[fix_manifest_csv.sh](fix_manifest_csv.sh) +4. Execute ./[fix_csv.sh](fix_csv.sh) ### [path_list.built.txt](path_list.built.txt) @@ -526,7 +525,7 @@ These paths are formed from [manifest.numbers](manifest.numbers) entries where the "created_by" (5th) field is null. See [manifest.numbers](manifest.numbers) for details on the fields. -This file is formed by the [fix_manifest_csv.sh](fix_manifest_csv.sh) tool. +This file is formed by the [fix_csv.sh](fix_csv.sh) tool. ### [path_list.found.txt](path_list.found.txt) @@ -542,7 +541,7 @@ This file is created by [gen_path_list.found.sh](gen_path_list.found.sh). This is the list of file paths, both primary and constructed files from winners. -This file is formed by the [fix_manifest_csv.sh](fix_manifest_csv.sh) tool. +This file is formed by the [fix_csv.sh](fix_csv.sh) tool. ### [path_list.required.txt](path_list.required.txt) @@ -554,7 +553,7 @@ These paths are formed from [manifest.numbers](manifest.numbers) entries where the "created_by" (5th) field is non-null. See [manifest.numbers](manifest.numbers) for details on the fields. -This file is formed by the [fix_manifest_csv.sh](fix_manifest_csv.sh) tool. +This file is formed by the [fix_csv.sh](fix_csv.sh) tool. ### [sql/winners.sql](sql/winners.sql) @@ -629,12 +628,4 @@ via the [macOS](https://www.apple.com/macos/) 3. Click ((Export)) * 3a. If needed click ((Replace)) -4. open a terminal window - -5. cd to the `tmp` directory - -6. tr -d '\015' < year_prize.csv > tmp.csv - -7. echo >> tmp.csv - -8. mv -f tmp.csv year_prize.csv +4. Execute ./[fix_csv.sh](fix_csv.sh) diff --git a/tmp/author.csv b/tmp/author.csv index 6192855e9..fae7f955b 100644 --- a/tmp/author.csv +++ b/tmp/author.csv @@ -1,4 +1,3 @@ -# sort_word,name,author_handle,url,email,location_code,deprecated_twitter,mastodon,alternate_url, # 0: lower case sort word,,,,,,,,, # 1: name,,,,,,,,, # 2: winner_handle,,,,,,,,, @@ -8,14 +7,15 @@ # 6: deprecated twitter handle,,,,,,,,, # 7: mastodon handle,,,,,,,,, # 8: alternate url,,,,,,,,, +# sort_word,name,author_handle,url,email,location_code,deprecated_twitter,mastodon,alternate_url, adamovsky,Ondřej Adamovský,Ondrej_Adamovsky,null,oa@cmail.cz,CZ,null,null,null, algmyr,Anton Älgmyr,Anton_Algmyr,algmyr@student.chalmers.se,@aalgmyr,SE,@aalgmyr,null,null, -anderson,Glyn Anderson,Glyn_Anderson,null,glyn@mataharistudios.com,null,null,null,null, anderson,Derek Anderson,Derek_Anderson,null,dereka9991@gmail.com,US,null,null,null, -anonymous,Anonymous 2015,Anonymous_2015,null,hirekokeshi@gmail.com,JP,@hirekoke,null,null, -anonymous,Anonymous 2005,Anonymous_2005,null,null,null,null,null,null, -anonymous,Anonymous 2004,Anonymous_2004,https://web.archive.org/web/20040324155828/http://bicoherent.topcities.com/,null,SG,null,null,null, +anderson,Glyn Anderson,Glyn_Anderson,null,glyn@mataharistudios.com,null,null,null,null, anonymous,Anonymous 1984,Anonymous_1984,null,null,US,null,null,null, +anonymous,Anonymous 2004,Anonymous_2004,https://web.archive.org/web/20040324155828/http://bicoherent.topcities.com/,null,SG,null,null,null, +anonymous,Anonymous 2005,Anonymous_2005,null,null,null,null,null,null, +anonymous,Anonymous 2015,Anonymous_2015,null,hirekokeshi@gmail.com,JP,@hirekoke,null,null, applegate,David Applegate,David_Applegate,http://www.caam.rice.edu/~davida,davida@caam.rice.edu,US,null,null,null, applin,Jack Applin,Jack_Applin,https://www.cs.colostate.edu/~applin/,neutron@fc.hp.com,US,null,null,null, arkko,Jari Arkko,Jari_Arkko,http://www.arkko.com,jari@arkko.com,null,null,null,null, diff --git a/tmp/author_wins.csv b/tmp/author_wins.csv index b715993a4..3cfc5b9fe 100644 --- a/tmp/author_wins.csv +++ b/tmp/author_wins.csv @@ -1,4 +1,4 @@ -t# author_handle,winner_id,winner_id,winner_id,…,,,,,,,,,,,,,,, +# author_handle,winner_id,winner_id,winner_id,…,,,,,,,,,,,,,,, Aaron_Grothe,2006_grothe,2012_grothe,,,,,,,,,,,,,,,,, Adar_Zeitak,2012_zeitak,,,,,,,,,,,,,,,,,, Adrian_Cable,2013_cable1,2013_cable2,2013_cable3,,,,,,,,,,,,,,,, @@ -133,9 +133,9 @@ Michael_H_Pawka,1986_pawka,,,,,,,,,,,,,,,,,, Michael_Savastio,1995_savastio,,,,,,,,,,,,,,,,,, Mike_Laman,1984_laman,,,,,,,,,,,,,,,,,, Moxen_N_Briddlebane,2000_briddlebane,,,,,,,,,,,,,,,,,, +NATORI_Shin,2000_natori,,,,,,,,,,,,,,,,,, Nathan_Otterness,2020_otterness,,,,,,,,,,,,,,,,,, Nathan_Sidwell,1992_nathan,,,,,,,,,,,,,,,,,, -NATORI_Shin,2000_natori,,,,,,,,,,,,,,,,,, Neil_Mix,1998_dloweneil,,,,,,,,,,,,,,,,,, Nicholas_Carlini,2020_carlini,,,,,,,,,,,,,,,,,, Nick_Johnson,2004_arachnid,,,,,,,,,,,,,,,,,, diff --git a/tmp/fix_manifest_csv.sh b/tmp/fix_csv.sh similarity index 74% rename from tmp/fix_manifest_csv.sh rename to tmp/fix_csv.sh index 6026d6821..aba933ee0 100755 --- a/tmp/fix_manifest_csv.sh +++ b/tmp/fix_csv.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# fix_numbers_csv - fix the exported manifest CSV file, form path_list.* files +# fix_csv - fix the exported CSV files, form path_list.* files # # When Apple numbers exports a CSV file from manifest.numbers, # and exports a CSV file from missing-manifest.numbers, @@ -224,6 +224,88 @@ else fi fi +# fix other CSV files +# +# We will fix author.csv, author_wins.csv, and year_prize.csv +# +for csv in author.csv author_wins.csv year_prize.csv; do + + # verify that a CSV file exists + # + if [[ ! -f $csv ]]; then + echo "$0: ERROR: CSV is missing: $csv" 1>&2 + exit 30 + fi + if [[ ! -s $csv ]]; then + echo "$0: ERROR: CSV is empty: $csv" 1>&2 + exit 31 + fi + + # setup temp CSV file + # + TMP_CSV="tmp.$$.$csv" + rm -f "$TMP_CSV" + if [[ -e $TMP_CSV ]]; then + echo "$0: ERROR: TMP_CSV cannot be removed: $TMP_CSV" 1>&2 + exit 32 + fi + + # remove carriage returns in CSV file + # + trap 'rm -f $TMP_CSV; exit' 0 1 2 3 15 + tr -d '\015' < "$csv" > "$TMP_CSV" + status="$?" + if [[ $status -ne 0 ]]; then + printf "$0: ERROR: TMP_CSV: tr -d '\\\015' < %s > %s failed, error: %s\n" "$csv" "$TMP_CSV" "$status" 1>&2 + exit 33 + fi + if [[ ! -s $TMP_CSV ]]; then + printf "$0: ERROR: TMP_CSV: tr -d '\\\015' < %s > %s produced an empty file" "$csv" "$TMP_CSV" 1>&2 + exit 34 + fi + + # sort CSV file + # + sort -t, -k1d,1 -k2d,2 "$TMP_CSV" -o "$TMP_CSV" + status="$?" + if [[ $status -ne 0 ]]; then + printf "$0: ERROR: sort -t, -k1d,1 -k2d,2 %s -o %s failed, error: %s\n" "$TMP_CSV" "$TMP_CSV" "$status" 1>&2 + exit 35 + fi + + # append newline if last line of CSV file if not empty + # + LAST_CHAR=$(tail -c 1 "$TMP_CSV") + if [[ -n $LAST_CHAR ]]; then + echo "" >> "$TMP_CSV" + fi + + # replace CSV file if different + # + if ! cmp -s "$TMP_CSV" "$csv"; then + + # replace CSV file with a better formatted version + # + echo "$0: updating $csv" + mv -f -v "$TMP_CSV" "$csv" + status="$?" + if [[ $status -ne 0 ]]; then + echo "$0: ERROR: mv -f -v $TMP_CSV $csv failed, error: $status" 1>&2 + exit 36 + fi + + else + + # CSV file is OK, remove the tmp file + # + rm -f "$TMP_CSV" + if [[ -e $TMP_CSV ]]; then + echo "$0: ERROR: TMP_CSV cannot be removed for cleanup: $TMP_CSV" 1>&2 + exit 77 + fi + fi +done + # All Done!!! -- Jessica Noll, Age 2 # exit 0