From 99cef95e67209d44cde6d256aae457fe62bb87c2 Mon Sep 17 00:00:00 2001 From: Landon Curt Noll Date: Sat, 25 Mar 2023 04:11:54 -0700 Subject: [PATCH] fix Makefile and README.md for 2019/mill --- 2019/mills/Makefile | 473 +++++++++++++------------------------------ 2019/mills/README.md | 185 ++++++++++------- 2 files changed, 250 insertions(+), 408 deletions(-) diff --git a/2019/mills/Makefile b/2019/mills/Makefile index bb972a110..997709756 100644 --- a/2019/mills/Makefile +++ b/2019/mills/Makefile @@ -1,14 +1,41 @@ #!/usr/bin/env make # -# 2019 makefile +# IOCCC 2019 winning entry - mills + +################################################################################ # -# This work by Landon Curt Noll, Simon Cooper, and Leonid A. Broukhis -# is licensed under: +# Regarding individual entries: +# +# IOCCC winning entry code may not work on your system. Even with standards, +# conditions and build environments change, especially when it comes to more +# extreme code such you might find in an IOCCC winner. Moreover, what was +# allowed and liked in one IOCCC contest may no longer be liked/allowed in +# later IOCCC years. +# +# Corrections on winning author and fixing typos in remarks are VERY WELCOME. +# +# GitHub pull requests are welcome: +# +# If you have a patch that might make running an entry work in modern Unix-like +# environments, please create a GitHub pull request for the IOCCC judges to +# consider. +# +# Suggestions and corrections regarding Makefiles, winning author information, +# fixing typos in remarks, and code patches are welcome via opening a GitHub +# issue or better yet, by creating a GitHub pull request for the IOCCC judges +# to consider. To form a GitHub issue or GitHub pull, please visit: +# +# https://github.com/ioccc-src/winner +# +################################################################################ + +# This work by Landon Curt Noll and Leonid A. Broukhis is licensed under: # # Creative Commons Attribution-ShareAlike 3.0 Unported License. # # See: http://creativecommons.org/licenses/by-sa/3.0/ + ############################# # shell used by this Makefile ############################# @@ -19,262 +46,7 @@ SHELL= bash # common tool locations ####################### # -A2P= a2p -AR= ar -ASA= asa -AT= at -ATQ= atq -ATRM= atrm -AWK= awk -BANNER= banner -BASE64= base64 -BASENAME= basename -BATCH= batch -BC= bc -BINHEX= binhex -BISON= bison -BUNZIP2= bunzip2 -BZCAT= bzcat -BZCMP= bzcmp -BZDIFF= bzdiff -BZEGREP= bzegrep -BZFGREP= bzfgrep -BZGREP= bzgrep -BZIP2= bzip2 -BZLESS= bzless -BZMORE= bzmore -C2PH= c2ph -C89= c89 -C99= c99 -CAL= cal -CALC= calc -CAT= cat -CD= cd -CHFLAGS= chflags -CHGRP= chgrp -CHMOD= chmod -CKSUM= cksum -CLANG= clang -CLANG_PLUSPLUS= clang++ -CLEAR= clear -CMP= cmp -COL= col -COLLDEF= colldef -COLRM= colrm -COLUMN= column -COMPRESS= compress -CP= cp -CPIO= cpio -CPP= cpp -CRC32= crc32 -CSH= csh -CSPLIT= csplit -CURL= curl -CUT= cut -C_PLUSPLUS= c++ -C_PLUSPLUS_FILT= c++filt -DATE= date -DC= dc -DD= dd -DF= df -DIFF3= diff3 -DIFF= diff -DIG= dig -DIRNAME= dirname -ED= ed -EGREP= egrep -ENV= env -EQN= eqn -ETAGS= etags -EXPECT= expect -EXPR= expr -FALSE= false -FGREP= fgrep -FILE= file -FIND2PERL= find2perl -FIND= find -FLEX= flex -FLEX_PLUSPLUS= flex++ -FMT= fmt -FOLD= fold -FS_USAGE= fs_usage -FUNZIP= funzip -FUSER= fuser -GCC= gcc -GDIFFMK= gdiffmk -GENCAT= gencat -GENSTRINGS= genstrings -GETOPT= getopt -GETOPTS= getopts -GNUMAKE= gnumake -GREP= grep -GROFF= groff -GROFFER= groffer -GROG= grog -GROPS= grops -GROTTY= grotty -GUNZIP= gunzip -GVIM= gvim -GZCAT= gzcat -GZEXE= gzexe -GZIP_PROG= gzip -G_PLUSPLUS= g++ -H2PH= h2ph -H2XS= h2xs -HASH= hash -HEAD= head -HOSTNAME_PROG= hostname -ICONV= iconv -ID= id -INDENT= indent -INFO= info -JOT= jot -KILL= kill -KSH= ksh -LAST= last -LD= ld -LEX= len -LINK= link -LN= ln -LS= ls -M4= m4 -MAKE= make -MAN= man -MKDIR= mkdir -MKFIFO= mkfifo -MKTEMP= mktemp -MV= mv -NANO= nano -NASM= nasm -NEQN= neqn -NICE= nice -NL= nl -NM= nm -NOHUP= nohup -NROFF= nroff -NSLOOKUP= nslookup -OD= od -OPENSSL= openssl -PASTE= paste -PATCH= patch -PATHCHK= pathchk -PAX= pax -PERL= perl -PICO= pico -PR= pr -PRINTENV= printenv -PS= ps -PTAR= ptar -PTARDIFF= ptardiff -PTARGREP= ptargrep -PWD= pwd -PYDOC= pydoc -PYTHON= python -PYTHONW= pythonw -READLINK= readlink -RENICE= renice -RESET= reset -REV= rev -RI= ri -RM= rm -RMDIR= rmdir -RSYNC= rsync -RUBY= ruby -RVIM= rvim -SAY= say -SCP= scp -SCREEN= screen -SCRIPT= script -SDIFF= sdiff -SED= sed -SEQ= seq -SFTP= sftp -SH= sh -SHA1= sha1 -SHA= sha -SHAR= shar -SHASUM5_18= shasum5.18 -SHASUM= shasum -SIZE= size -SLEEP= sleep -SORT= sort -SSH= ssh -STAT= stat -STRIP= strip -STTY= stty -SUM= sum -SYNC= sync -TABS= tabs -TAIL= tail -TAR= tar -TEE= tee -TEST= test -TFTP= tftp -TIDY= tidy -TIME= time -TOP= top -TOUCH= touch -TPUT= tout -TPUT= tput -TR= tr -TROFF= troff -TRUE= true -TSET= tset -TSORT= tsort -UL= ul -UNAME= uname -UNCOMPRESS= uncompress -UNEXPAND= unexpand -UNIFDEF= unifdef -UNIFDEFALL= unifdefall -UNIQ= uniq -UNITS= units -UNLINK= unlink -UNZIP= unzip -UNZIPSFX= unzipsfx -UPTIME= uptime -UUDECODE= uudecode -UUENCODE= uuencode -UUIDGEN= uuidgen -VI= vi -VIEW= view -VIM= vim -VIMDIFF= vimdiff -W= w -WAIT4PATH= wait4path -WAIT= wait -WC= wc -WHAT= what -WHATIS= whatis -WHICH= which -WHO= who -WHOAMI= whoami -WHOIS= whois -WRTIE= write -XAR= xar -XARGS= xargs -XATTR= xattr -XXD= xxd -YACC= yacc -YES= yes -ZCAT= zcat -ZCMP= zcmp -ZDIFF= zdiff -ZEGREP= zegrep -ZFGREP= zfgrep -ZFORCE= zforce -ZGREP= zgrep -ZIP= zip -ZIPCLOAK= zipcloak -ZIPGREP= zipgrep -ZIPINFO= zipinfo -ZIPNOTE= zipnote -ZIPSPLIT= zipsplit -ZLESS= zless -ZMORE= zmore -ZNEW= znew -ZPRINT= zprint -ZSH= zsh +include ../../var.mk # Set X11_LIBDIR to the directory where the X11 library resides # @@ -288,29 +60,25 @@ X11_LIBDIR= /opt/X11/lib #X11_INCDIR= /usr/X11/include X11_INCDIR= /opt/X11/include -# Compiler warnings -# -#CWARN= -#CWARN= -Wall -#CWARN= -Wall -Wextra -CWARN= -Wall -Wextra -pedantic ${CSILENCE} -#CWARN= -Wall -Wextra -Weverything -#CWARN= -Wall -Wextra -Weverything -pedantic -#CWARN= -Wall -Wextra -Weverything -pedantic ${CSILENCE} - -# Silence warnings that ${CWARN} would normally complain about +# Common C compiler warnings to silence # #CSILENCE= -#CSILENCE= -Wno-implicit-int -CSILENCE= -Wno-missing-field-initializers -Wno-for-loop-analysis +CSILENCE= -Wno-for-loop-analysis -Wno-missing-field-initializers -# compiler standard +# Common C compiler warning flags +# +#CWARN= ${CSILENCE} +#CWARN= -Wall ${CSILENCE} +#CWARN= -Wall -Wextra ${CSILENCE} +CWARN= -Wall -Wextra -pedantic ${CSILENCE} + +# Compiler standard # #CSTD= #CSTD= -ansi -CSTD= -std=c11 +CSTD= -std=gnu17 -# compiler bit architecture +# Compiler bit architecture # # Some entries require 32-bitness: # ARCH= -m32 @@ -322,21 +90,21 @@ CSTD= -std=c11 # ARCH= -# defines that are needed to compile +# Defines that are needed to compile # #CDEFINE= -#CDEFINE= -DIOCCC=26 CDEFINE= ${param} -# include files that are needed to compile +# Include files that are needed to compile # CINCLUDE= #CINCLUDE= -include stdlib.h #CINCLUDE= -include stdio.h #CINCLUDE= -include stdlib.h -include stdio.h -#CINCLUDE= -I ${X11_INCDIR} +#CINCLUDE= -I ${X11_INCDIR} -I ${X11_INCDIR}/X11 +#CINCLUDE= `${SDL2_CONFIG} --cflags` -# optimization +# Optimization # # Most compiles will safely use -O2. Some can use only -O1 or -O. # A few compilers have broken optimizers or this entry make break @@ -349,37 +117,58 @@ CINCLUDE= #OPT= -O2 OPT= -O3 -# default flags for ANSI C compilation +# Default flags for ANSI C compilation # CFLAGS= ${CSTD} ${CWARN} ${ARCH} ${CDEFINE} ${CINCLUDE} ${OPT} # Libraries needed to build # #LIBS= -#LIBS= -L ${X11_LIBDIR} LIBS= -lm +#LIBS= -L ${X11_LIBDIR} -lX11 +#LIBS= `${SDL2_CONFIG} --libs` -# ANSI compiler -# -# Set CC to the name of your ANSI compiler. -# -# Some entries seem to need gcc. If you have gcc, set -# both CC and MAY_NEED_GCC to gcc. -# -# If you do not have gcc, set CC to the name of your ANSI compiler, and -# set MAY_NEED_GCC to either ${CC} (and hope for the best) or to just : -# to disable such programs. +# C compiler to use # CC= cc #CC=clang -MAY_NEED_GCC= gcc +#CC=gcc + +# Compiler add-ons or replacements for clang only +# +ifeq ($(CC),clang) +# +CSILENCE+= -Wno-double-promotion -Wno-float-conversion -Wno-format-nonliteral \ + -Wno-implicit-float-conversion -Wno-implicit-int-conversion \ + -Wno-missing-prototypes -Wno-missing-variable-declarations \ + -Wno-poison-system-directories -Wno-shadow -Wno-shorten-64-to-32 \ + -Wno-sign-conversion -Wno-strict-prototypes -Wno-vla +# +CWARN+= -Weverything +# +endif + +# Specific add-ons or replacements for gcc only +# +ifeq ($(CC),gcc) +# +#CSILENCE+= +# +#CWARN+= +# +endif ############################## # Special flags for this entry ############################## # -OBJ= prog.o +ENTRY= mills +PROG= prog +# +OBJ= ${PROG}.o +CSRC= ${PROG}.o +# INPUTS_GZ= IOCCC-hints.txt.gz Shakespeare.txt.gz \ IOCCC-Rules-Guidelines.txt.gz Eugene_Onegin.txt.gz INPUTS= IOCCC-hints.txt Shakespeare.txt \ @@ -398,7 +187,7 @@ ALT_TARGET= # hyperparameters param = -DRS=.15 -DTR=.95 -DLR=.002 -DLD=.97 -DLE=10 -DCL=5 -DB1=.9 -DB2=.999 \ -DEP=1e-8 -DWD=8e-5 -DDI=100 -DSL=200 -DN=50 -DTP=1 \ - -DPF='"cp%02d_%.3f"' + -DPF='"cp%02d_%.3f"' ############################################################## # Networks @@ -540,14 +329,19 @@ gru3 = ${gru} ${three_layer} # prog = ${rnn1} + ################# # build the entry ################# # -all: ${TARGET} ${DATA} +all: data ${TARGET} @${TRUE} -${TARGET}: prog.c Makefile ${INPUTS} +.PHONY: all alt data everything clean clobber install \ + love haste waste maker easter_egg sandwich \ + supernova deep_magic magic charon pluto + +${TARGET}: ${PROG}.c Makefile ${INPUTS} ${CC} ${CFLAGS} -o $@ $< $($@) ${LIBS} # alternative executable @@ -583,14 +377,21 @@ IOCCC-Rules-Guidelines.output.txt: IOCCC-Rules-Guidelines.cp98_0.175 ${generator Eugene_Onegin.output.txt: Eugene_Onegin.cp11_1.188 ${generator} ${generator} < $< | ${HEAD} -n 1000 > $@ +# both all and alt +# +everything: all alt + + ############### # utility rules ############### # -everything: all alt - clean: ${RM} -f ${OBJ} ${ALT_OBJ} + @-if [ -f indent.c ]; then \ + echo ${RM} -f indent.c; \ + ${RM} -f indent.c; \ + fi cpclean: ${RM} -f cp*_*.* @@ -604,20 +405,28 @@ clobber: clean cpclean echo 'ate sandwich'; \ fi -nuke: clobber - @${TRUE} - -dist_clean: nuke - @${TRUE} - install: - @echo "Surely you are performing, Dr. May!" + @echo "Dr. Spock says that is not logical!" @${TRUE} -# backwards compatibility +# you may be able to misunderstand the source by reading indent.c # -build: all - @${TRUE} +indent.c: ${PROG}.c + @if which "${GINDENT}" >/dev/null 2>&1; then \ + echo ${RM} -f $@; \ + ${RM} -f $@; \ + echo "${GINDENT} < $< > $@"; \ + ${GINDENT} < $< > $@; \ + elif which "${INDENT}" >/dev/null 2>&1; then \ + echo ${RM} -f $@; \ + ${RM} -f $@; \ + echo "${INDENT} < $< > $@"; \ + ${INDENT} < $< > $@; \ + else \ + echo "no intent tool found, ident $< yourself, sorry"; \ + echo "exit 1"; \ + exit 1; \ + fi ################## @@ -636,22 +445,19 @@ waste: @echo 'haste' @${TRUE} -make: - @echo 'Attend a maker faire' +maker: + @echo 'Participate in a maker faire' @${TRUE} easter_egg: - @echo you expected to often mis-understand this $${RANDOM} magic - @echo chongo '' "/\\oo/\\" - @echo Eggy + @echo "you expected to often misunderstand this $${RANDOM} magic" + @echo "chongo '' /\\oo/\\" + @echo "Eggy" -fabricate fashion form frame manufacture produce: make - @${TRUE} - -sandwich: +sandwich: easter_egg @if [ `id -u` -eq 0 ]; then \ echo 'Okay.'; \ - echo $${RANDOM}`date +%s`$${RANDOM} > $@; \ + echo "$${RANDOM}`date +%s`$${RANDOM}" > $@; \ else \ echo 'What? Make it yourself.'; \ exit 1; \ @@ -661,18 +467,19 @@ sandwich: # as well as the West Coast Computer Faire and # you might be confused different. :-) # -supernova: nuke +supernova: sandwich @-if [ -r .code_anal ]; then \ ${RM} -f .code_anal_v6; \ else \ echo "planet deniers, like some members of the IAU, are so cute when they try to defend their logic"; \ fi - @echo A $@ helps ${MAKE} the elements that help form planets + @echo "A $@ helps ${MAKE} the elements that help form planets" @${TRUE} -deep_magic: +deep_magic: supernova @-if [ -r .code_anal ]; then \ - ccode_analysis --deep_magic fc6b6af226b7cc598de3d48fc20ed0bd54f7b4f7f0f651a32b9cf8345b2a3b3f --FNV1a_hash_512_bit "prog"; \ + ccode_analysis --deep_magic 7889066cdbdf09b0408021754586e895be641cc9be29e7eafd3ca65359541f17 \ + --FNV1a_hash_512_bit -v2 "${PROG}" "${PROG}.c"; \ else \ echo "Wrong! Do it again!"; \ sleep 2; \ @@ -684,7 +491,7 @@ deep_magic: magic: deep_magic @-if [ -r .code_anal ]; then \ - ccode_analysis --level 391581 --mode 216193 --FNV1a_hash_512_bit "prog"; \ + ccode_analysis --level 20003 --mode 5717175 --FNV1a_hash_512_bit -v2 "${PROG}" "${PROG}.c"; \ else \ echo "If you don't eat yer meat, you can't have any pudding!"; \ echo "How can you have any pudding if you don't eat yer meat?!"; \ @@ -693,14 +500,14 @@ magic: deep_magic # The IOCCC resident astronomer states # -charon: supernova - @echo $@ is a dwarf planet - @echo dwarf is a type of planet - @echo therefore IAU, $@ is a planet +charon: magic + @echo "$@ is a dwarf planet" + @echo "dwarf is a type of planet" + @echo "therefore IAU, $@ is a planet" -pluto: supernova +pluto: charon ${MAKE} charon - @echo $@ is a dwarf planet - @echo dwarf is a type of planet - @echo therefore, $@ is a planet - @echo get used to having lots of planets becasue supernovae can ${MAKE} a lot of them + @echo "$@ is a dwarf planet" + @echo "dwarf is a type of planet" + @echo "therefore, $@ is a planet" + @echo "having lots of planets is good feature because supernovae can help ${MAKE} a lot of them" diff --git a/2019/mills/README.md b/2019/mills/README.md index f0b2cdd80..be53988ee 100644 --- a/2019/mills/README.md +++ b/2019/mills/README.md @@ -1,35 +1,38 @@ # Most in need to be tweeted -Christopher Mills -Twitter: @MisterXopher + Christopher Mills + Twitter: @MisterXopher +# To build: -## Judges' comments: -### To use: +```sh +make +``` - make +### To run: - make cpclean - # Let this run for about about an hour and then kill it: - ./prog Shakespeare.txt - ./prog < $(< ls -1tr cp* | tail -1) | head -100 +```sh +make cpclean +# Let this run for about about an hour and then kill it: +./prog Shakespeare.txt +./prog < $(< ls -1tr cp* | tail -1) | head -100 +``` ### Try: - # If you are on a 64-bit system: +```sh +make test-64bit - make test-64bit +less IOCCC-Rules-Guidelines.output.txt - less IOCCC-Rules-Guidelines.output.txt +less IOCCC-hints.output.txt - less IOCCC-hints.output.txt +less Eugene_Onegin.output.txt +``` - less Eugene_Onegin.output.txt + However, as the binary model files used to produce the output are in an implementation-specific format, your mileage may vary. - # However, as the binary model files used to produce the output are in an implementation-specific format, - # your mileage may vary. - -### Selected Judges Remarks: +## Judges' comments: Can a machine learn? @@ -43,11 +46,9 @@ Can it write rules and guidelines for the IOCCC? You decide. :-) - ## Author's comments: -Welcome to OMLET! 🍳 -================= +### Welcome to OMLET! 🍳 OMLET is the _Obfuscated Machine Learning Environment Toolkit_, a micro-framework for experimenting with [recurrent neural networks][1]. OMLET @@ -79,9 +80,7 @@ training, but you can have even more fun by downloading some larger datasets: + [_War and Peace_][31] + [Donald Trump's 2016 campaign speeches][32] - -Getting started with OMLET --------------------------- +### Getting started with OMLET OMLET has three operating modes: @@ -92,15 +91,24 @@ OMLET has three operating modes: For your first OMLET experiment, we will try training a simple single-level RNN to write some [Shakespere][5] plays. Start by typing - make +```sh +make +``` After it builds, train it using - ./prog shakespere.txt +```sh +./prog shakespere.txt +``` This will immediately start outputting gibberish to the output, e.g. - ./prog shakespere.txt +```sh +./prog shakespere.txt +``` + +produces: + sins ohennAu T-teooclelp tiThoWy @@ -240,7 +248,9 @@ doing `Control-C`. You can continue training from a previous checkpoint by providing the checkpoint file name as the second parameter, for example: - ./prog shakespere.txt cp01_1.970 +```sh +./prog shakespere.txt cp01_1.970 +``` After the validation cycle finishes, OMLET begins the next epoch by restarting training at the beginning of the training set. Training continues forever, @@ -254,7 +264,9 @@ parameters to generate data. Inference mode takes the checkpoint file as *standard input* (not on the command line) and hence must be run with a command like - ./prog < cp55_1.807 +```sh +./prog < cp55_1.807 +``` Running it produces an infinite amount of generated output, until you hit `Control-C` to stop it. @@ -264,15 +276,16 @@ will want to delete all the checkpoint files because the format depends on both the network and the input -- using the wrong checkpoint is likely to cause a crash. -Experimenting with different networks -------------------------------------- +### Experimenting with different networks The default network for OMLET (the one you get if you type `make` with the executable name `prog`) is the simplest recurrent neural network. It looks like - h = tanh(Wxh * x + Whh * h' + Bh) - y = Why * h + By +```c +h = tanh(Wxh * x + Whh * h' + Bh) +y = Why * h + By +``` where @@ -292,20 +305,31 @@ It is the presence of the hidden state vector that allows the RNN to "remember" the past. We can see what would happen if we removed this hidden state. If you type - make lin1 +```sh +make lin1 +``` OMLET will create an [ADALINE][10] network that does - y = Wxy * x + By +```c +y = Wxy * x + By +``` This simple [linear][11] [feed-forward network][12]. You can run it with - ./lin1 shakespeare.txt +```sh +./lin1 shakespeare.txt +``` The linear network won't be able to get past the gibberish stage, because it lacks history: - ./lin1 shakespeare.txt +```sh +./lin1 shakespeare.txt +``` + +produces: + UERond w, @@ -343,15 +367,16 @@ above. Alas, even adding a nonlinearity to the feed-forward network still lack the history provided by the hidden state vector (although if you want to try it yourself, you can do so with `make per1`). -Going deeper ------------- +### Going deeper We can try to improve the RNN's performance by stacking RNN modules atop each other: - h1 = RNN(h1', x) - h2 = RNN(h2', h1) - y = Why * h2 + By +```c +h1 = RNN(h1', x) +h2 = RNN(h2', h1) +y = Why * h2 + By +``` with `RNN(h, x)` defined as above. Each RNN module has its own set of parameters and its own hidden state vector. This will improve the network's @@ -369,11 +394,15 @@ performance, at the cost of a much larger parameter space You can try the deeper network by doing - make rnn2 +```sh +make rnn2 +``` (or even `make rnn3` if you want a three-layer RNN) and train it with - ./rnn2 shakespeare.txt +```sh +./rnn2 shakespeare.txt +``` The additional depth should allow the network to make better predictions (it can represent more complicated history), but it may take a long time to train @@ -381,8 +410,7 @@ can represent more complicated history), but it may take a long time to train because of the [vanishing and exploding gradient problem][17], which might keep it from ever reaching its potential. -LSTMs and GRUs --------------- +### LSTMs and GRUs RNNs are particularly hard to train because the they are trained using [bankpropagation through time][18]. The RNN is trained by effectively @@ -395,11 +423,15 @@ gradient, worsening the exploding and vanishing gradient problem. solve this problem. Christopher Olah gives a good description of them at his [blog posting][20]. You can build a two-level LSTM by doing - make lstm2 +```sh +make lstm2 +``` and train with it with - ./lstm2 shakespeare.txt +```sh +./lstm2 shakespeare.txt +``` The LSTM is much easier to train because it explicitly decides how to update its hidden state via "gates". These gates are called @@ -411,11 +443,13 @@ its hidden state via "gates". These gates are called The basic LSTM equations are - f = sigmoid(Wxf * x + Whf * h' + Bf) - i = sigmoid(Wxi * x + Whi * h' + Bi) - o = sigmoid(Wxo * x + Who * h' + Bo) - c = f * c' + i * tanh(Wxc * x + Whc * h' + Bc) - h = o * tanh(c) +```c +f = sigmoid(Wxf * x + Whf * h' + Bf) +i = sigmoid(Wxi * x + Whi * h' + Bi) +o = sigmoid(Wxo * x + Who * h' + Bo) +c = f * c' + i * tanh(Wxc * x + Whc * h' + Bc) +h = o * tanh(c) +``` Where @@ -441,10 +475,11 @@ simplified versions of an LSTM which combine the gates together, meaning they require fewer learned parameters. This allows them to train faster than a generic LSTM. You can build a two-layer GRU with - make gru2 +```sh +make gru2 +``` -Building your own networks --------------------------- +### Building your own networks The OMLET `Makefile` comes with one-, two- and three-layer RNNs, LSTMs and GRUs, along with simpler feed-forward networks like multi-layer perceptrons and @@ -454,9 +489,11 @@ compiler's command-line by using `-D` directives. The network is defined by a `-DNW='...'` command which consists of a series of comma-separated assignments. For example, the simple one-layer RNN could be defined like - -DNW=` x = I(n), hp = I(128), \ - h = C(hp, T(A(L(128, x), L(128, hp)))), \ - y = L(n, h)' +```make +-DNW=` x = I(n), hp = I(128), \ + h = C(hp, T(A(L(128, x), L(128, hp)))), \ + y = L(n, h)' +``` The network declares `x` as an input vector (there must be a declaration for `x`). It is declared as `I(n)`, which is an input vector of size `n`, which @@ -497,16 +534,18 @@ training mode, this is used to generate the loss which is backpropagated. As an example of a more complicated network, we can look at a two-layer GRU network: - -DHS=128, \ - -DNW=' x = I(n), \ - y = L(n, MD(MD(x)))' \ - -DBK=' hp = I(HS), \ - z = S(A(L(HS, x), L(HS, hp))), \ - r = S(A(L(HS, x), L(HS, hp))), \ - c = T(A(L(HS, x), L(HS, hp))), \ - zc = OG(1, -1, z), \ - h = C(hp, A(M(zc, hp), M(z, c))), \ - y = h' +```make +-DHS=128, \ +-DNW=' x = I(n), \ + y = L(n, MD(MD(x)))' \ +-DBK=' hp = I(HS), \ + z = S(A(L(HS, x), L(HS, hp))), \ + r = S(A(L(HS, x), L(HS, hp))), \ + c = T(A(L(HS, x), L(HS, hp))), \ + zc = OG(1, -1, z), \ + h = C(hp, A(M(zc, hp), M(z, c))), \ + y = h' +``` We are using a few new tricks here -- first, we are defining `HS` as the size of the hidden and cell vectors. There's nothing special about this name, its @@ -544,8 +583,7 @@ Note: even if you don't use `MD` in your network, you should still define `BK` by adding `-DBK='y=x'` to the command line, otherwise you will get a compile-time error. -Hyperparameters ---------------- +### Hyperparameters OMLET has a large number of training and inference parameters which can be changed by the user. All of these are set by `-D` on the compile command line. @@ -586,7 +624,6 @@ The list of hyperparameters follows: wish to add a subdirectory to the name to keep checkpoint files out of the current directory. - [1]: https://en.wikipedia.org/wiki/Recurrent_neural_network [2]: https://en.wikipedia.org/wiki/Deep_learning [3]: https://www.tensorflow.org @@ -625,12 +662,10 @@ The list of hyperparameters follows: [43]: https://www.fast.ai/2018/07/02/adam-weight-decay/ [44]: http://karpathy.github.io/2015/05/21/rnn-effectiveness/ +## Copyright: --------------------------------------------------------------------------------- -