From 77b2e0fc6f8304d06a14d742409fba03b818bea2 Mon Sep 17 00:00:00 2001 From: Bobby Jaros Date: Thu, 17 Dec 2015 22:36:36 -0800 Subject: [PATCH 1/6] parser: paragraphids and nnparse newparse can optionally output paragraphids and sentenceids for each token. p1 s1 w1 p2 s2 w2 p3 s3 w3 p4 s4 w4 p5 s5 w5 p6 s6 w6 nnparse harnesses this functionality in a very simple version of this, which assumes each newline denotes a paragraph and each ". " or "? " or "! " denotes a new sentence. --- src/main/C/newparse/makefile.gcc | 4 +- src/main/C/newparse/makefile.w32 | 4 +- src/main/C/newparse/newparse.cpp | 17 ++++++- src/main/C/newparse/nnparse.flex | 76 ++++++++++++++++++++++++++++++++ src/main/C/newparse/utils.cpp | 38 ++++++++++++++++ src/main/C/newparse/utils.h | 4 ++ 6 files changed, 138 insertions(+), 5 deletions(-) create mode 100755 src/main/C/newparse/nnparse.flex diff --git a/src/main/C/newparse/makefile.gcc b/src/main/C/newparse/makefile.gcc index bd42f9d5..8bd62318 100755 --- a/src/main/C/newparse/makefile.gcc +++ b/src/main/C/newparse/makefile.gcc @@ -2,10 +2,10 @@ .SUFFIXES: .SUFFIXES: .c .cpp .o .exe .lxc .flex -EXES=xmltweet.exe xmlwiki.exe trec.exe tparse.exe parsevw.exe tparse2.exe +EXES=xmltweet.exe xmlwiki.exe trec.exe tparse.exe parsevw.exe tparse2.exe nnparse.exe OBJS=newparse.o utils.o gzstream.o -.SECONDARY: xmltweet.lxc xmlwiki.lxc trec.lxc +.SECONDARY: xmltweet.lxc xmlwiki.lxc trec.lxc nnparse.lxc all: $(EXES) diff --git a/src/main/C/newparse/makefile.w32 b/src/main/C/newparse/makefile.w32 index 574c92c9..eb58a8c0 100755 --- a/src/main/C/newparse/makefile.w32 +++ b/src/main/C/newparse/makefile.w32 @@ -2,10 +2,10 @@ .SUFFIXES: .SUFFIXES: .c .cpp .obj .exe .lxc .flex -EXES=xmltweet.exe xmlwiki.exe trec.exe tparse.exe parsevw.exe tparse2.exe +EXES=xmltweet.exe xmlwiki.exe trec.exe tparse.exe parsevw.exe tparse2.exe nnparse.exe OBJS=newparse.obj utils.obj gzstream.obj -.SECONDARY: xmltweet.lxc xmlwiki.lxc trec.lxc +.SECONDARY: xmltweet.lxc xmlwiki.lxc trec.lxc nnparse.lxc all: $(EXES) diff --git a/src/main/C/newparse/newparse.cpp b/src/main/C/newparse/newparse.cpp index c51f19da..9851d4e9 100755 --- a/src/main/C/newparse/newparse.cpp +++ b/src/main/C/newparse/newparse.cpp @@ -10,6 +10,8 @@ ivector wcount; ivector tokens; +ivector paragraphids; +ivector sentenceids; unhash unh; strhash htab; @@ -18,9 +20,14 @@ extern int yylex(void); extern FILE* yyin; int numlines=0; +int doparagraphids=0; +int paragraphid=0; +int sentenceid=0; } int checkword(char *str) { + paragraphids.push_back(paragraphid); + sentenceids.push_back(sentenceid); return checkword(str, htab, wcount, tokens, unh); } @@ -98,9 +105,17 @@ int main(int argc, char ** argv) { pos = rname.rfind('/'); if (pos == string::npos) pos = rname.rfind('\\'); if (pos != string::npos) rname = rname.substr(pos+1, rname.size()); - writeIntVec(tokens, odname+rname+".imat"+suffix, membuf); + if (doparagraphids) { + writeIntVec3Cols(paragraphids, sentenceids, tokens, odname+rname+".imat"+suffix, membuf); + } else { + writeIntVec(tokens, odname+rname+".imat"+suffix, membuf); + } tokens.clear(); + sentenceids.clear(); + paragraphids.clear(); numlines = 0; + sentenceid = 0; + paragraphid = 0; here = strtok(NULL, " ,"); } fprintf(stderr, "\nWriting Dictionary\n"); diff --git a/src/main/C/newparse/nnparse.flex b/src/main/C/newparse/nnparse.flex new file mode 100755 index 00000000..620a4b75 --- /dev/null +++ b/src/main/C/newparse/nnparse.flex @@ -0,0 +1,76 @@ +/* Scanner for neural net datasources */ + +%{ + extern int checkword(char *); + extern void addtok(int tok); + extern int parsedate(char * str); + extern int numlines; + extern int doparagraphids; + extern int sentenceid; + extern int paragraphid; + + #define YY_USER_ACTION doparagraphids=1; // Macro happens at initialization +%} + +%option never-interactive +%option noyywrap + +LETTER [a-zA-Z_] +DIGIT [0-9] +PUNCT [;:,.?!] + +%% + +{LETTER}+ { + int iv = checkword(yytext); + } + +"<"{LETTER}+">" { + int iv = checkword(yytext); + } + +"" { + int iv = checkword(yytext); + } + +{PUNCT} { + int iv = checkword(yytext); + } + +"..""."* { + char ell[] = "..."; + int iv = checkword(ell); + } + +". " { + sentenceid++; + char ell[] = "."; + int iv = checkword(ell); + } + +"? " { + sentenceid++; + char ell[] = "?"; + int iv = checkword(ell); + } + +"! " { + sentenceid++; + char ell[] = "!"; + int iv = checkword(ell); + } + +[\n] { + numlines++; + paragraphid++; + sentenceid = 0; + if (numlines % 1000000 == 0) { + fprintf(stderr, "\r%05d lines", numlines); + fflush(stderr); + } + } + +. {} + +%% + diff --git a/src/main/C/newparse/utils.cpp b/src/main/C/newparse/utils.cpp index aeefc650..8e25727e 100755 --- a/src/main/C/newparse/utils.cpp +++ b/src/main/C/newparse/utils.cpp @@ -364,6 +364,44 @@ int writeIntVec(ivector & im, string fname, int buffsize) { return 0; } +int writeIntVec2Cols(ivector & im1, ivector & im2, string fname, int buffsize) { + int fmt, nrows, nnz; + int ncols = 2; + + ostream *ofstr = open_out_buf(fname.c_str(), buffsize); + fmt = 110; + nrows = im1.size(); + nnz = nrows * ncols; + ofstr->write((const char *)&fmt, 4); + ofstr->write((const char *)&nrows, 4); + ofstr->write((const char *)&ncols, 4); + ofstr->write((const char *)&nnz, 4); + ofstr->write((const char *)&im1[0], 4 * nrows); + ofstr->write((const char *)&im2[0], 4 * nrows); + closeos(ofstr); + return 0; +} + +int writeIntVec3Cols(ivector & im1, ivector & im2, ivector & im3, string fname, int buffsize) { + int fmt, nrows, nnz; + int ncols = 3; + + ostream *ofstr = open_out_buf(fname.c_str(), buffsize); + fmt = 110; + nrows = im1.size(); + nnz = nrows * ncols; + ofstr->write((const char *)&fmt, 4); + ofstr->write((const char *)&nrows, 4); + ofstr->write((const char *)&ncols, 4); + ofstr->write((const char *)&nnz, 4); + ofstr->write((const char *)&im1[0], 4 * nrows); + ofstr->write((const char *)&im2[0], 4 * nrows); + ofstr->write((const char *)&im3[0], 4 * nrows); + closeos(ofstr); + return 0; +} + + int writeDIntVec(divector & im, string fname, int buffsize) { int fmt, nrows, ncols, nnz; ostream *ofstr = open_out_buf(fname.c_str(), buffsize); diff --git a/src/main/C/newparse/utils.h b/src/main/C/newparse/utils.h index fc4ed3c4..d810a0b4 100755 --- a/src/main/C/newparse/utils.h +++ b/src/main/C/newparse/utils.h @@ -162,6 +162,10 @@ void closeos(ostream *ofs); int writeIntVec(ivector & im, string fname, int buffsize); +int writeIntVec2Cols(ivector & im1, ivector & im2, string fname, int buffsize); + +int writeIntVec3Cols(ivector & im1, ivector & im2, ivector & im3, string fname, int buffsize); + int writeDIntVec(divector & im, string fname, int buffsize); int writeQIntVec(qvector & im, string fname, int buffsize); From c8ddefc3c4ca56ac198340c14be2e220c02b9fbc Mon Sep 17 00:00:00 2001 From: Bobby Jaros Date: Thu, 17 Dec 2015 22:44:01 -0800 Subject: [PATCH 2/6] Prepare data for consumption by SeqToSeq LSTMs. Starts with the output of nnparse.exe, two paired files each with this format: p1 s1 w1 p2 s2 w2 p3 s3 w3 p4 s4 w4 p5 s5 w5 p6 s6 w6 (For SeqToSeq we assume each line contains one sentence, so the paragraphid (the first column) denotes the sentence and sentenceid (the second column) is always ignored). The two parsed sentence IMats are paired line-by-line: the ith line of the src IMat corresponds to the ith line of the dst IMat. Produces two paired SMat's of the following form: w00 w01 w02 w03 w04 w05 ... w10 w11 w12 w13 w14 w15P ... w20 w21 w22 w23P w24 w25P ... w30 w31P w32 ... w40P w32P w33 ... where wij is the dictionary index of the i'th word in the j'th sentence and words with a P suffix are padding symbols. The columns of the two output SMat's are still paired: column j of the src output SMat and column j of the dst output SMat correspond to line j of the src input and line j of the dst input respectively. Furthermore, the sentences are collated into batches of similar lengths. The minibatches are randomly permuted after collation to avoid training bias. See in-file docs for additional options. --- scripts/prepSeqToSeq.ssc | 22 + .../scala/BIDMach/tools/SeqToSeqData.scala | 533 ++++++++++++++++++ 2 files changed, 555 insertions(+) create mode 100644 scripts/prepSeqToSeq.ssc create mode 100644 src/main/scala/BIDMach/tools/SeqToSeqData.scala diff --git a/scripts/prepSeqToSeq.ssc b/scripts/prepSeqToSeq.ssc new file mode 100644 index 00000000..1b5de190 --- /dev/null +++ b/scripts/prepSeqToSeq.ssc @@ -0,0 +1,22 @@ +:silent +import BIDMach.tools.{SeqToSeqData,SeqToSeqDict,printmat} + +// Example of how to use SeqToSeqData + +// Options: +val opts = new SeqToSeqData.Options; +opts.srcvocabmaxsize = 150000; // Max vocabulary size. If <= 0, no maxsize performed. If >= 1, must provide dict name. +opts.dstvocabmaxsize = 80000; // Max vocabulary size. If <= 0, no maxsize performed. If >= 1, must provide dict name. +opts.srcminlen = 1; // Minimum sentence length, discard shorter sentences +opts.srcmaxlen = 12; // Maximum sentence length, truncate longer sentences +opts.dstminlen = 1; // Minimum sentence length, discard shorter sentences +opts.dstmaxlen = 12; // Maximum sentence length, truncate longer sentences +opts.revsrc = true; // Reverse the src sentences +opts.revdst = false; // Reverse the dst sentences + + +// Make data +val sd = new SeqToSeqData(opts); +sd.prepSeqToSeqDataWildcard("/path/to/dir/", + ("src_*.imat","dst_*.imat"), + (("srcdict.sbmat","srcdict.imat"),("dstdict.sbmat","dstdict.imat"))) \ No newline at end of file diff --git a/src/main/scala/BIDMach/tools/SeqToSeqData.scala b/src/main/scala/BIDMach/tools/SeqToSeqData.scala new file mode 100644 index 00000000..a1ff8b9c --- /dev/null +++ b/src/main/scala/BIDMach/tools/SeqToSeqData.scala @@ -0,0 +1,533 @@ +package BIDMach.tools + +import BIDMat.{Mat,SBMat,CMat,CSMat,Dict,DMat,FMat,IDict,IMat,HMat,GMat,GIMat,GSMat,SMat,SDMat} +import BIDMat.MatFunctions._ +import BIDMat.SciFunctions._ +import java.io.File + +/** + * @author bjaros + * + * Prepare data for consumption by SeqToSeq LSTMs. + * + * The SeqToSeqData class processes a pair of parsed sentences, each in IMat format, and produces SMat + * outputs of length-collated sentences for SeqToSeq training. + * + * Parsed sentences are IMat of format (e.g. the result of running nnparse.exe): + * p1 s1 w1 + * p2 s2 w2 + * p3 s3 w3 + * p4 s4 w4 + * p5 s5 w5 + * p6 s6 w6 + * e.g. + * 0 0 96 + * 0 0 17 + * 0 0 23 + * 1 0 7 + * 1 0 31 + * 2 0 86 + * would be first sentence with id's "<96> <17> <23>" and second sentence with + * id's "7 31" and third sentence with id's "<86>". + * + * (For SeqToSeq we assume each line contains one sentence, so the paragraphid (the first column) + * denotes the sentence and sentenceid (the second column) is always ignored). + * + * The two parsed sentence IMats are paired line-by-line: the ith line of the src IMat corresponds + * to the ith line of the dst IMat. + * + * The output is two SMat's of the following form: + * + * w00 w01 w02 w03 w04 w05 ... + * w10 w11 w12 w13 w14 w15P ... + * w20 w21 w22 w23P w24 w25P ... + * w30 w31P w32 ... + * w40P w32P w33 ... + * + * where + * wij is the dictionary index of the i'th word in the j'th sentence and + * words with a P suffix are padding symbols. + * + * The columns of the two output SMat's are still paired: column j of the src output SMat and + * column j of the dst output SMat correspond to line j of the src input and line j of the dst input + * respectively. + * + * Furthermore, the sentences are collated into batches of similar lengths. + * + * The minibatches are randomly permuted after collation to avoid training bias. + * + * Use opts.srcvocabmincount & dstvocabmincount to trim the dictionary to a minimum count (and update + * the output matrices correspondingly). + * Use opts.srcvocabmaxsize & dstvocabmaxsize to trim the dictionary to a maximum size (and update + * the output matricescorrespondingly). + * + * The dictionaries corresponding to the output matrices are saved to the outputdir as well. + * + * LIMITATIONS: + * + * Since float values are used to hold word ids, the maximum dictionary size is 16M. Use SDMat if this is a problem. + */ + +object SeqToSeqDict { + val specialsyms = cscol("","","",""); + val padsym = 1; // Index of special padding symbol + val eossym = 2; // Index of special end-of-sentence symbol + val oovsym = 3; // Index of out-of-vocabulary symbol + val numspecialsyms = specialsyms.length; + + def apply(csmat:CSMat):Dict = { + val cs0 = if (csmat(padsym)==specialsyms(padsym)) { // Already SeqToSeqDict? + csmat; + } else { // Add specialsyms + specialsyms on csmat; + } + val out = new Dict(cs0); + return out; + } + + def apply(sbmatpath:String):Dict = { + val cs = CSMat(loadSBMat(sbmatpath)); + return SeqToSeqDict(cs); + } + + def apply(fpaths:(String,String)):Dict = { + /* + * Load dict with counts. + * Input is tuple of path to dict (an sbmat) and path to dict counts (an imat). + */ + val sbmatfpath = fpaths._1; + val imatfpath = fpaths._2; + val out = SeqToSeqDict(sbmatfpath); + if (imatfpath != null) { + val dictcnt = loadIMat(imatfpath); + out.counts = if (dictcnt(0)==Double.MaxValue) { // Already SeqToSeqDict? + DMat(dictcnt); + } else { + Double.MaxValue*ones(numspecialsyms,1) on DMat(dictcnt); // TODO correct MaxValue? + } + } + return out; + } + + def top(dict:Dict, maxsize:Int):Dict = { + /* + * Return a dict with the top #maxsize tokens by count + */ + val (ss, ii) = sortrows(dict.counts(numspecialsyms->dict.length).t); // Sort the dict counts of non-special symbols + val ii2 = ii((ii.length-1) to (ii.length-maxsize) by -1); // Take the top maxsize counts. Reverse order for convenience when inspecting. + ii2 ~ ii2 + numspecialsyms; // Offset those indices to again count for special symbols. + val cstr = specialsyms on dict.cstr(ii2.t); // Recreate the cstr + val cnt = Double.MaxValue*ones(numspecialsyms,1) on dict.counts(ii2.t); // Recreate the counts + Dict(cstr, cnt) + } +} + +object printmat { + /* + * Utility to print out the words corresponding to given indices in an IMat or SMat. + * Each column is assumed to be a sentence. + */ + def apply(mat:IMat, dict:Dict):Unit = { + /* + * Translate the indices in IMat into words and print out sentences (each column is a sentence). + */ + apply(sparse(mat),dict); + } + + def apply(mat:SMat, dict:Dict, maxcols:Int=100):Unit = { + /* + * Translate the indices in integer SMat into words and print out sentences (each column is a sentence). + */ + for (j <- 0 until mat.ncols) { + var i = 0; + var rowdone = false; + print("[%d] " format j); + while ((i < mat.nrows) & !rowdone) { + val indx:Int = mat(i,j).toInt; + if (indx==0) { + rowdone = true; + } else { + val token:String = dict(indx); + print("%s (%d) " format (token,indx)); + } + i += 1; + } + println("") + if (j==maxcols-1) { + println("[Truncated printing at %d cols. Adjust using maxcols argument to printmat]" format maxcols); + return; + } + } + } +} + +class SeqToSeqData(val opts:SeqToSeqData.Opts = new SeqToSeqData.Options) { + var dictinitialized:Boolean=false; + var srcdict:Dict=null; + var dstdict:Dict=null; + var srctrimmapping:IMat=null; + var dsttrimmapping:IMat=null; + + def getStartsAndLens(parsedSents:IMat):(IMat,IMat) = { + /* + * Given parsedSents IMat of format described above: + * e.g. + * 0 0 96 + * 0 0 17 + * 0 0 23 + * 1 0 7 + * 1 0 31 + * 2 0 86 + * + * Return starts, a [nsents x 1] matrix: + * 0 + * 3 + * 5 + * + * and lens, also a [nsents x 1] matrix + * 3 + * 2 + * 1 + * + * Also filters out sentences shorter than opts.minlen. + */ + val starts0 = 0 on 1+find(parsedSents(0->(parsedSents.nrows-1),opts.sentcol) != + parsedSents((1->parsedSents.nrows),opts.sentcol)); // Starting indices. [nbatches x 1] + val posts0 = starts0 on parsedSents.nrows; + val lens0 = posts0(1->posts0.nrows) - posts0(0->(posts0.nrows-1)); + // We would be done here, except that there could be zero-length sentences which would have + // been skipped altogether. We want to make sure sentence #i is in the ith slot of starts & lens. + val senti = parsedSents(starts0,opts.sentcol); + val numsents = 1+maxi(parsedSents(?,opts.sentcol))(0); + val starts = izeros(numsents,1); + starts(senti) = starts0; + val lens = izeros(1,numsents); + lens(0,senti) = lens0; + return (starts,lens) + +// val starts = 0 on 1+find(parsedSents(0->(parsedSents.nrows-1),opts.sentcol) != +// parsedSents((1->parsedSents.nrows),opts.sentcol)); // Starting indices. [nbatches x 1] +// val posts = starts on parsedSents.nrows; +// val lens = posts(1->posts.nrows) - posts(0->(posts.nrows-1)); +// return (starts,lens) + } + + def getoutputdir(inputdir:String, outputdir0:String=null):String = { + /* + * Return the outputdir (or default to inputdir+"/output" if none provided), mkdir'ing + * if necessary. + */ + val outputdir = if (outputdir0==null) { + val suffix = "%s%s%s%s_sl%d-%d_dl%d-%d_b%d%s%s" format ( + if (opts.srcvocabmaxsize > 0) "_s%d" format opts.srcvocabmaxsize else "", + if (opts.srcvocabmincount > 1) "_smin%d" format opts.srcvocabmincount else "", + if (opts.dstvocabmaxsize > 0) "_d%d" format opts.dstvocabmaxsize else "", + if (opts.dstvocabmincount > 1) "_dmin%d" format opts.dstvocabmincount else "", + opts.srcminlen, opts.srcmaxlen, opts.dstminlen, opts.dstmaxlen, opts.bsize, + if (opts.revsrc) "_revsrc" else "", if (opts.revdst) "_revdst" else ""); + inputdir+"/../seq2seq%s" format suffix + } else { + outputdir0; + } + // Make outputdir + val outputdirFile = new File(outputdir); + if (!outputdirFile.exists()) { + val successful = outputdirFile.mkdirs(); + assert(successful, "Failed to make outputdir at %s" format outputdir); + } + return outputdir; + } + + + def loadDict(dictfpaths:(String,String), outputdir:String=null, vocabmaxsize:Int=0, vocabmincount:Int=0):(Dict,IMat) = { + /* + * Given a tuple of paths to dict sbmat and (optionally) imat of counts, + * 1) load the dictionary + * 2) optionally trim the dictionary based on vocabmaxsize + * 3) if outputdir provided, save the dictionary to outputdir + * 4) return the dictionary and optionally (based on (2)), the mapping from the original indices + * to the trimmed indices + */ + if (dictfpaths._1 == null) { + assert(vocabmaxsize <= 0, "If vocabmaxsize > 0, you must provide a dict and counts"); + assert(vocabmincount <= 1, "If vocabmincount > 1, you must provide a dict and counts"); + return (null,null); + } else { + var origDict = SeqToSeqDict(dictfpaths); + var needmap = false; + var dict:Dict = origDict; + if (vocabmincount > 1) { + needmap = true; + dict = origDict.trim(vocabmincount); + } + if ((vocabmaxsize > 0) && (vocabmaxsize < origDict.length)) { + needmap = true; + dict = SeqToSeqDict.top(dict, vocabmaxsize); + } + val trimmapping = if (needmap) { + origDict --> dict; + } else { + null; + } + // Save dict to outputdir (with same filenames) + if (outputdir!=null) { + val sbmatfname = (new File(dictfpaths._1)).getName; + val imatfname = (new File(dictfpaths._2)).getName; + val sbmatfpath = outputdir+"/"+sbmatfname; + val imatfpath = outputdir+"/"+imatfname; + println("Saving dicts to %s and %s" format (sbmatfpath,imatfpath)); + saveSBMat(sbmatfpath, SBMat(dict.cstr)); + saveIMat(imatfpath, IMat(dict.counts)); + } + return (dict,trimmapping) + } + } + + def loadData(fpath:String, trimmapping:IMat=null):IMat = { + /* + * 1) Load the (imat) data from fpath + * 2) If trimmapping provided, map the indices due to trimming and fill in oovsym + * 3) Offset to make room for special characters + */ + val parsedSents = loadIMat(fpath); + parsedSents(?,opts.wordcol) += SeqToSeqDict.numspecialsyms - 1; + if (trimmapping!=null) { // trimming + parsedSents(?,opts.wordcol) = trimmapping(parsedSents(?,opts.wordcol)); + val ii = find(parsedSents(?,opts.wordcol)<0); + parsedSents(ii,opts.wordcol) = SeqToSeqDict.oovsym; + } + return parsedSents; + } + + def prepSeqToSeqDataWildcard(inputdir:String, fnamepatterns:(String,String)):Unit = { + prepSeqToSeqDataWildcard(inputdir, fnamepatterns, ((null,null),(null,null)), null); + } + + def prepSeqToSeqDataWildcard(inputdir:String, fnamepatterns:(String,String), + outputdir0:String):Unit = { + prepSeqToSeqDataWildcard(inputdir, fnamepatterns, ((null,null),(null,null)), outputdir0); + } + + def prepSeqToSeqDataWildcard(inputdir:String, fnamepatterns:(String,String), + dictfnames:((String,String),(String,String))=((null,null),(null,null)), + outputdir0:String=null):Unit = { + /* + * patterns: a tuple of 2 strings (1 for src pattern, 1 for dstpattern) with single asterisk + * denoting wildcard. + */ + val files = new File(inputdir).listFiles; + assert(files!=null,"No directory %s" format inputdir) + val allfnames = files.map(_.getName).sorted; + + val srcpattern = fnamepatterns._1; + val srcparts = srcpattern.split("\\*"); + val srcfiles0 = allfnames.filter(_.startsWith(srcparts(0))); + val srcfiles = if (srcparts.length==2) srcfiles0.filter(_.endsWith(srcparts(1))) else srcfiles0; + + val dstpattern = fnamepatterns._2; + val dstparts = dstpattern.split("\\*"); + val dstfiles0 = allfnames.filter(_.startsWith(dstparts(0))); + val dstfiles = if (dstparts.length==2) dstfiles0.filter(_.endsWith(dstparts(1))) else dstfiles0; + + val outputdir = getoutputdir(inputdir,outputdir0); + + assert(srcfiles.length==dstfiles.length); + for (ifile <- 0 until srcfiles.length) { + // try { + val srcfname = srcfiles(ifile); + val dstfname = dstfiles(ifile); + println("Processing %s <--> %s" format (srcfname,dstfname)); + val (srcmat,dstmat) = prepSeqToSeqData(inputdir, (srcfname, dstfname), dictfnames, outputdir); + // Save + saveSMat(outputdir + "/src%04d.smat.lz4" format ifile, srcmat); + saveSMat(outputdir + "/dst%04d.smat.lz4" format ifile, dstmat); + // } + // catch { + // case _: Exception => {println("problem with file %d" format ifile)} + // case _: Throwable => {println("problem with file %d" format ifile)} + // } + } + } + + def prepSeqToSeqData(inputdir:String, fnames:(String,String)):(SMat,SMat) = { + prepSeqToSeqData(inputdir, fnames, ((null,null),(null,null)), null) + } + + def prepSeqToSeqData(inputdir:String, fnames:(String,String), outputdir0:String):(SMat,SMat) = { + prepSeqToSeqData(inputdir, fnames, ((null,null),(null,null)), outputdir0) + } + + def prepSeqToSeqData(inputdir:String, fnames:(String,String), dictfnames:((String,String),(String,String)), + outputdir0:String=null):(SMat,SMat) = { + val srcpath:String = inputdir+"/"+fnames._1; + val dstpath:String = inputdir+"/"+fnames._2; + val srcdictfpath:String = if (dictfnames._1._1==null) null else inputdir+"/"+dictfnames._1._1; + val srcdictcntfpath:String = if (dictfnames._1._2==null) null else inputdir+"/"+dictfnames._1._2; + val dstdictfpath:String = if (dictfnames._2._1==null) null else inputdir+"/"+dictfnames._2._1; + val dstdictcntfpath:String = if (dictfnames._2._2==null) null else inputdir+"/"+dictfnames._2._2; + val outputdir = getoutputdir(inputdir,outputdir0); + prepSeqToSeqData((srcpath,dstpath), ((srcdictfpath,srcdictcntfpath),(dstdictfpath,dstdictcntfpath)), outputdir); + } + + def prepSeqToSeqData(fpaths:(String,String), dictfpaths:((String,String),(String,String)), + outputdir:String):(SMat,SMat) = { + if (!dictinitialized) { + val res1 = loadDict(dictfpaths._1, outputdir, opts.srcvocabmaxsize, opts.srcvocabmincount); + srcdict = res1._1; srctrimmapping = res1._2; + val res2 = loadDict(dictfpaths._2, outputdir, opts.dstvocabmaxsize, opts.dstvocabmincount); + dstdict = res2._1; dsttrimmapping = res2._2; + dictinitialized=true; + } + val srcsents:IMat = loadData(fpaths._1,srctrimmapping); + val dstsents:IMat = loadData(fpaths._2,dsttrimmapping); + + val (srcstartsAll,srclensAll) = getStartsAndLens(srcsents); + val (dststartsAll,dstlensAll) = getStartsAndLens(dstsents); + + // Filter sentences where the src or dst isn't long enough + val numsents = math.min(srclensAll.length, dstlensAll.length); // In case src/dst finished with empty (length-0) sentences + val iigmonotonic = find((srclensAll >= opts.srcminlen)(0->numsents) *@ + (dstlensAll >= opts.dstminlen)(0->numsents)) // Check min length threshold. [nsents x 1] + val iig = if (opts.maintainordering) { + iigmonotonic; + } else { + iigmonotonic(randperm(iigmonotonic.length).t); // Randomize. Otherwise sorting would be monotonic over sentences with ties in lengths. [nsents x 1] + } + + // Make starts and lens mats + val srcstarts = srcstartsAll(iig); + val srclens = srclensAll(iig); + srclens(find(srclens>opts.srcmaxlen)) = opts.srcmaxlen; + val dststarts = dststartsAll(iig); + val dstlens = dstlensAll(iig); + dstlens(find(dstlens>opts.dstmaxlen)) = opts.dstmaxlen; + + // Sort by lengths (unless opts.maintainordering) + val (ss, ii) = if (opts.maintainordering) { + (null, icol(0->srclens.length)) // Original order + } else { + val sortbylens = maxi(dstlens) * srclens + dstlens; // primary sort by srclens; secondary sort by dstlens + sortrows(sortbylens); // lex sort the length pairs and get permutation indices in ii. ii: [sents x 1] + } + + val nsents = srcstarts.size; + val nbatches = nsents / opts.bsize; + val nsents2 = opts.bsize * (nsents / opts.bsize); // Round length to a multiple of batch size + val ii2 = ii((nsents - nsents2)->nsents); // Drop the shortest sentences, giving a multiple of batch size. [1 x nsents2] + val i2dsorted = ii2.view(opts.bsize, nbatches).t; // Put inds in a 2d matrix, with columns which are minibatches. [nbatches x bsize] + + if (opts.maintainordering) { + assert(nsents2 == nsents, "%d != %d. If opts.maintainordering, you want the bsize to divide evenly into nsents (%d)" format (nsents,nsents2,nsents)) + } + + val i2d = if (opts.maintainordering) { + i2dsorted + } else { + val ip = randperm(nbatches); + i2dsorted(ip,?); // Randomly permute the minibatches. *Sentence indices divided into batches.* [nbatches x bsize] + } + + val srcmat = mkmat(srcsents, srcstarts, srclens, i2d, opts.revsrc, true); + val dstmat = mkmat(dstsents, dststarts, dstlens, i2d, opts.revdst, false); + return (srcmat,dstmat); + } + + def mkmat(parsedSents:IMat, starts:IMat, lens:IMat, i2d:IMat, rev:Boolean=false, rightjustify:Boolean=false):SMat = { + /* + * parsedSents: in format described above. + * starts: the starting index (in parsedSents) of each sentence. [nsents x 1] + * lens: the length of each sentence in parsedSents. [nsents x 1] + * i2d: the sentence index for each location in the batch, [nbatches x bsize] + * rev: whether to reverse the order of the word indices in the sentence + * rightjustify: whether to have all sentences end on the last column of the batch matrix + * (default of false, i.e. leftjustify, has all sentences start on the first col of the batch matrix) + * + * 5 6 7 8, with n=6, padsym=1 --> + * rev=0 rightjustify=0 5 6 7 8 1 1 + * rev=0 rightjustify=1 1 1 5 6 7 8 + * rev=1 rightjustify=0 8 7 6 5 1 1 + * rev=1 rightjustify=1 1 1 8 7 6 5 + */ + val starts2d = starts(i2d); // Sentence start indices arranged by minibatch. [nbatches x bsize] + val lens2d = lens(i2d); // Sentence lengths arranged by minibatch. [nbatches x bsize] + val maxlen = maxi(lens2d,2); // Max length in each minibatch - others get padded to this. [nsents2 x 1] + val nnz = sum(maxlen).v*opts.bsize; // Number of non-zeros in matrix. + + val nsents = starts.size; + val nbatches = nsents / opts.bsize; + val nsents2 = opts.bsize * (nsents / opts.bsize); // Round length to a multiple of batch size + + // Prepare output + val i = izeros(nnz, 1); // row, col, val matrices for the final SMat + val j = izeros(nnz, 1); + val v = zeros(nnz, 1); + var p = 0; + + var ibatch = 0; + var longest = 0; + while (ibatch < nbatches) { + val n = maxlen(ibatch); // max length for this minibatch + if (n>longest) longest=n; + + val blk = izeros(n, opts.bsize); + val thisstarts = starts2d(ibatch,?); // Start index for each sentence of batch. [1 x bsize] + val thislens = lens2d(ibatch,?); // Length for each sentence of batch. [1 x bsize] + if (rightjustify ^ rev) { // rightjustify + rev is the outcome of reversing leftjustify + val thisends = thisstarts + thislens - 1; + var posi = 0; + while (posi < n) { // Step through each position in sentence + val validi = (thislens > posi); // (only for sentences where position < its length). [1 x bsize] + val ii = (thisends - posi) *@ validi; // Indices in parsedSents + val vals = (parsedSents(ii,opts.wordcol).t - SeqToSeqDict.padsym) *@ validi; // Values from parsed sents (offset by padsym, so we can add back in next step). [1 x bsize] + blk(n-1-posi,?) = vals + SeqToSeqDict.padsym; + posi += 1; + } + } else { + var posi = 0; + while (posi < n) { // Step through each position in sentence + val validi = (thislens > posi); // (only for sentences where position < its length). [1 x bsize] + val ii = (thisstarts + posi) *@ validi; // Indices in parsedSents + val vals = (parsedSents(ii,opts.wordcol).t - SeqToSeqDict.padsym) *@ validi; // Values from parsed sents (offset by padsym, so we can add back in next step). [1 x bsize] + blk(posi,?) = vals + SeqToSeqDict.padsym; + posi += 1; + } + } + if (rev) { // Reverse the sentences + val revinds = icol((n-1) to 0 by -1); + blk <-- blk(revinds,?); + } + + val (ii, jj, vv) = find3(blk); // back to sparse indices. [nnz x 1] == [n*bsize x 1] + val ilen = ii.length; + i(p->(p+ilen),0) = ii; // Add the src data to the global buffers + j(p->(p+ilen),0) = jj + ibatch*opts.bsize; // Offset the column indices appropriately + v(p->(p+ilen),0) = vv; + p += ilen; + + ibatch += 1; + } + val mat = sparse(i, j, v, longest, nsents2); + return mat; + } +} + +object SeqToSeqData { + trait Opts { + var srcvocabmaxsize = -1; // Max vocabulary size. If <= 0, no maxsize performed. If >= 1, must provide dict name. + var srcvocabmincount = 0; // Vocabulary minimum counts. If <= 1, no trimming performed. If > 1, must provide dict name. + var srcmaxlen = 40; // Maximum sentence length, truncate longer sentences + var srcminlen = 1; // Minimum sentence length, discard shorter sentences + var dstvocabmaxsize = -1; // Max vocabulary size. If <= 0, no maxsize performed. If >= 1, must provide dict name. + var dstvocabmincount = 0; // Vocabulary minimum counts. If <= 1, no trimming performed. If > 1, must provide dict name. + var dstmaxlen = 40; // Maximum sentence length, truncate longer sentences + var dstminlen = 1; // Minimum sentence length, discard shorter sentences + var revsrc = true; // Reverse the src sentences + var revdst = false; // Reverse the dst sentences + var maintainordering = false; // Output matrix columns are in same order as input + var bsize = 128; // Batch size + var sentcol = 0; // Column of input parsed data containing sentence ids + var wordcol = 2; // Column of input parsed data containing word ids + } + + class Options extends Opts {} +} \ No newline at end of file From 9a1633fd580a84358e754566df440eb76dd7da6c Mon Sep 17 00:00:00 2001 From: Bobby Jaros Date: Fri, 18 Dec 2015 14:07:16 -0800 Subject: [PATCH 3/6] Small documentation edits Notes accumulated as I read through code. Also renamed in_gate2 to g in LSTM.scala, for clarity. --- src/main/scala/BIDMach/models/Model.scala | 1 + src/main/scala/BIDMach/networks/Net.scala | 1 + .../scala/BIDMach/networks/SeqToSeq.scala | 8 ++-- .../scala/BIDMach/networks/layers/LSTM.scala | 39 ++++++++++++------- .../scala/BIDMach/networks/layers/Layer.scala | 2 +- .../networks/layers/SoftmaxOutputLayer.scala | 2 + src/main/scala/BIDMach/updaters/Grad.scala | 2 +- 7 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/main/scala/BIDMach/models/Model.scala b/src/main/scala/BIDMach/models/Model.scala index 381edc2f..7b4a1a47 100755 --- a/src/main/scala/BIDMach/models/Model.scala +++ b/src/main/scala/BIDMach/models/Model.scala @@ -162,6 +162,7 @@ abstract class Model(val opts:Model.Opts = new Model.Options) extends Serializab def evalbatch(mats:Array[Mat], ipass:Int, here:Long):FMat // Scores (log likelihoods) def dobatchg(amats:Array[Mat], ipass:Int, here:Long) = { + // If useGPU==false then gmats is an alias to amats if (useGPU) copyMats(amats, gmats); dobatch(gmats, ipass, here); } diff --git a/src/main/scala/BIDMach/networks/Net.scala b/src/main/scala/BIDMach/networks/Net.scala index cd5c360f..b88b2d9b 100644 --- a/src/main/scala/BIDMach/networks/Net.scala +++ b/src/main/scala/BIDMach/networks/Net.scala @@ -133,6 +133,7 @@ class Net(override val opts:Net.Opts = new Net.Options) extends Model(opts) { } } + // no backward pass def evalbatch(mats:Array[Mat], ipass:Int, pos:Long):FMat = { if (batchSize < 0) batchSize = gmats(0).ncols; if (batchSize == gmats(0).ncols) { diff --git a/src/main/scala/BIDMach/networks/SeqToSeq.scala b/src/main/scala/BIDMach/networks/SeqToSeq.scala index bb1cbed1..3775cba7 100644 --- a/src/main/scala/BIDMach/networks/SeqToSeq.scala +++ b/src/main/scala/BIDMach/networks/SeqToSeq.scala @@ -44,8 +44,8 @@ class SeqToSeq(override val opts:SeqToSeq.Opts = new SeqToSeq.Options) extends N height = opts.height; heightDiff = if (opts.netType == 0) 2 else 1; fullheight = height + preamble_rows; - inwidth = opts.inwidth; - outwidth = opts.outwidth; + inwidth = opts.inwidth; // aka srcwidth + outwidth = opts.outwidth; // aka dstwidth width = inwidth + outwidth; layers = new Array[Layer](fullheight * width + outwidth * heightDiff); leftedge = InputLayer(this); // dummy layer, left edge of zeros @@ -273,7 +273,7 @@ class SeqToSeq(override val opts:SeqToSeq.Opts = new SeqToSeq.Options) extends N object SeqToSeq { trait Opts extends Net.Opts { var inwidth = 1; // Max src sentence length - var outwidth = 1; // Max dst sentence lenth + var outwidth = 1; // Max dst sentence length var height = 1; // Number of LSTM layers vertically var nvocab = 100000; // Vocabulary size var kind = 0; // LSTM type, see below @@ -290,7 +290,7 @@ object SeqToSeq { class Options extends Opts {} - def mkNetModel(fopts:Model.Opts) = { + def mkNetModel(fopts:Model.Opts) = { new SeqToSeq(fopts.asInstanceOf[SeqToSeq.Opts]) } diff --git a/src/main/scala/BIDMach/networks/layers/LSTM.scala b/src/main/scala/BIDMach/networks/layers/LSTM.scala index 8b756f80..8ba0cf37 100644 --- a/src/main/scala/BIDMach/networks/layers/LSTM.scala +++ b/src/main/scala/BIDMach/networks/layers/LSTM.scala @@ -71,9 +71,9 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { val il4 = new LinNode{inputs(0) = i; modelName = prefix + "LSTM_input_tanh"; hasBias = this.hasBias}; val ph4 = new LinNode{inputs(0) = prev_h; modelName = prefix + "LSTM_prev_h_tanh"; hasBias = this.hasBias}; val sum4 = new AddNode{inputs(0) = il4; inputs(1) = ph4}; - val in_gate2 = new TanhNode{inputs(0) = sum4}; + val g = new TanhNode{inputs(0) = sum4}; - val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = in_gate2}; + val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = g}; val f_prod = new MulNode{inputs(0) = forget_gate; inputs(1) = prev_c}; val next_c = new AddNode{inputs(0) = in_prod; inputs(1) = f_prod}; @@ -84,7 +84,7 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { il1, ph1, sum1, in_gate, // Otherwise the ordering should support forward-backward inference il2, ph2, sum2, out_gate, il3, ph3, sum3, forget_gate, - il4, ph4, sum4, in_gate2, + il4, ph4, sum4, g, in_prod, f_prod, next_c, next_tanh, next_h); @@ -92,7 +92,13 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { outputNumbers = Array(lopts.length-1, lopts.length-3); // Specifies the output layer numbers (next_h and next_c) } - // LSTM with all layers grouped into 1 - not very stable to train + // LSTM with all linear weights grouped into single matrix, to more fully use GPU. + // Observed not very stable to train + // + // | Wi_in Wh_in | + // | Wi_out Wh_out | | i | + // | Wi_forget Wh_forget | | h | + // | Wi_g Wh_g | def constructGraph1 = { val prev_h = new CopyNode; @@ -106,9 +112,9 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { val in_gate = new SigmoidNode{inputs(0) = sp(0)}; val out_gate = new SigmoidNode{inputs(0) = sp(1)}; val forget_gate = new SigmoidNode{inputs(0) = sp(2)}; - val in_gate2 = new TanhNode{inputs(0) = sp(3)}; + val g = new TanhNode{inputs(0) = sp(3)}; - val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = in_gate2}; + val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = g}; val f_prod = new MulNode{inputs(0) = forget_gate; inputs(1) = prev_c}; val next_c = new AddNode{inputs(0) = in_prod; inputs(1) = f_prod}; @@ -120,7 +126,7 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { in_gate, // Otherwise the ordering should support forward-backward inference out_gate, forget_gate, - in_gate2, + g, in_prod, f_prod, next_c, next_tanh, next_h); @@ -129,6 +135,8 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { } // LSTM with 4 linear layers, with h and i stacked as inputs + // | Wi_in Wh_in | | i | + | Wi_out Wh_out | | i | + | Wi_forget Wh_forget | | i | + | Wi_g Wh_g | | i | + // | h | | h | | h | | h | def constructGraph2 = { val prev_h = new CopyNode; @@ -147,9 +155,9 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { val forget_gate = new SigmoidNode{inputs(0) = lin3}; val lin4 = new LinNode{inputs(0) = prev_hi; modelName = prefix + "LSTM_tanh"; outdim = dim; hasBias = this.hasBias} - val in_gate2 = new TanhNode{inputs(0) = lin4}; + val g = new TanhNode{inputs(0) = lin4}; - val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = in_gate2}; + val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = g}; val f_prod = new MulNode{inputs(0) = forget_gate; inputs(1) = prev_c}; val next_c = new AddNode{inputs(0) = in_prod; inputs(1) = f_prod}; @@ -161,7 +169,7 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { lin1, in_gate, // Otherwise the ordering should support forward-backward inference lin2, out_gate, lin3, forget_gate, - lin4, in_gate2, + lin4, g, in_prod, f_prod, next_c, next_tanh, next_h); @@ -169,7 +177,10 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { outputNumbers = Array(lopts.length-1, lopts.length-3); // Specifies the output layer numbers (next_h and next_c) } - // LSTM with two sets of layers, paired outputs. More stable to train than the single linlayer network + // LSTM with two sets of linear weights, paired outputs. More stable to train than the single linlayer network + // + // | Wi_in Wh_in | | i | + | Wi_forget Wh_forget | | i | + // | Wi_out Wh_out | | h | | Wi_g Wh_g | | h | def constructGraph3 = { val prev_h = new CopyNode; @@ -185,9 +196,9 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { val in_gate = new SigmoidNode{inputs(0) = sp1(0)}; val out_gate = new SigmoidNode{inputs(0) = sp1(1)}; val forget_gate = new SigmoidNode{inputs(0) = sp2(0)}; - val in_gate2 = new TanhNode{inputs(0) = sp2(1)}; + val g = new TanhNode{inputs(0) = sp2(1)}; - val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = in_gate2}; + val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = g}; val f_prod = new MulNode{inputs(0) = forget_gate; inputs(1) = prev_c}; val next_c = new AddNode{inputs(0) = in_prod; inputs(1) = f_prod}; @@ -199,7 +210,7 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { in_gate, // Otherwise the ordering should support forward-backward inference out_gate, forget_gate, - in_gate2, + g, in_prod, f_prod, next_c, next_tanh, next_h); diff --git a/src/main/scala/BIDMach/networks/layers/Layer.scala b/src/main/scala/BIDMach/networks/layers/Layer.scala index 6d409297..1e3da088 100644 --- a/src/main/scala/BIDMach/networks/layers/Layer.scala +++ b/src/main/scala/BIDMach/networks/layers/Layer.scala @@ -37,7 +37,7 @@ import BIDMach.networks._ * * * - * Currently only four Layer types need params: + * Currently only four Layer types need function parameters: - LinLayer: "outside" holds the output dimensions of the FClayer (input dimension set by previous layer). - GLMLayer: "links" holds the links matrix (integer optss for loss types, see GLM), for the output of that layer. Its size should match the number of targets. - NormLayer: "targetNorm" holds a target per-element norm, and "weight" is the weight for this term in the derivative calculation. diff --git a/src/main/scala/BIDMach/networks/layers/SoftmaxOutputLayer.scala b/src/main/scala/BIDMach/networks/layers/SoftmaxOutputLayer.scala index 1e3d4ca9..8e7d0b0f 100644 --- a/src/main/scala/BIDMach/networks/layers/SoftmaxOutputLayer.scala +++ b/src/main/scala/BIDMach/networks/layers/SoftmaxOutputLayer.scala @@ -17,8 +17,10 @@ import BIDMach.networks._ /** * Softmax layer. Output = exp(input) / sum(exp(input)) + * Differs from SoftmaxLayer in that it includes a score method. */ + class SoftmaxOutputLayer(override val net:Net, override val opts:SoftmaxOutputNodeOpts = new SoftmaxOutputNode) extends Layer(net, opts) with OutputLayer { var coloffsets:Mat = null; var zero:Mat = null; diff --git a/src/main/scala/BIDMach/updaters/Grad.scala b/src/main/scala/BIDMach/updaters/Grad.scala index 3546bad6..77a87c22 100755 --- a/src/main/scala/BIDMach/updaters/Grad.scala +++ b/src/main/scala/BIDMach/updaters/Grad.scala @@ -124,7 +124,7 @@ object Grad { var pexp:FMat = 0.5f var waitsteps = 3 var mask:FMat = null - var policies:Array[(Float, Float)=>Float] = null + var policies:Array[(Float, Float)=>Float] = null // One policy for each model mat var momentum:FMat = null var nesterov:FMat = null var langevin = 0f; From 442c1ba01c03574b1aa39d82ca3eaed32350c4b3 Mon Sep 17 00:00:00 2001 From: Bobby Jaros Date: Tue, 8 Mar 2016 15:40:44 -0800 Subject: [PATCH 4/6] Bad merge, fixing these two files --- .../scala/BIDMach/networks/SeqToSeq.scala | 723 ++++++++--------- .../scala/BIDMach/networks/layers/LSTM.scala | 735 +++++++++--------- 2 files changed, 717 insertions(+), 741 deletions(-) diff --git a/src/main/scala/BIDMach/networks/SeqToSeq.scala b/src/main/scala/BIDMach/networks/SeqToSeq.scala index 7eb1845c..92ca8d5e 100644 --- a/src/main/scala/BIDMach/networks/SeqToSeq.scala +++ b/src/main/scala/BIDMach/networks/SeqToSeq.scala @@ -1,385 +1,340 @@ -package BIDMach.networks - -import BIDMat.{Mat,SBMat,CMat,CSMat,DMat,FMat,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,SMat,SDMat} -import BIDMat.MatFunctions._ -import BIDMat.SciFunctions._ -import BIDMach.datasources._ -import BIDMach.updaters._ -import BIDMach.mixins._ -import BIDMach.models._ -import BIDMach.networks.layers._ -import BIDMach._ - -/* +package BIDMach.networks + +import BIDMat.{Mat,SBMat,CMat,CSMat,DMat,FMat,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,SMat,SDMat} +import BIDMat.MatFunctions._ +import BIDMat.SciFunctions._ +import BIDMach.datasources._ +import BIDMach.datasinks._ +import BIDMach.updaters._ +import BIDMach.mixins._ +import BIDMach.models._ +import BIDMach.networks.layers._ +import BIDMach._ + +/* * LSTM next Word prediction model, which comprises a rectangular grid of LSTM compound layers. - */ -class SeqToSeq(override val opts:SeqToSeq.Opts = new SeqToSeq.Options) extends Net(opts) { - - var PADrow:Mat = null; - var OOVelem:Mat = null; - var leftedge:Layer = null; - var leftStart:Mat = null; - var dstxdata:Mat = null; - var dstxdata0:Mat = null; - var srcGrid:LayerMat = null; - var dstGrid:LayerMat = null; - var srcNodeGrid:NodeMat = null; - var dstNodeGrid:NodeMat = null; - var height = 0; - var fullheight = 0; - var inwidth = 0; - var outwidth = 0; - var width = 0; - var srcn = 0; - var dstxn = 0; - var dstyn = 0; - val preamble_rows = 2; - // define some getters/setters on the grid - var heightDiff = 2; - def lindex(r:Int, c:Int) = if (c < inwidth) (r + c * fullheight) else (inwidth * fullheight + r + (c - inwidth) * (fullheight + heightDiff)); - def getlayer(r:Int, c:Int):Layer = layers(lindex(r,c)); - def setlayer(r:Int, c:Int, ll:Layer) = {layers(lindex(r,c)) = ll}; - - override def createLayers = { - height = opts.height; - heightDiff = if (opts.netType == 0) 2 else 1; - fullheight = height + preamble_rows; - inwidth = opts.inwidth; // aka srcwidth - outwidth = opts.outwidth; // aka dstwidth - width = inwidth + outwidth; - layers = new Array[Layer](fullheight * width + outwidth * heightDiff); - leftedge = InputLayer(this); // dummy layer, left edge of zeros - - var gridopts = new LSTMNode.GridOpts; - gridopts.dim = opts.dim; - gridopts.aopts = opts.aopts; - gridopts.hasBias = opts.hasBias; - gridopts.kind = opts.kind; - gridopts.outDim = opts.nvocab; - - srcNodeGrid = LSTMNode.grid(height, inwidth, gridopts); - dstNodeGrid = LSTMNode.grid(height, outwidth, gridopts); - - srcGrid = srcNodeGrid.map(_.create(this)); - dstGrid = dstNodeGrid.map(_.create(this)); - -// srcGrid(?,inwidth-1).data.zip(dstGrid(?,0).data).map{case (from:Layer, to:Layer) => {to.input = from; to.setInput(1, from(1));}} - - // the preamble (bottom) layers - val lopts1 = new LinNode{modelName = "srcWordMap"; outdim = opts.dim; aopts = opts.aopts; hasBias=opts.hasBias}; - val lopts2 = new LinNode{modelName = "dstWordMap"; outdim = opts.dim; aopts = opts.aopts; hasBias=opts.hasBias}; - for (j <- 0 until width) { - setlayer(0, j, InputLayer(this)); - if (j < inwidth) { - setlayer(1, j, LinLayer(this, lopts1).setInput(0, getlayer(0, j))); - } else { - setlayer(1, j, LinLayer(this, lopts2).setInput(0, getlayer(0, j))); - } - } - - // the main grid - for (i <- 0 until height) { - val loptsSrc = new LSTMNode{dim = opts.dim; aopts = opts.aopts; kind = opts.kind; hasBias = opts.hasBias}; - val loptsDst = new LSTMNode{dim = opts.dim; aopts = opts.aopts; kind = opts.kind; hasBias = opts.hasBias}; - loptsSrc.prefix = if (opts.bylevel) "SrcLevel_%d" format i; else "Src"; - loptsDst.prefix = if (opts.bylevel) "DstLevel_%d" format i; else "Dst"; - loptsSrc.constructGraph; - loptsDst.constructGraph; - for (j <- 0 until width) { - val layer = LSTMLayer(this, if (j < inwidth) loptsSrc else loptsDst); - layer.setInput(2, getlayer(i-1+preamble_rows, j)); // input 2 (i) is from layer below - if (j > 0) { - layer.setInput(0, getlayer(i+preamble_rows, j-1)); // input 0 (prev_h) is layer to the left, output 0 (h) - layer.setInput(1, getlayer(i+preamble_rows, j-1)(1)); // input 1 (prev_c) is layer to the left, output 1 (c) - } else { - layer.setInput(0, leftedge); // in first column, just use dummy (zeros) input - layer.setInput(1, leftedge); - } - setlayer(i+preamble_rows, j, layer); - } - } - - // the top layers - output_layers = new Array[Layer](outwidth); - if (opts.netType == 0) { - val lopts3 = new LinNode{modelName = "outWordMap"; outdim = opts.nvocab; aopts = opts.aopts; hasBias = opts.hasBias}; - val sopts = new SoftmaxOutputNode{scoreType = opts.scoreType}; - for (j <- 0 until outwidth) { - val linlayer = LinLayer(this, lopts3).setInput(0, getlayer(fullheight-1, j+inwidth)); - setlayer(fullheight, inwidth + j, linlayer); - val smlayer = SoftmaxOutputLayer(this, sopts).setInput(0, linlayer); - setlayer(fullheight+1, inwidth + j, smlayer); - output_layers(j) = smlayer; - } - } else { - val nsopts = new NegsampOutputNode{modelName = "outWordMap"; outdim = opts.nvocab; aopts = opts.aopts; hasBias = opts.hasBias; - scoreType = opts.scoreType; nsamps = opts.nsamps; expt = opts.expt}; - for (j <- 0 until outwidth) { - val nslayer = NegsampOutputLayer(this, nsopts).setInput(0, getlayer(fullheight-1, j+inwidth)); - setlayer(fullheight, inwidth + j, nslayer); - output_layers(j) = nslayer; - } - } - } - - def mapOOV(in:Mat) = { - if (OOVelem.asInstanceOf[AnyRef] == null) { - OOVelem = convertMat(iones(1,1) * opts.OOVsym); - } - in ~ in + ((in >= opts.nvocab) ∘ (OOVelem - in)) - } - - override def assignInputs(gmats:Array[Mat], ipass:Int, pos:Long) { - val src = gmats(0); - val dstx = gmats(1); - srcn = src.nnz/src.ncols; - if (srcn*src.ncols != src.nnz) throw new RuntimeException("SeqToSeq src batch not fixed length"); - val srcdata = int(src.contents.view(srcn, batchSize).t); // IMat with columns corresponding to word positions, with batchSize rows. - - val dstxn0 = dstx.nnz/dstx.ncols; - if (dstxn0*dstx.ncols != dstx.nnz) throw new RuntimeException("SeqToSeq dstx batch not fixed length"); - val dstxdata0 = int(dstx.contents.view(dstxn0, batchSize).t); - dstxn = dstxn0 + (if (opts.addStart) 1 else 0); - if (opts.addStart && (leftStart.asInstanceOf[AnyRef] == null)) { - leftStart = convertMat(izeros(batchSize, 1)); - } - val dstxdata = if (opts.addStart) (leftStart \ dstxdata0) else dstxdata0; - - mapOOV(srcdata); - mapOOV(dstxdata); - val srcmat = oneHot(srcdata.contents, opts.nvocab); - val dstxmat = oneHot(dstxdata.contents, opts.nvocab); - srcn = math.min(srcn, opts.inwidth); - if (srcn < inwidth) initPrevCol; - for (i <- 0 until srcn) { - val cols = srcmat.colslice(i*batchSize, (i+1)*batchSize); - getlayer(0, inwidth + i - srcn).output = cols; - } - dstxn = math.min(dstxn, opts.outwidth); - for (i <- 0 until dstxn) { - val cols = dstxmat.colslice(i*batchSize, (i+1)*batchSize); - getlayer(0, inwidth + i).output = cols; - } - if (leftedge.output.asInstanceOf[AnyRef] == null) { - leftedge.output = convertMat(zeros(opts.dim \ batchSize)); - } - } - - def initPrevCol = { - for (i <- 0 until height) { - val leftlayer = getlayer(i+preamble_rows, inwidth-srcn-1); - if (leftlayer.output.asInstanceOf[AnyRef] == null) { - leftlayer.output = convertMat(zeros(opts.dim \ batchSize)); - } - if (leftlayer.outputs(1).asInstanceOf[AnyRef] == null) { - leftlayer.setOutput(1, convertMat(zeros(opts.dim \ batchSize))); - } - } - } - - override def assignTargets(gmats:Array[Mat], ipass:Int, pos:Long) { - val dsty = if (gmats.length > 2) gmats(2) else gmats(1); - val dstyn0 = dsty.nnz/dsty.ncols; - if (dstyn0*dsty.ncols != dsty.nnz) throw new RuntimeException("SeqToSeq dsty batch not fixed length"); - val dstydata = int(dsty.contents.view(dstyn0, batchSize).t); - mapOOV(dstydata); - val dstyn1 = math.min(dstyn0 - (if (opts.addStart) 0 else 1), opts.outwidth); - for (j <- 0 until dstyn1) { - val incol = if (opts.addStart) dstydata.colslice(j,j+1).t else dstydata.colslice(j+1,j+2).t - output_layers(j).target = incol; - } - if (PADrow.asInstanceOf[AnyRef] == null) { - PADrow = convertMat(iones(1, batchSize) * opts.PADsym); - } - dstyn = math.min(dstyn1 + 1, opts.outwidth); - if (dstyn1 < opts.outwidth) { - output_layers(dstyn1).target = PADrow; - } - } - - override def dobatch(gmats:Array[Mat], ipass:Int, pos:Long):Unit = { - if (batchSize < 0) batchSize = gmats(0).ncols; - if (batchSize == gmats(0).ncols) { // discard odd-sized minibatches - assignInputs(gmats, ipass, pos); - assignTargets(gmats, ipass, pos); - if (mask.asInstanceOf[AnyRef] != null) { - modelmats(0) ~ modelmats(0) ∘ mask; - } - val minlayer = lindex(0, inwidth - srcn); - val maxlayer = lindex(0, inwidth + dstxn); - var i = minlayer; - while (i < maxlayer) { - if (opts.debug > 0) { - println("dobatch forward %d %s" format (i, layers(i).getClass)) - } - layers(i).forward; - i += 1; - } - for (j <- 0 until output_layers.length) { - output_layers(j) match { - case _:OutputLayer => {} - case _ => { - if (output_layers(j).deriv.asInstanceOf[AnyRef] != null) output_layers(j).deriv.set(1); - } - } - } - if (opts.aopts == null) { - for (j <- 0 until updatemats.length) updatemats(j).clear; - } - while (i > minlayer) { - i -= 1; - if (opts.debug > 0) { - println("dobatch backward %d %s" format (i, layers(i).getClass)) - } - layers(i).backward(ipass, pos); - } - } - } - - override def evalbatch(mats:Array[Mat], ipass:Int, pos:Long):FMat = { - if (batchSize < 0) batchSize = gmats(0).ncols; - if (batchSize == gmats(0).ncols) { - assignInputs(gmats, ipass, pos); - assignTargets(gmats, ipass, pos); - if (mask.asInstanceOf[AnyRef] != null) { - modelmats(0) ~ modelmats(0) ∘ mask; - } - val minlayer = lindex(0, inwidth - srcn); - val maxlayer = lindex(0, inwidth + dstxn); - var i = minlayer; - while (i < maxlayer) { - if (opts.debug > 0) { - println("evalbatch forward %d %s" format (i, layers(i).getClass)) - } - layers(i).forward; - i += 1; - } - if (putBack >= 0) { - output_layers(dstxn-1).output.colslice(0, gmats(0).ncols, gmats(1)); - } - var score = 0f - var j = 0; - while (j < dstxn-1) { - score += output_layers(j).score.v; - j += 1; - } - row(score/(dstxn-1)) - } else { - zeros(1, 1); - } - } -} - -object SeqToSeq { - trait Opts extends Net.Opts { - var inwidth = 1; // Max src sentence length - var outwidth = 1; // Max dst sentence length - var height = 1; // Number of LSTM layers vertically - var nvocab = 100000; // Vocabulary size - var kind = 0; // LSTM type, see below - var bylevel = true; // Use different models for each level - var netType = 0; // Network type, 0 = softmax output, 1 = Neg Sampling output - var PADsym = 1; // Padding symbol - var OOVsym = 2; // OOV symbol - var STARTsym = 1; // Start symbol - var addStart = true; // Add the start symbol to dst sentences - var scoreType = 0; // Score type, 0 = LL, 1 = accuracy, 2 = LL of full Softmax, 3 = accuracy of full Softmax - var nsamps = 100; // Number of negative samples - var expt = 0.8f; // Negative sampling exponent (tail boost) - } - - class Options extends Opts {} - - def mkNetModel(fopts:Model.Opts) = { - new SeqToSeq(fopts.asInstanceOf[SeqToSeq.Opts]) - } - - def mkUpdater(nopts:Updater.Opts) = { - new ADAGrad(nopts.asInstanceOf[ADAGrad.Opts]) - } - - def mkRegularizer(nopts:Mixin.Opts):Array[Mixin] = { - Array(new L1Regularizer(nopts.asInstanceOf[L1Regularizer.Opts])) - } - - class LearnOptions extends Learner.Options with SeqToSeq.Opts with MatSource.Opts with ADAGrad.Opts with L1Regularizer.Opts - - def learner(mat0:Mat, mat1:Mat, regularize:Boolean = false) = { - val opts = new LearnOptions; - opts.batchSize = 128; - val nn = new Learner( - new MatSource(Array(mat0, mat1), opts), - new SeqToSeq(opts), - if (regularize) Array(new L1Regularizer(opts)) else null, - new ADAGrad(opts), - null, - opts) - (nn, opts) - } - - def learnerX(mat0:Mat, mat1:Mat) = { - val opts = new LearnOptions; - opts.batchSize = math.min(100000, mat0.ncols/30 + 1); - val nn = new Learner( - new MatSource(Array(mat0, mat1), opts), - new SeqToSeq(opts), - null, - null, - null, - opts) - (nn, opts) - } - - class FDSopts extends Learner.Options with SeqToSeq.Opts with FileSource.Opts with ADAGrad.Opts with L1Regularizer.Opts - - def learner(fn1:String, fn2:String, regularize:Boolean, adagrad:Boolean):(Learner, FDSopts) = learner(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0)), regularize, adagrad); - - def learner(fn1:String, fn2:String):(Learner, FDSopts) = learner(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0)), false, true); - - def learnerX(fn1:String, fn2:String):(Learner, FDSopts) = learnerX(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0))); - - def learner(fnames:List[(Int)=>String]):(Learner, FDSopts) = learner(fnames, false, true); - - def learner(fnames:List[(Int)=>String], regularize:Boolean, adagrad:Boolean):(Learner, FDSopts) = { - val opts = new FDSopts; - opts.fnames = fnames - opts.batchSize = 128; - opts.eltsPerSample = 500; - implicit val threads = threadPool(4); - val ds = new FileSource(opts) - val nn = new Learner( - ds, - new SeqToSeq(opts), - if (regularize) Array(new L1Regularizer(opts)) else null, - if (adagrad) new ADAGrad(opts) else new Grad(opts), - null, - opts) - (nn, opts) - } - - def learnerX(fnames:List[(Int)=>String]):(Learner, FDSopts) = { - val opts = new FDSopts; - opts.fnames = fnames - opts.batchSize = 128; - opts.eltsPerSample = 500; - implicit val threads = threadPool(4); - val ds = new FileSource(opts) - val nn = new Learner( - ds, - new SeqToSeq(opts), - null, - null, - null, - opts) - (nn, opts) - } - - def load(fname:String):SeqToSeq = { - val mm = new SeqToSeq; - mm.loadMetaData(fname); - mm.load(fname); - mm - } - -} - + */ +class SeqToSeq(override val opts:SeqToSeq.Opts = new SeqToSeq.Options) extends Net(opts) { + + var PADrow:Mat = null; + var OOVelem:Mat = null; + var leftEdge:Layer = null; + var leftStart:Mat = null; + var dstxdata:Mat = null; + var dstxdata0:Mat = null; + var srcGrid:LayerMat = null; + var dstGrid:LayerMat = null; + var srcGridOpts:LSTMNode.GridOpts = null; + var dstGridOpts:LSTMNode.GridOpts = null; + var height = 0; + var inwidth = 0; + var outwidth = 0; + var width = 0; + var srcn = 0; + var dstxn = 0; + var dstyn = 0; + val preamble_rows = 2; + + override def createLayers = { + height = opts.height; + inwidth = opts.inwidth; + outwidth = opts.outwidth; + leftEdge = InputLayer(this); // dummy layer, left edge of zeros + + srcGridOpts = new LSTMNode.GridOpts; + srcGridOpts.copyFrom(opts); + srcGridOpts.modelName = "src_level%d"; + srcGridOpts.netType = LSTMNode.gridTypeNoOutput; + srcGrid = LSTMLayer.grid(this, height, inwidth, srcGridOpts); + layers = srcGrid.data.filter(_ != null); + for (i <- 0 until height) srcGrid(i+preamble_rows, 0).setInputs(leftEdge, leftEdge); + + if (! opts.embed) { + dstGridOpts = new LSTMNode.GridOpts; + dstGridOpts.copyFrom(opts); + dstGridOpts.modelName = "dst_level%d"; + dstGridOpts.netType = LSTMNode.gridTypeSoftmaxOutput; + dstGridOpts.outdim = opts.nvocab; + dstGrid = LSTMLayer.grid(this, height, outwidth, dstGridOpts); + + srcGrid link dstGrid; + layers = layers ++ dstGrid.data.filter(_ != null); + output_layers = new Array[Layer](outwidth); + for (i <- 0 until outwidth) output_layers(i) = dstGrid(dstGrid.nrows-1, i); + } + } + + def mapOOV(in:Mat) = { + if (OOVelem.asInstanceOf[AnyRef] == null) { + OOVelem = convertMat(iones(1,1) * opts.OOVsym); + } + in ~ in + ((in >= opts.nvocab) ∘ (OOVelem - in)) + } + + override def assignInputs(gmats:Array[Mat], ipass:Int, pos:Long) = { + val src = gmats(0); + srcn = src.nnz/src.ncols; + if (srcn*src.ncols != src.nnz) throw new RuntimeException("SeqToSeq src batch not fixed length"); + val srcdata = int(src.contents.view(srcn, batchSize).t); // IMat with columns corresponding to word positions, with batchSize rows. + mapOOV(srcdata); + val srcmat = oneHot(srcdata.contents, opts.nvocab); + srcn = math.min(srcn, opts.inwidth); + if (srcn < inwidth) initPrevCol; + for (i <- 0 until srcn) { + val cols = srcmat.colslice(i*batchSize, (i+1)*batchSize); + srcGrid(0, inwidth + i - srcn).output = cols; + } + if (leftEdge.output.asInstanceOf[AnyRef] == null) { + leftEdge.output = convertMat(zeros(opts.dim \ batchSize)); + } + + if (! opts.embed) { + val dstx = gmats(1); + val dstxn0 = dstx.nnz/dstx.ncols; + if (dstxn0*dstx.ncols != dstx.nnz) throw new RuntimeException("SeqToSeq dstx batch not fixed length"); + val dstxdata0 = int(dstx.contents.view(dstxn0, batchSize).t); + dstxn = dstxn0 + (if (opts.addStart) 1 else 0); + if (opts.addStart && (leftStart.asInstanceOf[AnyRef] == null)) { + leftStart = convertMat(izeros(batchSize, 1)); + } + val dstxdata = if (opts.addStart) (leftStart \ dstxdata0) else dstxdata0; + mapOOV(dstxdata); + val dstxmat = oneHot(dstxdata.contents, opts.nvocab); + + dstxn = math.min(dstxn, opts.outwidth); + for (i <- 0 until dstxn) { + val cols = dstxmat.colslice(i*batchSize, (i+1)*batchSize); + dstGrid(0, i).output = cols; + } + } + } + + def initPrevCol = { + for (i <- 0 until height) { + val leftlayer = srcGrid(i+preamble_rows, inwidth-srcn-1); + if (leftlayer.output.asInstanceOf[AnyRef] == null) { + leftlayer.output = convertMat(zeros(opts.dim \ batchSize)); + } + if (leftlayer.outputs(1).asInstanceOf[AnyRef] == null) { + leftlayer.setOutput(1, convertMat(zeros(opts.dim \ batchSize))); + } + } + } + + override def assignTargets(gmats:Array[Mat], ipass:Int, pos:Long) { + val dsty = if (gmats.length > 2) gmats(2) else gmats(1); + val dstyn0 = dsty.nnz/dsty.ncols; + if (dstyn0*dsty.ncols != dsty.nnz) throw new RuntimeException("SeqToSeq dsty batch not fixed length"); + val dstydata = int(dsty.contents.view(dstyn0, batchSize).t); + mapOOV(dstydata); + val dstyn1 = math.min(dstyn0 - (if (opts.addStart) 0 else 1), opts.outwidth); + for (j <- 0 until dstyn1) { + val incol = if (opts.addStart) dstydata.colslice(j,j+1).t else dstydata.colslice(j+1,j+2).t + output_layers(j).target = incol; + } + if (PADrow.asInstanceOf[AnyRef] == null) { + PADrow = convertMat(iones(1, batchSize) * opts.PADsym); + } + dstyn = math.min(dstyn1 + 1, opts.outwidth); + if (dstyn1 < opts.outwidth) { + output_layers(dstyn1).target = PADrow; + } + } + + override def dobatch(gmats:Array[Mat], ipass:Int, pos:Long):Unit = { + if (batchSize < 0) batchSize = gmats(0).ncols; + if (batchSize == gmats(0).ncols) { // discard odd-sized minibatches + assignInputs(gmats, ipass, pos); + assignTargets(gmats, ipass, pos); + if (mask.asInstanceOf[AnyRef] != null) { + modelmats(0) ~ modelmats(0) ∘ mask; + } + val mincol = inwidth - srcn; + val maxcol = dstxn; + srcGrid.forward(mincol, inwidth-1, opts.debug); + dstGrid.forward(0, maxcol-1, opts.debug); + + output_layers.map((layer:Layer) => layer match { + case _:OutputLayer => {} + case _ => {if (layer.deriv.asInstanceOf[AnyRef] != null) layer.deriv.set(1);} + }) + if (opts.aopts == null) updatemats.map(_.clear); + + dstGrid.backward(0, maxcol-1, opts.debug, ipass, pos); + srcGrid.backward(mincol, inwidth-1, opts.debug, ipass, pos); + } + } + + override def evalbatch(mats:Array[Mat], ipass:Int, pos:Long):FMat = { + if (batchSize < 0) batchSize = gmats(0).ncols; + if (batchSize == gmats(0).ncols) { + assignInputs(gmats, ipass, pos); + if (mask.asInstanceOf[AnyRef] != null) { + modelmats(0) ~ modelmats(0) ∘ mask; + } + val mincol = inwidth - srcn; + srcGrid.forward(mincol, inwidth-1, opts.debug); + if (! opts.embed) { + val maxcol = dstxn; + assignTargets(gmats, ipass, pos); + dstGrid.forward(0, maxcol-1, opts.debug); + if (putBack >= 0) { + output_layers(dstxn-1).output.colslice(0, gmats(0).ncols, gmats(1)); + } + var score = 0f; + var j = 0; + while (j < dstxn-1) { + score += output_layers(j).score.v; + j += 1; + } + row(score/(dstxn-1)); + } else { + if (ogmats != null) { + ogmats(0) = srcGrid(height+preamble_rows-1, srcGrid.ncols-1).output.asMat; + } + zeros(1,1); + } + } else { + zeros(1, 1); + } + } +} + +object SeqToSeq { + trait Opts extends Net.Opts { + var inwidth = 1; // Max src sentence length + var outwidth = 1; // Max dst sentence lenth + var height = 1; // Number of LSTM layers vertically + var nvocab = 100000; // Vocabulary size + var kind = 0; // LSTM type, see below + var bylevel = true; // Use different models for each level + var netType = 0; // Network type, 0 = softmax output, 1 = Neg Sampling output + var PADsym = 1; // Padding symbol + var OOVsym = 2; // OOV symbol + var STARTsym = 1; // Start symbol + var addStart = true; // Add the start symbol to dst sentences + var scoreType = 0; // Score type, 0 = LL, 1 = accuracy, 2 = LL of full Softmax, 3 = accuracy of full Softmax + var nsamps = 100; // Number of negative samples + var expt = 0.8f; // Negative sampling exponent (tail boost) + var embed = false; + + } + + class Options extends Opts {} + + def mkNetModel(fopts:Model.Opts) = { + new SeqToSeq(fopts.asInstanceOf[SeqToSeq.Opts]) + } + + def mkUpdater(nopts:Updater.Opts) = { + new ADAGrad(nopts.asInstanceOf[ADAGrad.Opts]) + } + + def mkRegularizer(nopts:Mixin.Opts):Array[Mixin] = { + Array(new L1Regularizer(nopts.asInstanceOf[L1Regularizer.Opts])) + } + + class LearnOptions extends Learner.Options with SeqToSeq.Opts with MatSource.Opts with ADAGrad.Opts with L1Regularizer.Opts + + def learner(mat0:Mat, mat1:Mat, regularize:Boolean = false) = { + val opts = new LearnOptions; + opts.batchSize = 128; + val nn = new Learner( + new MatSource(Array(mat0, mat1), opts), + new SeqToSeq(opts), + if (regularize) Array(new L1Regularizer(opts)) else null, + new ADAGrad(opts), + null, + opts) + (nn, opts) + } + + def learnerX(mat0:Mat, mat1:Mat) = { + val opts = new LearnOptions; + opts.batchSize = math.min(100000, mat0.ncols/30 + 1); + val nn = new Learner( + new MatSource(Array(mat0, mat1), opts), + new SeqToSeq(opts), + null, + null, + null, + opts) + (nn, opts) + } + + class FDSopts extends Learner.Options with SeqToSeq.Opts with FileSource.Opts with ADAGrad.Opts with L1Regularizer.Opts + + def learner(fn1:String, fn2:String, regularize:Boolean, adagrad:Boolean):(Learner, FDSopts) = learner(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0)), regularize, adagrad); + + def learner(fn1:String, fn2:String):(Learner, FDSopts) = learner(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0)), false, true); + + def learnerX(fn1:String, fn2:String):(Learner, FDSopts) = learnerX(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0))); + + def learner(fnames:List[(Int)=>String]):(Learner, FDSopts) = learner(fnames, false, true); + + def learner(fnames:List[(Int)=>String], regularize:Boolean, adagrad:Boolean):(Learner, FDSopts) = { + val opts = new FDSopts; + opts.fnames = fnames + opts.batchSize = 128; + opts.eltsPerSample = 500; + implicit val threads = threadPool(4); + val ds = new FileSource(opts) + val nn = new Learner( + ds, + new SeqToSeq(opts), + if (regularize) Array(new L1Regularizer(opts)) else null, + if (adagrad) new ADAGrad(opts) else new Grad(opts), + null, + opts) + (nn, opts) + } + + def learnerX(fnames:List[(Int)=>String]):(Learner, FDSopts) = { + val opts = new FDSopts; + opts.fnames = fnames + opts.batchSize = 128; + opts.eltsPerSample = 500; + implicit val threads = threadPool(4); + val ds = new FileSource(opts) + val nn = new Learner( + ds, + new SeqToSeq(opts), + null, + null, + null, + opts) + (nn, opts) + } + + class FEopts extends Learner.Options with SeqToSeq.Opts with FileSource.Opts with FileSink.Opts + + def embed(model:SeqToSeq, ifname:String, ofname:String):(Learner, FEopts) = { + val opts = new FEopts; + opts.copyFrom(model.opts); + opts.fnames = List(FileSource.simpleEnum(ifname,1,0)); + opts.ofnames = List(FileSource.simpleEnum(ofname,1,0)); + opts.embed = true; + val newmod = new SeqToSeq(opts); + newmod.refresh = false; + model.copyTo(newmod); + implicit val threads = threadPool(4); + val ds = new FileSource(opts) + val nn = new Learner( + new FileSource(opts), + newmod, + null, + null, + new FileSink(opts), + opts) + (nn, opts) + } + + def load(fname:String):SeqToSeq = { + val mm = new SeqToSeq; + mm.loadMetaData(fname); + mm.load(fname); + mm + } + +} + diff --git a/src/main/scala/BIDMach/networks/layers/LSTM.scala b/src/main/scala/BIDMach/networks/layers/LSTM.scala index de148936..139e6260 100644 --- a/src/main/scala/BIDMach/networks/layers/LSTM.scala +++ b/src/main/scala/BIDMach/networks/layers/LSTM.scala @@ -1,358 +1,379 @@ -package BIDMach.networks.layers - -import BIDMat.{Mat,ND,SBMat,CMat,CSMat,DMat,FMat,FND,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,GND,SMat,SDMat} -import BIDMat.MatFunctions._ -import BIDMat.SciFunctions._ -import BIDMach.datasources._ -import BIDMach.updaters._ -import BIDMach.mixins._ -import BIDMach.models._ -import BIDMach.networks._ -import BIDMach._ -import scala.util.hashing.MurmurHash3; -import scala.collection.mutable.HashMap; - -/** +package BIDMach.networks.layers + +import BIDMat.{Mat,ND,SBMat,CMat,CSMat,DMat,FMat,FND,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,GND,SMat,SDMat} +import BIDMat.MatFunctions._ +import BIDMat.SciFunctions._ +import BIDMach.datasources._ +import BIDMach.updaters._ +import BIDMach.mixins._ +import BIDMach.models._ +import BIDMach.networks._ +import BIDMach._ +import scala.util.hashing.MurmurHash3; +import scala.collection.mutable.HashMap; + +/** * LSTM unit - */ - -class LSTMLayer(override val net:Net, override val opts:LSTMNode = new LSTMNode) extends CompoundLayer(net, opts) { - override val _inputs = new Array[LayerTerm](3); - override val _outputs = new Array[ND](2); - override val _derivs = new Array[ND](2); - - override def toString = { - "LSTM@"+Integer.toHexString(hashCode % 0x10000).toString - } -} - -trait LSTMNodeOpts extends CompoundNodeOpts { - var dim = 0; - var kind = 0; - var hasBias = false; - var scoreType = 0; - var outDim = 0; -} - -class LSTMNode extends CompoundNode with LSTMNodeOpts { - - override val inputs:Array[NodeTerm] = Array(null, null, null); -// override val inputTerminals:Array[Int] = Array(0,0,0); - - def constructGraph = { - kind match { - case 0 => constructGraph0 - case 1 => constructGraph1 - case 2 => constructGraph2 - case 3 => constructGraph3 - case 4 => constructGraph4 - case _ => throw new RuntimeException("LSTMLayer type %d not recognized" format kind); - } - } - - // Basic LSTM topology with 8 linear layers - - def constructGraph0 = { - val prev_h = new CopyNode; - val prev_c = new CopyNode; - val i = new CopyNode; - - val il1 = new LinNode{inputs(0) = i; modelName = prefix + "LSTM_input_in_gate"; hasBias = this.hasBias}; - val ph1 = new LinNode{inputs(0) = prev_h; modelName = prefix + "LSTM_prev_h_in_gate"; hasBias = this.hasBias}; - val sum1 = new AddNode{inputs(0) = il1; inputs(1) = ph1}; - val in_gate = new SigmoidNode{inputs(0) = sum1}; - - val il2 = new LinNode{inputs(0) = i; modelName = prefix + "LSTM_input_out_gate"; hasBias = this.hasBias}; - val ph2 = new LinNode{inputs(0) = prev_h; modelName = prefix + "LSTM_prev_h_out_gate"; hasBias = this.hasBias}; - val sum2 = new AddNode{inputs(0) = il2; inputs(1) = ph2}; - val out_gate = new SigmoidNode{inputs(0) = sum2}; - - val il3 = new LinNode{inputs(0) = i; modelName = prefix + "LSTM_input_forget_gate"; hasBias = this.hasBias}; - val ph3 = new LinNode{inputs(0) = prev_h; modelName = prefix + "LSTM_prev_h_forget_gate"; hasBias = this.hasBias}; - val sum3 = new AddNode{inputs(0) = il3; inputs(1) = ph3}; - val forget_gate = new SigmoidNode{inputs(0) = sum3}; - - val il4 = new LinNode{inputs(0) = i; modelName = prefix + "LSTM_input_tanh"; hasBias = this.hasBias}; - val ph4 = new LinNode{inputs(0) = prev_h; modelName = prefix + "LSTM_prev_h_tanh"; hasBias = this.hasBias}; - val sum4 = new AddNode{inputs(0) = il4; inputs(1) = ph4}; - val g = new TanhNode{inputs(0) = sum4}; - - val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = g}; - val f_prod = new MulNode{inputs(0) = forget_gate; inputs(1) = prev_c}; - val next_c = new AddNode{inputs(0) = in_prod; inputs(1) = f_prod}; - - val next_tanh = new TanhNode{inputs(0) = next_c;}; - val next_h = new MulNode{inputs(0) = out_gate; inputs(1) = next_tanh}; - - lopts = Array(prev_h, prev_c, i, // First 3 layers should be inputs - il1, ph1, sum1, in_gate, // Otherwise the ordering should support forward-backward inference - il2, ph2, sum2, out_gate, - il3, ph3, sum3, forget_gate, - il4, ph4, sum4, g, - in_prod, f_prod, next_c, - next_tanh, next_h); - - lopts.map(_.parent = this); - outputNumbers = Array(lopts.length-1, lopts.length-3); // Specifies the output layer numbers (next_h and next_c) - } - - // LSTM with all linear weights grouped into single matrix, to more fully use GPU. - // Observed not very stable to train - // - // | Wi_in Wh_in | - // | Wi_out Wh_out | | i | - // | Wi_forget Wh_forget | | h | - // | Wi_g Wh_g | - - def constructGraph1 = { - val prev_h = new CopyNode; - val prev_c = new CopyNode; - val i = new CopyNode; - - val prev_hi = new StackNode{inputs(0) = prev_h; inputs(1) = i}; - val il = new LinNode{inputs(0) = prev_hi; modelName = prefix + "LSTM_all"; outdim = 4*dim; hasBias = this.hasBias}; - val sp = new SplitVertNode{inputs(0) = il; nparts = 4;} - - val in_gate = new SigmoidNode{inputs(0) = sp(0)}; - val out_gate = new SigmoidNode{inputs(0) = sp(1)}; - val forget_gate = new SigmoidNode{inputs(0) = sp(2)}; - val g = new TanhNode{inputs(0) = sp(3)}; - - val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = g}; - val f_prod = new MulNode{inputs(0) = forget_gate; inputs(1) = prev_c}; - val next_c = new AddNode{inputs(0) = in_prod; inputs(1) = f_prod}; - - val next_tanh = new TanhNode{inputs(0) = next_c;}; - val next_h = new MulNode{inputs(0) = out_gate; inputs(1) = next_tanh}; - - lopts = Array(prev_h, prev_c, i, // First 3 layers should be inputs - prev_hi, il, sp, - in_gate, // Otherwise the ordering should support forward-backward inference - out_gate, - forget_gate, - g, - in_prod, f_prod, next_c, - next_tanh, next_h); - - lopts.map(_.parent = this); - outputNumbers = Array(lopts.length-1, lopts.length-3); // Specifies the output layer numbers (next_h and next_c) - } - - // LSTM with 4 linear layers, with h and i stacked as inputs - // | Wi_in Wh_in | | i | + | Wi_out Wh_out | | i | + | Wi_forget Wh_forget | | i | + | Wi_g Wh_g | | i | - // | h | | h | | h | | h | - - def constructGraph2 = { - val prev_h = new CopyNode; - val prev_c = new CopyNode; - val i = new CopyNode; - - val prev_hi = new StackNode{inputs(0) = prev_h; inputs(1) = i}; - - val lin1 = new LinNode{inputs(0) = prev_hi; modelName = prefix + "LSTM_in_gate"; outdim = dim; hasBias = this.hasBias} - val in_gate = new SigmoidNode{inputs(0) = lin1}; - - val lin2 = new LinNode{inputs(0) = prev_hi; modelName = prefix + "LSTM_out_gate"; outdim = dim; hasBias = this.hasBias} - val out_gate = new SigmoidNode{inputs(0) = lin2}; - - val lin3 = new LinNode{inputs(0) = prev_hi; modelName = prefix + "LSTM_forget_gate"; outdim = dim; hasBias = this.hasBias} - val forget_gate = new SigmoidNode{inputs(0) = lin3}; - - val lin4 = new LinNode{inputs(0) = prev_hi; modelName = prefix + "LSTM_tanh"; outdim = dim; hasBias = this.hasBias} - val g = new TanhNode{inputs(0) = lin4}; - - val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = g}; - val f_prod = new MulNode{inputs(0) = forget_gate; inputs(1) = prev_c}; - val next_c = new AddNode{inputs(0) = in_prod; inputs(1) = f_prod}; - - val next_tanh = new TanhNode{inputs(0) = next_c;}; - val next_h = new MulNode{inputs(0) = out_gate; inputs(1) = next_tanh}; - - lopts = Array(prev_h, prev_c, i, // First 3 layers should be inputs - prev_hi, - lin1, in_gate, // Otherwise the ordering should support forward-backward inference - lin2, out_gate, - lin3, forget_gate, - lin4, g, - in_prod, f_prod, next_c, - next_tanh, next_h); - - lopts.map(_.parent = this); - outputNumbers = Array(lopts.length-1, lopts.length-3); // Specifies the output layer numbers (next_h and next_c) - } - - // LSTM with two sets of linear weights, paired outputs. More stable to train than the single linlayer network - // - // | Wi_in Wh_in | | i | + | Wi_forget Wh_forget | | i | - // | Wi_out Wh_out | | h | | Wi_g Wh_g | | h | - - def constructGraph3 = { - val prev_h = new CopyNode; - val prev_c = new CopyNode; - val i = new CopyNode; - - val prev_hi = new StackNode{inputs(0) = prev_h; inputs(1) = i}; - val il1 = new LinNode{inputs(0) = prev_hi; modelName = prefix + "LSTM_in_out"; outdim = 2*dim; hasBias = this.hasBias}; - val sp1 = new SplitVertNode{inputs(0) = il1; nparts = 2;} - val il2 = new LinNode{inputs(0) = prev_hi; modelName = prefix + "LSTM_forget_tanh"; outdim = 2*dim; hasBias = this.hasBias}; - val sp2 = new SplitVertNode{inputs(0) = il2; nparts = 2;} - - val in_gate = new SigmoidNode{inputs(0) = sp1(0)}; - val out_gate = new SigmoidNode{inputs(0) = sp1(1)}; - val forget_gate = new SigmoidNode{inputs(0) = sp2(0)}; - val g = new TanhNode{inputs(0) = sp2(1)}; - - val in_prod = new MulNode{inputs(0) = in_gate; inputs(1) = g}; - val f_prod = new MulNode{inputs(0) = forget_gate; inputs(1) = prev_c}; - val next_c = new AddNode{inputs(0) = in_prod; inputs(1) = f_prod}; - - val next_tanh = new TanhNode{inputs(0) = next_c;}; - val next_h = new MulNode{inputs(0) = out_gate; inputs(1) = next_tanh}; - - lopts = Array(prev_h, prev_c, i, // First 3 layers should be inputs - prev_hi, il1, sp1, il2, sp2, - in_gate, // Otherwise the ordering should support forward-backward inference - out_gate, - forget_gate, - g, - in_prod, f_prod, next_c, - next_tanh, next_h); - - lopts.map(_.parent = this); - outputNumbers = Array(lopts.length-1, lopts.length-3); // Specifies the output layer numbers (next_h and next_c) - } - - // LSTM with 4 linear layers, with h and i stacked as inputs - - def constructGraph4 = { - import BIDMach.networks.layers.Node._ - - val in_h = copy; - val in_c = copy; - val in_i = copy; - val h_on_i = in_h on in_i; - - val lin1 = linear(h_on_i)(prefix+"LSTM_in_gate", outdim=dim, hasBias=hasBias); - val lin2 = linear(h_on_i)(prefix+"LSTM_out_gate", outdim=dim, hasBias=hasBias); - val lin3 = linear(h_on_i)(prefix+"LSTM_forget_gate", outdim=dim, hasBias=hasBias); - val lin4 = linear(h_on_i)(prefix+"LSTM_tanh_gate", outdim=dim, hasBias=hasBias); - - val in_gate = σ(lin1); - val out_gate = σ(lin2); - val forget_gate = σ(lin3); - val in_sat = tanh(lin4); - - val in_prod = in_gate ∘ in_sat; - val f_prod = forget_gate ∘ in_c; - val out_c = in_prod + f_prod; - - val out_tanh = tanh(out_c); - val out_h = out_gate ∘ out_tanh; - - grid = in_h \ lin1 \ in_gate \ in_prod \ out_tanh on - in_c \ lin2 \ out_gate \ f_prod \ out_h on - in_i \ lin3 \ forget_gate \ out_c \ null on - h_on_i \ lin4 \ in_sat \ null \ null; - - lopts = grid.data.filter(_ != null); - lopts.map(_.parent = this); - outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); - - } - - override def clone:LSTMNode = { - copyTo(new LSTMNode).asInstanceOf[LSTMNode]; - } - - override def create(net:Net):LSTMLayer = { - LSTMLayer(net, this); - } - - override def toString = { - "LSTM@"+Integer.toHexString(hashCode % 0x10000).toString - } - - def h = apply(0); - - def c = apply(1); - } - - - -object LSTMNode { - - def apply() = { - val n = new LSTMNode; - n.constructGraph; - n - } - - def apply(opts:LSTMNodeOpts) = { - val n = new LSTMNode; - opts.copyOpts(n); - n.constructGraph; - n - } - - class GridOpts extends LSTMNodeOpts {var netType = 0}; - - def grid(nrows:Int, ncols:Int, opts:GridOpts) = { - import BIDMach.networks.layers.Node._ - val nlin = 2; - val nsoft = opts.netType match { - case 0 => 0; - case 1 => 1; - case 2 => 2; - } - val gr = NodeMat(nrows + nlin + nsoft, ncols); - - for (k <- 0 until ncols) { - gr(0, k) = input - } - - for (k <- 0 until ncols) { - gr(1, k) = linear(gr(0, k))(opts.modelName+"Grid_input", outdim=opts.dim, hasBias = opts.hasBias) - } - - for (k <- 0 until ncols) { - for (j <- nlin until nrows + nlin) { - val below = gr(j-1, k); - if (k > 0) { - val left = gr(j, k-1).asInstanceOf[LSTMNode] - gr(j, k) = lstm(h=left.h, c=left.c, i=below)(opts); - } else { - gr(j, k) = lstm(h=null, c=null, i=below)(opts); - } - } - } - - opts.netType match { - case 0 => {} - case 1 => { - for (k <- 0 until ncols) { - gr(nrows + nlin, k) = linear(gr(nrows + nlin - 1, k))(name=opts.modelName+"Grid_output", outdim=opts.outDim, hasBias = opts.hasBias) - gr(nrows + nlin + 1, k) = softmaxout(gr(nrows + nlin, k))(opts.scoreType); - } - } - case 2 => { - for (k <- 0 until ncols) { - gr(nrows + nlin, k) = negsamp(gr(nrows + nlin - 1, k))(name=opts.modelName+"Grid_output", outdim=opts.outDim, hasBias=opts.hasBias, scoreType=opts.scoreType) - } - } - } - gr - } -} - -object LSTMLayer { - - def apply(net:Net) = new LSTMLayer(net, new LSTMNode); - - def apply(net:Net, opts:LSTMNode) = { - val x = new LSTMLayer(net, opts); - x.construct; - x; - } - - -} + */ + +class LSTMLayer(override val net:Net, override val opts:LSTMNode = new LSTMNode) extends CompoundLayer(net, opts) { + override val _inputs = new Array[LayerTerm](3); + override val _outputs = new Array[ND](2); + override val _derivs = new Array[ND](2); + + override def toString = { + "LSTM@"+Integer.toHexString(hashCode % 0x10000).toString + } +} + +trait LSTMNodeOpts extends CompoundNodeOpts { + var dim = 0; + var kind = 1; + var hasBias = false; + var scoreType = 0; + var outdim = 0; + + def copyOpts(opts:LSTMNodeOpts):LSTMNodeOpts = { + super.copyOpts(opts); + opts.dim = dim; + opts.kind = kind; + opts.hasBias = hasBias; + opts.scoreType = scoreType; + opts.outdim = outdim; + opts; + } +} + +class LSTMNode extends CompoundNode with LSTMNodeOpts { + + override val inputs:Array[NodeTerm] = Array(null, null, null); +// override val inputTerminals:Array[Int] = Array(0,0,0); + + def constructGraph = { + kind match { + case 0 => constructGraph0 + case 1 => constructGraph1 + case 2 => constructGraph2 + case 3 => constructGraph3 + case 4 => constructGraph4 + case _ => throw new RuntimeException("LSTMLayer type %d not recognized" format kind); + } + } + + // Basic LSTM topology with 8 linear layers + + def constructGraph0 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + + val in_h = copy; + val in_c = copy; + val in_i = copy; + + val lin1 = linear(in_h)(prefix+"LSTM_h_in_gate", outdim=odim, hasBias=hasBias); + val lin2 = linear(in_h)(prefix+"LSTM_h_out_gate", outdim=odim, hasBias=hasBias); + val lin3 = linear(in_h)(prefix+"LSTM_h_forget_gate", outdim=odim, hasBias=hasBias); + val lin4 = linear(in_h)(prefix+"LSTM_h_tanh_gate", outdim=odim, hasBias=hasBias); + + val lin5 = linear(in_i)(prefix+"LSTM_i_in_gate", outdim=odim, hasBias=hasBias); + val lin6 = linear(in_i)(prefix+"LSTM_i_out_gate", outdim=odim, hasBias=hasBias); + val lin7 = linear(in_i)(prefix+"LSTM_i_forget_gate", outdim=odim, hasBias=hasBias); + val lin8 = linear(in_i)(prefix+"LSTM_i_tanh_gate", outdim=odim, hasBias=hasBias); + + val sum1 = lin1 + lin5; + val sum2 = lin2 + lin6; + val sum3 = lin3 + lin7; + val sum4 = lin4 + lin8; + + val in_gate = σ(sum1); + val out_gate = σ(sum2); + val forget_gate = σ(sum3); + val in_sat = tanh(sum4); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = (in_h on in_c on in_i on null) \ (lin1 \ lin5 \ sum1 \ in_gate \ in_prod \ out_tanh on + lin2 \ lin6 \ sum2 \ out_gate \ f_prod \ out_h on + lin3 \ lin7 \ sum3 \ forget_gate \ out_c \ null on + lin4 \ lin8 \ sum4 \ in_sat \ null \ null); + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); + } + + // LSTM with 4 linear layers, with h and i stacked as inputs + // | Wi_in Wh_in | | i | + | Wi_out Wh_out | | i | + | Wi_forget Wh_forget | | i | + | Wi_g Wh_g | | i | + // | h | | h | + + def constructGraph1 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + + val in_h = copy; + val in_c = copy; + val in_i = copy; + val h_over_i = in_h over in_i; + + val lin1 = linear(h_over_i)(prefix+"LSTM_in_gate", outdim=odim, hasBias=hasBias); + val lin2 = linear(h_over_i)(prefix+"LSTM_out_gate", outdim=odim, hasBias=hasBias); + val lin3 = linear(h_over_i)(prefix+"LSTM_forget_gate", outdim=odim, hasBias=hasBias); + val lin4 = linear(h_over_i)(prefix+"LSTM_tanh_gate", outdim=odim, hasBias=hasBias); + + val in_gate = σ(lin1); + val out_gate = σ(lin2); + val forget_gate = σ(lin3); + val in_sat = tanh(lin4); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = in_h \ lin1 \ in_gate \ in_prod \ out_tanh on + in_c \ lin2 \ out_gate \ f_prod \ out_h on + in_i \ lin3 \ forget_gate \ out_c \ null on + h_over_i \ lin4 \ in_sat \ null \ null; + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); + + } + + // LSTM with 1 linear layer, with h and i stacked as inputs, and all 4 output stacked + // Observed not very stable to train + // + // | Wi_in Wh_in | + // | Wi_out Wh_out | | i | + // | Wi_forget Wh_forget | | h | + // | Wi_g Wh_g | + + def constructGraph2 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + val in_h = copy; + val in_c = copy; + val in_i = copy; + val h_over_i = in_h over in_i; + + val lin = linear(h_over_i)(prefix+"LSTM_all", outdim=4*odim, hasBias=hasBias); + val sp = splitvert(lin, 4); + + val in_gate = σ(sp(0)); + val out_gate = σ(sp(1)); + val forget_gate = σ(sp(2)); + val in_sat = tanh(sp(3)); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = in_h \ lin \ in_gate \ in_prod \ out_tanh on + in_c \ sp \ out_gate \ f_prod \ out_h on + in_i \ null \ forget_gate \ out_c \ null on + h_over_i \ null \ in_sat \ null \ null; + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) + } + + // LSTM with two sets of layers, paired outputs. More stable to train than the single linlayer network + // | Wi_in Wh_in | | i | + | Wi_forget Wh_forget | | i | + // | Wi_out Wh_out | | h | | Wi_g Wh_g | | h | + + def constructGraph3 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + val in_h = copy; + val in_c = copy; + val in_i = copy; + val h_over_i = in_h over in_i; + + val lin1 = linear(h_over_i)(prefix+"LSTM_in_out", outdim=2*odim, hasBias=hasBias); + val sp1 = splitvert(lin1, 2); + val lin2 = linear(h_over_i)(prefix+"LSTM_forget_tanh", outdim=2*odim, hasBias=hasBias); + val sp2 = splitvert(lin2, 2); + + val in_gate = σ(sp1(0)); + val out_gate = σ(sp1(1)); + val forget_gate = σ(sp2(0)); + val in_sat = tanh(sp2(1)); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = in_h \ lin1 \ in_gate \ in_prod \ out_tanh on + in_c \ sp1 \ out_gate \ f_prod \ out_h on + in_i \ lin2 \ forget_gate \ out_c \ null on + h_over_i \ sp2 \ in_sat \ null \ null; + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) + } + + // LSTM with 2 linear layers from h and i respectively + + def constructGraph4 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + val in_h = copy; + val in_c = copy; + val in_i = copy; + + val linh = linear(in_h)(prefix+"LSTM_h", outdim=4*odim, hasBias=hasBias); + val sph = splitvert(linh, 4); + val lini = linear(in_i)(prefix+"LSTM_i", outdim=4*odim, hasBias=hasBias); + val spi = splitvert(lini, 4); + + val lin1 = sph(0) + spi(0); + val lin2 = sph(1) + spi(1); + val lin3 = sph(2) + spi(2); + val lin4 = sph(3) + spi(3); + + val in_gate = σ(lin1); + val out_gate = σ(lin2); + val forget_gate = σ(lin3); + val in_sat = tanh(lin4); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = (in_h on in_c on in_i on null) \ (linh \ lin1 \ in_gate \ in_prod \ out_tanh on + sph \ lin2 \ out_gate \ f_prod \ out_h on + lini \ lin3 \ forget_gate \ out_c \ null on + spi \ lin4 \ in_sat \ null \ null); + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) + } + + override def clone:LSTMNode = { + copyTo(new LSTMNode).asInstanceOf[LSTMNode]; + } + + override def create(net:Net):LSTMLayer = { + LSTMLayer(net, this); + } + + override def toString = { + "LSTM@"+Integer.toHexString(hashCode % 0x10000).toString + } + + def h = apply(0); + + def c = apply(1); + } + + + +object LSTMNode { + + final val gridTypeNoOutput = 0; + final val gridTypeSoftmaxOutput = 1; + final val gridTypeNegsampOutput = 2; + + def apply() = { + val n = new LSTMNode; + n.constructGraph; + n + } + + def apply(opts:LSTMNodeOpts) = { + val n = new LSTMNode; + opts.copyOpts(n); + n.constructGraph; + n + } + + class GridOpts extends LSTMNodeOpts {var netType = 0; var bylevel = true}; + + def grid(nrows:Int, ncols:Int, opts:GridOpts):NodeMat = { + import BIDMach.networks.layers.Node._ + val nlin = 2; + val odim = opts.outdim; + val idim = opts.dim; + val nsoft = opts.netType match { + case `gridTypeNoOutput` => 0; + case `gridTypeNegsampOutput` => 1; + case `gridTypeSoftmaxOutput` => 2; + } + val gr = NodeMat(nrows + nlin + nsoft, ncols); + + for (k <- 0 until ncols) { + gr(0, k) = input + } + + val modelName = opts.modelName; + + for (k <- 0 until ncols) { + gr(1, k) = linear(gr(0, k))((modelName format 0) +"_bottom", outdim=idim, hasBias = opts.hasBias) + } + + for (k <- 0 until ncols) { + for (j <- nlin until nrows + nlin) { + val modelName = if (opts.bylevel) (opts.modelName format j-nlin) else (opts.modelName format 0) + val below = gr(j-1, k); + if (k > 0) { + val left = gr(j, k-1).asInstanceOf[LSTMNode] + gr(j, k) = lstm(h=left.h, c=left.c, i=below, m=modelName)(opts); + } else { + gr(j, k) = lstm(h=null, c=null, i=below, m=modelName)(opts); + } + } + } + + opts.netType match { + case `gridTypeNoOutput` => {} + case `gridTypeSoftmaxOutput` => { + for (k <- 0 until ncols) { + gr(nrows + nlin, k) = linear(gr(nrows + nlin - 1, k))(name=opts.modelName+"_top", outdim=odim, hasBias = opts.hasBias) + gr(nrows + nlin + 1, k) = softmaxout(gr(nrows + nlin, k))(opts.scoreType); + } + } + case `gridTypeNegsampOutput` => { + for (k <- 0 until ncols) { + gr(nrows + nlin, k) = negsamp(gr(nrows + nlin - 1, k))(name=opts.modelName+"_top", outdim=odim, hasBias=opts.hasBias, scoreType=opts.scoreType) + } + } + } + gr + } +} + +object LSTMLayer { + + def apply(net:Net) = new LSTMLayer(net, new LSTMNode); + + def apply(net:Net, opts:LSTMNode) = { + val x = new LSTMLayer(net, opts); + x.construct; + x; + } + + def grid(net:Net, nrows:Int, ncols:Int, opts:LSTMNode.GridOpts):LayerMat = { + val nodeGrid = LSTMNode.grid(nrows, ncols, opts); + LayerMat(nodeGrid, net); + } +} From 2d17811189a9fcb837206ae474042363dd45c34c Mon Sep 17 00:00:00 2001 From: Bobby Jaros Date: Tue, 8 Mar 2016 16:51:22 -0800 Subject: [PATCH 5/6] Bad merge, fixing these two files (again) --- .../scala/BIDMach/networks/SeqToSeq.scala | 678 ++++++++-------- .../scala/BIDMach/networks/layers/LSTM.scala | 746 +++++++++--------- 2 files changed, 707 insertions(+), 717 deletions(-) diff --git a/src/main/scala/BIDMach/networks/SeqToSeq.scala b/src/main/scala/BIDMach/networks/SeqToSeq.scala index 92ca8d5e..5595254c 100644 --- a/src/main/scala/BIDMach/networks/SeqToSeq.scala +++ b/src/main/scala/BIDMach/networks/SeqToSeq.scala @@ -1,340 +1,340 @@ -package BIDMach.networks - -import BIDMat.{Mat,SBMat,CMat,CSMat,DMat,FMat,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,SMat,SDMat} -import BIDMat.MatFunctions._ -import BIDMat.SciFunctions._ -import BIDMach.datasources._ -import BIDMach.datasinks._ -import BIDMach.updaters._ -import BIDMach.mixins._ -import BIDMach.models._ -import BIDMach.networks.layers._ -import BIDMach._ - -/* +package BIDMach.networks + +import BIDMat.{Mat,SBMat,CMat,CSMat,DMat,FMat,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,SMat,SDMat} +import BIDMat.MatFunctions._ +import BIDMat.SciFunctions._ +import BIDMach.datasources._ +import BIDMach.datasinks._ +import BIDMach.updaters._ +import BIDMach.mixins._ +import BIDMach.models._ +import BIDMach.networks.layers._ +import BIDMach._ + +/* * LSTM next Word prediction model, which comprises a rectangular grid of LSTM compound layers. - */ -class SeqToSeq(override val opts:SeqToSeq.Opts = new SeqToSeq.Options) extends Net(opts) { - - var PADrow:Mat = null; - var OOVelem:Mat = null; - var leftEdge:Layer = null; - var leftStart:Mat = null; - var dstxdata:Mat = null; - var dstxdata0:Mat = null; - var srcGrid:LayerMat = null; - var dstGrid:LayerMat = null; - var srcGridOpts:LSTMNode.GridOpts = null; - var dstGridOpts:LSTMNode.GridOpts = null; - var height = 0; - var inwidth = 0; - var outwidth = 0; - var width = 0; - var srcn = 0; - var dstxn = 0; - var dstyn = 0; - val preamble_rows = 2; - - override def createLayers = { - height = opts.height; - inwidth = opts.inwidth; - outwidth = opts.outwidth; - leftEdge = InputLayer(this); // dummy layer, left edge of zeros - - srcGridOpts = new LSTMNode.GridOpts; - srcGridOpts.copyFrom(opts); - srcGridOpts.modelName = "src_level%d"; - srcGridOpts.netType = LSTMNode.gridTypeNoOutput; - srcGrid = LSTMLayer.grid(this, height, inwidth, srcGridOpts); - layers = srcGrid.data.filter(_ != null); - for (i <- 0 until height) srcGrid(i+preamble_rows, 0).setInputs(leftEdge, leftEdge); - - if (! opts.embed) { - dstGridOpts = new LSTMNode.GridOpts; - dstGridOpts.copyFrom(opts); - dstGridOpts.modelName = "dst_level%d"; - dstGridOpts.netType = LSTMNode.gridTypeSoftmaxOutput; - dstGridOpts.outdim = opts.nvocab; - dstGrid = LSTMLayer.grid(this, height, outwidth, dstGridOpts); - - srcGrid link dstGrid; - layers = layers ++ dstGrid.data.filter(_ != null); - output_layers = new Array[Layer](outwidth); - for (i <- 0 until outwidth) output_layers(i) = dstGrid(dstGrid.nrows-1, i); - } - } - - def mapOOV(in:Mat) = { - if (OOVelem.asInstanceOf[AnyRef] == null) { - OOVelem = convertMat(iones(1,1) * opts.OOVsym); - } - in ~ in + ((in >= opts.nvocab) ∘ (OOVelem - in)) - } - - override def assignInputs(gmats:Array[Mat], ipass:Int, pos:Long) = { - val src = gmats(0); - srcn = src.nnz/src.ncols; - if (srcn*src.ncols != src.nnz) throw new RuntimeException("SeqToSeq src batch not fixed length"); - val srcdata = int(src.contents.view(srcn, batchSize).t); // IMat with columns corresponding to word positions, with batchSize rows. - mapOOV(srcdata); - val srcmat = oneHot(srcdata.contents, opts.nvocab); - srcn = math.min(srcn, opts.inwidth); - if (srcn < inwidth) initPrevCol; - for (i <- 0 until srcn) { - val cols = srcmat.colslice(i*batchSize, (i+1)*batchSize); - srcGrid(0, inwidth + i - srcn).output = cols; - } - if (leftEdge.output.asInstanceOf[AnyRef] == null) { - leftEdge.output = convertMat(zeros(opts.dim \ batchSize)); - } - - if (! opts.embed) { - val dstx = gmats(1); - val dstxn0 = dstx.nnz/dstx.ncols; - if (dstxn0*dstx.ncols != dstx.nnz) throw new RuntimeException("SeqToSeq dstx batch not fixed length"); - val dstxdata0 = int(dstx.contents.view(dstxn0, batchSize).t); - dstxn = dstxn0 + (if (opts.addStart) 1 else 0); - if (opts.addStart && (leftStart.asInstanceOf[AnyRef] == null)) { - leftStart = convertMat(izeros(batchSize, 1)); - } - val dstxdata = if (opts.addStart) (leftStart \ dstxdata0) else dstxdata0; - mapOOV(dstxdata); - val dstxmat = oneHot(dstxdata.contents, opts.nvocab); - - dstxn = math.min(dstxn, opts.outwidth); - for (i <- 0 until dstxn) { - val cols = dstxmat.colslice(i*batchSize, (i+1)*batchSize); - dstGrid(0, i).output = cols; - } - } - } - - def initPrevCol = { - for (i <- 0 until height) { - val leftlayer = srcGrid(i+preamble_rows, inwidth-srcn-1); - if (leftlayer.output.asInstanceOf[AnyRef] == null) { - leftlayer.output = convertMat(zeros(opts.dim \ batchSize)); - } - if (leftlayer.outputs(1).asInstanceOf[AnyRef] == null) { - leftlayer.setOutput(1, convertMat(zeros(opts.dim \ batchSize))); - } - } - } - - override def assignTargets(gmats:Array[Mat], ipass:Int, pos:Long) { - val dsty = if (gmats.length > 2) gmats(2) else gmats(1); - val dstyn0 = dsty.nnz/dsty.ncols; - if (dstyn0*dsty.ncols != dsty.nnz) throw new RuntimeException("SeqToSeq dsty batch not fixed length"); - val dstydata = int(dsty.contents.view(dstyn0, batchSize).t); - mapOOV(dstydata); - val dstyn1 = math.min(dstyn0 - (if (opts.addStart) 0 else 1), opts.outwidth); - for (j <- 0 until dstyn1) { - val incol = if (opts.addStart) dstydata.colslice(j,j+1).t else dstydata.colslice(j+1,j+2).t - output_layers(j).target = incol; - } - if (PADrow.asInstanceOf[AnyRef] == null) { - PADrow = convertMat(iones(1, batchSize) * opts.PADsym); - } - dstyn = math.min(dstyn1 + 1, opts.outwidth); - if (dstyn1 < opts.outwidth) { - output_layers(dstyn1).target = PADrow; - } - } - - override def dobatch(gmats:Array[Mat], ipass:Int, pos:Long):Unit = { - if (batchSize < 0) batchSize = gmats(0).ncols; - if (batchSize == gmats(0).ncols) { // discard odd-sized minibatches - assignInputs(gmats, ipass, pos); - assignTargets(gmats, ipass, pos); - if (mask.asInstanceOf[AnyRef] != null) { - modelmats(0) ~ modelmats(0) ∘ mask; - } - val mincol = inwidth - srcn; - val maxcol = dstxn; - srcGrid.forward(mincol, inwidth-1, opts.debug); - dstGrid.forward(0, maxcol-1, opts.debug); - - output_layers.map((layer:Layer) => layer match { - case _:OutputLayer => {} - case _ => {if (layer.deriv.asInstanceOf[AnyRef] != null) layer.deriv.set(1);} - }) - if (opts.aopts == null) updatemats.map(_.clear); - - dstGrid.backward(0, maxcol-1, opts.debug, ipass, pos); - srcGrid.backward(mincol, inwidth-1, opts.debug, ipass, pos); - } - } - - override def evalbatch(mats:Array[Mat], ipass:Int, pos:Long):FMat = { - if (batchSize < 0) batchSize = gmats(0).ncols; - if (batchSize == gmats(0).ncols) { - assignInputs(gmats, ipass, pos); - if (mask.asInstanceOf[AnyRef] != null) { - modelmats(0) ~ modelmats(0) ∘ mask; - } - val mincol = inwidth - srcn; - srcGrid.forward(mincol, inwidth-1, opts.debug); - if (! opts.embed) { - val maxcol = dstxn; - assignTargets(gmats, ipass, pos); - dstGrid.forward(0, maxcol-1, opts.debug); - if (putBack >= 0) { - output_layers(dstxn-1).output.colslice(0, gmats(0).ncols, gmats(1)); - } - var score = 0f; - var j = 0; - while (j < dstxn-1) { - score += output_layers(j).score.v; - j += 1; - } - row(score/(dstxn-1)); - } else { - if (ogmats != null) { - ogmats(0) = srcGrid(height+preamble_rows-1, srcGrid.ncols-1).output.asMat; - } - zeros(1,1); - } - } else { - zeros(1, 1); - } - } -} - -object SeqToSeq { - trait Opts extends Net.Opts { - var inwidth = 1; // Max src sentence length - var outwidth = 1; // Max dst sentence lenth - var height = 1; // Number of LSTM layers vertically - var nvocab = 100000; // Vocabulary size - var kind = 0; // LSTM type, see below - var bylevel = true; // Use different models for each level - var netType = 0; // Network type, 0 = softmax output, 1 = Neg Sampling output - var PADsym = 1; // Padding symbol - var OOVsym = 2; // OOV symbol - var STARTsym = 1; // Start symbol - var addStart = true; // Add the start symbol to dst sentences - var scoreType = 0; // Score type, 0 = LL, 1 = accuracy, 2 = LL of full Softmax, 3 = accuracy of full Softmax - var nsamps = 100; // Number of negative samples - var expt = 0.8f; // Negative sampling exponent (tail boost) - var embed = false; - - } - - class Options extends Opts {} - - def mkNetModel(fopts:Model.Opts) = { - new SeqToSeq(fopts.asInstanceOf[SeqToSeq.Opts]) - } - - def mkUpdater(nopts:Updater.Opts) = { - new ADAGrad(nopts.asInstanceOf[ADAGrad.Opts]) - } - - def mkRegularizer(nopts:Mixin.Opts):Array[Mixin] = { - Array(new L1Regularizer(nopts.asInstanceOf[L1Regularizer.Opts])) - } - - class LearnOptions extends Learner.Options with SeqToSeq.Opts with MatSource.Opts with ADAGrad.Opts with L1Regularizer.Opts - - def learner(mat0:Mat, mat1:Mat, regularize:Boolean = false) = { - val opts = new LearnOptions; - opts.batchSize = 128; - val nn = new Learner( - new MatSource(Array(mat0, mat1), opts), - new SeqToSeq(opts), - if (regularize) Array(new L1Regularizer(opts)) else null, - new ADAGrad(opts), - null, - opts) - (nn, opts) - } - - def learnerX(mat0:Mat, mat1:Mat) = { - val opts = new LearnOptions; - opts.batchSize = math.min(100000, mat0.ncols/30 + 1); - val nn = new Learner( - new MatSource(Array(mat0, mat1), opts), - new SeqToSeq(opts), - null, - null, - null, - opts) - (nn, opts) - } - - class FDSopts extends Learner.Options with SeqToSeq.Opts with FileSource.Opts with ADAGrad.Opts with L1Regularizer.Opts - - def learner(fn1:String, fn2:String, regularize:Boolean, adagrad:Boolean):(Learner, FDSopts) = learner(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0)), regularize, adagrad); - - def learner(fn1:String, fn2:String):(Learner, FDSopts) = learner(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0)), false, true); - - def learnerX(fn1:String, fn2:String):(Learner, FDSopts) = learnerX(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0))); - - def learner(fnames:List[(Int)=>String]):(Learner, FDSopts) = learner(fnames, false, true); - - def learner(fnames:List[(Int)=>String], regularize:Boolean, adagrad:Boolean):(Learner, FDSopts) = { - val opts = new FDSopts; - opts.fnames = fnames - opts.batchSize = 128; - opts.eltsPerSample = 500; - implicit val threads = threadPool(4); - val ds = new FileSource(opts) - val nn = new Learner( - ds, - new SeqToSeq(opts), - if (regularize) Array(new L1Regularizer(opts)) else null, - if (adagrad) new ADAGrad(opts) else new Grad(opts), - null, - opts) - (nn, opts) - } - - def learnerX(fnames:List[(Int)=>String]):(Learner, FDSopts) = { - val opts = new FDSopts; - opts.fnames = fnames - opts.batchSize = 128; - opts.eltsPerSample = 500; - implicit val threads = threadPool(4); - val ds = new FileSource(opts) - val nn = new Learner( - ds, - new SeqToSeq(opts), - null, - null, - null, - opts) - (nn, opts) - } - - class FEopts extends Learner.Options with SeqToSeq.Opts with FileSource.Opts with FileSink.Opts - - def embed(model:SeqToSeq, ifname:String, ofname:String):(Learner, FEopts) = { - val opts = new FEopts; - opts.copyFrom(model.opts); - opts.fnames = List(FileSource.simpleEnum(ifname,1,0)); - opts.ofnames = List(FileSource.simpleEnum(ofname,1,0)); - opts.embed = true; - val newmod = new SeqToSeq(opts); - newmod.refresh = false; - model.copyTo(newmod); - implicit val threads = threadPool(4); - val ds = new FileSource(opts) - val nn = new Learner( - new FileSource(opts), - newmod, - null, - null, - new FileSink(opts), - opts) - (nn, opts) - } - - def load(fname:String):SeqToSeq = { - val mm = new SeqToSeq; - mm.loadMetaData(fname); - mm.load(fname); - mm - } - -} - + */ +class SeqToSeq(override val opts:SeqToSeq.Opts = new SeqToSeq.Options) extends Net(opts) { + + var PADrow:Mat = null; + var OOVelem:Mat = null; + var leftEdge:Layer = null; + var leftStart:Mat = null; + var dstxdata:Mat = null; + var dstxdata0:Mat = null; + var srcGrid:LayerMat = null; + var dstGrid:LayerMat = null; + var srcGridOpts:LSTMNode.GridOpts = null; + var dstGridOpts:LSTMNode.GridOpts = null; + var height = 0; + var inwidth = 0; + var outwidth = 0; + var width = 0; + var srcn = 0; + var dstxn = 0; + var dstyn = 0; + val preamble_rows = 2; + + override def createLayers = { + height = opts.height; + inwidth = opts.inwidth; + outwidth = opts.outwidth; + leftEdge = InputLayer(this); // dummy layer, left edge of zeros + + srcGridOpts = new LSTMNode.GridOpts; + srcGridOpts.copyFrom(opts); + srcGridOpts.modelName = "src_level%d"; + srcGridOpts.netType = LSTMNode.gridTypeNoOutput; + srcGrid = LSTMLayer.grid(this, height, inwidth, srcGridOpts); + layers = srcGrid.data.filter(_ != null); + for (i <- 0 until height) srcGrid(i+preamble_rows, 0).setInputs(leftEdge, leftEdge); + + if (! opts.embed) { + dstGridOpts = new LSTMNode.GridOpts; + dstGridOpts.copyFrom(opts); + dstGridOpts.modelName = "dst_level%d"; + dstGridOpts.netType = LSTMNode.gridTypeSoftmaxOutput; + dstGridOpts.outdim = opts.nvocab; + dstGrid = LSTMLayer.grid(this, height, outwidth, dstGridOpts); + + srcGrid link dstGrid; + layers = layers ++ dstGrid.data.filter(_ != null); + output_layers = new Array[Layer](outwidth); + for (i <- 0 until outwidth) output_layers(i) = dstGrid(dstGrid.nrows-1, i); + } + } + + def mapOOV(in:Mat) = { + if (OOVelem.asInstanceOf[AnyRef] == null) { + OOVelem = convertMat(iones(1,1) * opts.OOVsym); + } + in ~ in + ((in >= opts.nvocab) ∘ (OOVelem - in)) + } + + override def assignInputs(gmats:Array[Mat], ipass:Int, pos:Long) = { + val src = gmats(0); + srcn = src.nnz/src.ncols; + if (srcn*src.ncols != src.nnz) throw new RuntimeException("SeqToSeq src batch not fixed length"); + val srcdata = int(src.contents.view(srcn, batchSize).t); // IMat with columns corresponding to word positions, with batchSize rows. + mapOOV(srcdata); + val srcmat = oneHot(srcdata.contents, opts.nvocab); + srcn = math.min(srcn, opts.inwidth); + if (srcn < inwidth) initPrevCol; + for (i <- 0 until srcn) { + val cols = srcmat.colslice(i*batchSize, (i+1)*batchSize); + srcGrid(0, inwidth + i - srcn).output = cols; + } + if (leftEdge.output.asInstanceOf[AnyRef] == null) { + leftEdge.output = convertMat(zeros(opts.dim \ batchSize)); + } + + if (! opts.embed) { + val dstx = gmats(1); + val dstxn0 = dstx.nnz/dstx.ncols; + if (dstxn0*dstx.ncols != dstx.nnz) throw new RuntimeException("SeqToSeq dstx batch not fixed length"); + val dstxdata0 = int(dstx.contents.view(dstxn0, batchSize).t); + dstxn = dstxn0 + (if (opts.addStart) 1 else 0); + if (opts.addStart && (leftStart.asInstanceOf[AnyRef] == null)) { + leftStart = convertMat(izeros(batchSize, 1)); + } + val dstxdata = if (opts.addStart) (leftStart \ dstxdata0) else dstxdata0; + mapOOV(dstxdata); + val dstxmat = oneHot(dstxdata.contents, opts.nvocab); + + dstxn = math.min(dstxn, opts.outwidth); + for (i <- 0 until dstxn) { + val cols = dstxmat.colslice(i*batchSize, (i+1)*batchSize); + dstGrid(0, i).output = cols; + } + } + } + + def initPrevCol = { + for (i <- 0 until height) { + val leftlayer = srcGrid(i+preamble_rows, inwidth-srcn-1); + if (leftlayer.output.asInstanceOf[AnyRef] == null) { + leftlayer.output = convertMat(zeros(opts.dim \ batchSize)); + } + if (leftlayer.outputs(1).asInstanceOf[AnyRef] == null) { + leftlayer.setOutput(1, convertMat(zeros(opts.dim \ batchSize))); + } + } + } + + override def assignTargets(gmats:Array[Mat], ipass:Int, pos:Long) { + val dsty = if (gmats.length > 2) gmats(2) else gmats(1); + val dstyn0 = dsty.nnz/dsty.ncols; + if (dstyn0*dsty.ncols != dsty.nnz) throw new RuntimeException("SeqToSeq dsty batch not fixed length"); + val dstydata = int(dsty.contents.view(dstyn0, batchSize).t); + mapOOV(dstydata); + val dstyn1 = math.min(dstyn0 - (if (opts.addStart) 0 else 1), opts.outwidth); + for (j <- 0 until dstyn1) { + val incol = if (opts.addStart) dstydata.colslice(j,j+1).t else dstydata.colslice(j+1,j+2).t + output_layers(j).target = incol; + } + if (PADrow.asInstanceOf[AnyRef] == null) { + PADrow = convertMat(iones(1, batchSize) * opts.PADsym); + } + dstyn = math.min(dstyn1 + 1, opts.outwidth); + if (dstyn1 < opts.outwidth) { + output_layers(dstyn1).target = PADrow; + } + } + + override def dobatch(gmats:Array[Mat], ipass:Int, pos:Long):Unit = { + if (batchSize < 0) batchSize = gmats(0).ncols; + if (batchSize == gmats(0).ncols) { // discard odd-sized minibatches + assignInputs(gmats, ipass, pos); + assignTargets(gmats, ipass, pos); + if (mask.asInstanceOf[AnyRef] != null) { + modelmats(0) ~ modelmats(0) ∘ mask; + } + val mincol = inwidth - srcn; + val maxcol = dstxn; + srcGrid.forward(mincol, inwidth-1, opts.debug); + dstGrid.forward(0, maxcol-1, opts.debug); + + output_layers.map((layer:Layer) => layer match { + case _:OutputLayer => {} + case _ => {if (layer.deriv.asInstanceOf[AnyRef] != null) layer.deriv.set(1);} + }) + if (opts.aopts == null) updatemats.map(_.clear); + + dstGrid.backward(0, maxcol-1, opts.debug, ipass, pos); + srcGrid.backward(mincol, inwidth-1, opts.debug, ipass, pos); + } + } + + override def evalbatch(mats:Array[Mat], ipass:Int, pos:Long):FMat = { + if (batchSize < 0) batchSize = gmats(0).ncols; + if (batchSize == gmats(0).ncols) { + assignInputs(gmats, ipass, pos); + if (mask.asInstanceOf[AnyRef] != null) { + modelmats(0) ~ modelmats(0) ∘ mask; + } + val mincol = inwidth - srcn; + srcGrid.forward(mincol, inwidth-1, opts.debug); + if (! opts.embed) { + val maxcol = dstxn; + assignTargets(gmats, ipass, pos); + dstGrid.forward(0, maxcol-1, opts.debug); + if (putBack >= 0) { + output_layers(dstxn-1).output.colslice(0, gmats(0).ncols, gmats(1)); + } + var score = 0f; + var j = 0; + while (j < dstxn-1) { + score += output_layers(j).score.v; + j += 1; + } + row(score/(dstxn-1)); + } else { + if (ogmats != null) { + ogmats(0) = srcGrid(height+preamble_rows-1, srcGrid.ncols-1).output.asMat; + } + zeros(1,1); + } + } else { + zeros(1, 1); + } + } +} + +object SeqToSeq { + trait Opts extends Net.Opts { + var inwidth = 1; // Max src sentence length + var outwidth = 1; // Max dst sentence lenth + var height = 1; // Number of LSTM layers vertically + var nvocab = 100000; // Vocabulary size + var kind = 0; // LSTM type, see below + var bylevel = true; // Use different models for each level + var netType = 0; // Network type, 0 = softmax output, 1 = Neg Sampling output + var PADsym = 1; // Padding symbol + var OOVsym = 2; // OOV symbol + var STARTsym = 1; // Start symbol + var addStart = true; // Add the start symbol to dst sentences + var scoreType = 0; // Score type, 0 = LL, 1 = accuracy, 2 = LL of full Softmax, 3 = accuracy of full Softmax + var nsamps = 100; // Number of negative samples + var expt = 0.8f; // Negative sampling exponent (tail boost) + var embed = false; + + } + + class Options extends Opts {} + + def mkNetModel(fopts:Model.Opts) = { + new SeqToSeq(fopts.asInstanceOf[SeqToSeq.Opts]) + } + + def mkUpdater(nopts:Updater.Opts) = { + new ADAGrad(nopts.asInstanceOf[ADAGrad.Opts]) + } + + def mkRegularizer(nopts:Mixin.Opts):Array[Mixin] = { + Array(new L1Regularizer(nopts.asInstanceOf[L1Regularizer.Opts])) + } + + class LearnOptions extends Learner.Options with SeqToSeq.Opts with MatSource.Opts with ADAGrad.Opts with L1Regularizer.Opts + + def learner(mat0:Mat, mat1:Mat, regularize:Boolean = false) = { + val opts = new LearnOptions; + opts.batchSize = 128; + val nn = new Learner( + new MatSource(Array(mat0, mat1), opts), + new SeqToSeq(opts), + if (regularize) Array(new L1Regularizer(opts)) else null, + new ADAGrad(opts), + null, + opts) + (nn, opts) + } + + def learnerX(mat0:Mat, mat1:Mat) = { + val opts = new LearnOptions; + opts.batchSize = math.min(100000, mat0.ncols/30 + 1); + val nn = new Learner( + new MatSource(Array(mat0, mat1), opts), + new SeqToSeq(opts), + null, + null, + null, + opts) + (nn, opts) + } + + class FDSopts extends Learner.Options with SeqToSeq.Opts with FileSource.Opts with ADAGrad.Opts with L1Regularizer.Opts + + def learner(fn1:String, fn2:String, regularize:Boolean, adagrad:Boolean):(Learner, FDSopts) = learner(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0)), regularize, adagrad); + + def learner(fn1:String, fn2:String):(Learner, FDSopts) = learner(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0)), false, true); + + def learnerX(fn1:String, fn2:String):(Learner, FDSopts) = learnerX(List(FileSource.simpleEnum(fn1,1,0), FileSource.simpleEnum(fn2,1,0))); + + def learner(fnames:List[(Int)=>String]):(Learner, FDSopts) = learner(fnames, false, true); + + def learner(fnames:List[(Int)=>String], regularize:Boolean, adagrad:Boolean):(Learner, FDSopts) = { + val opts = new FDSopts; + opts.fnames = fnames + opts.batchSize = 128; + opts.eltsPerSample = 500; + implicit val threads = threadPool(4); + val ds = new FileSource(opts) + val nn = new Learner( + ds, + new SeqToSeq(opts), + if (regularize) Array(new L1Regularizer(opts)) else null, + if (adagrad) new ADAGrad(opts) else new Grad(opts), + null, + opts) + (nn, opts) + } + + def learnerX(fnames:List[(Int)=>String]):(Learner, FDSopts) = { + val opts = new FDSopts; + opts.fnames = fnames + opts.batchSize = 128; + opts.eltsPerSample = 500; + implicit val threads = threadPool(4); + val ds = new FileSource(opts) + val nn = new Learner( + ds, + new SeqToSeq(opts), + null, + null, + null, + opts) + (nn, opts) + } + + class FEopts extends Learner.Options with SeqToSeq.Opts with FileSource.Opts with FileSink.Opts + + def embed(model:SeqToSeq, ifname:String, ofname:String):(Learner, FEopts) = { + val opts = new FEopts; + opts.copyFrom(model.opts); + opts.fnames = List(FileSource.simpleEnum(ifname,1,0)); + opts.ofnames = List(FileSource.simpleEnum(ofname,1,0)); + opts.embed = true; + val newmod = new SeqToSeq(opts); + newmod.refresh = false; + model.copyTo(newmod); + implicit val threads = threadPool(4); + val ds = new FileSource(opts) + val nn = new Learner( + new FileSource(opts), + newmod, + null, + null, + new FileSink(opts), + opts) + (nn, opts) + } + + def load(fname:String):SeqToSeq = { + val mm = new SeqToSeq; + mm.loadMetaData(fname); + mm.load(fname); + mm + } + +} + diff --git a/src/main/scala/BIDMach/networks/layers/LSTM.scala b/src/main/scala/BIDMach/networks/layers/LSTM.scala index 139e6260..bc4a3d27 100644 --- a/src/main/scala/BIDMach/networks/layers/LSTM.scala +++ b/src/main/scala/BIDMach/networks/layers/LSTM.scala @@ -1,379 +1,369 @@ -package BIDMach.networks.layers - -import BIDMat.{Mat,ND,SBMat,CMat,CSMat,DMat,FMat,FND,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,GND,SMat,SDMat} -import BIDMat.MatFunctions._ -import BIDMat.SciFunctions._ -import BIDMach.datasources._ -import BIDMach.updaters._ -import BIDMach.mixins._ -import BIDMach.models._ -import BIDMach.networks._ -import BIDMach._ -import scala.util.hashing.MurmurHash3; -import scala.collection.mutable.HashMap; - -/** +package BIDMach.networks.layers + +import BIDMat.{Mat,ND,SBMat,CMat,CSMat,DMat,FMat,FND,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,GND,SMat,SDMat} +import BIDMat.MatFunctions._ +import BIDMat.SciFunctions._ +import BIDMach.datasources._ +import BIDMach.updaters._ +import BIDMach.mixins._ +import BIDMach.models._ +import BIDMach.networks._ +import BIDMach._ +import scala.util.hashing.MurmurHash3; +import scala.collection.mutable.HashMap; + +/** * LSTM unit - */ - -class LSTMLayer(override val net:Net, override val opts:LSTMNode = new LSTMNode) extends CompoundLayer(net, opts) { - override val _inputs = new Array[LayerTerm](3); - override val _outputs = new Array[ND](2); - override val _derivs = new Array[ND](2); - - override def toString = { - "LSTM@"+Integer.toHexString(hashCode % 0x10000).toString - } -} - -trait LSTMNodeOpts extends CompoundNodeOpts { - var dim = 0; - var kind = 1; - var hasBias = false; - var scoreType = 0; - var outdim = 0; - - def copyOpts(opts:LSTMNodeOpts):LSTMNodeOpts = { - super.copyOpts(opts); - opts.dim = dim; - opts.kind = kind; - opts.hasBias = hasBias; - opts.scoreType = scoreType; - opts.outdim = outdim; - opts; - } -} - -class LSTMNode extends CompoundNode with LSTMNodeOpts { - - override val inputs:Array[NodeTerm] = Array(null, null, null); -// override val inputTerminals:Array[Int] = Array(0,0,0); - - def constructGraph = { - kind match { - case 0 => constructGraph0 - case 1 => constructGraph1 - case 2 => constructGraph2 - case 3 => constructGraph3 - case 4 => constructGraph4 - case _ => throw new RuntimeException("LSTMLayer type %d not recognized" format kind); - } - } - - // Basic LSTM topology with 8 linear layers - - def constructGraph0 = { - import BIDMach.networks.layers.Node._ - val odim = dim; - - val in_h = copy; - val in_c = copy; - val in_i = copy; - - val lin1 = linear(in_h)(prefix+"LSTM_h_in_gate", outdim=odim, hasBias=hasBias); - val lin2 = linear(in_h)(prefix+"LSTM_h_out_gate", outdim=odim, hasBias=hasBias); - val lin3 = linear(in_h)(prefix+"LSTM_h_forget_gate", outdim=odim, hasBias=hasBias); - val lin4 = linear(in_h)(prefix+"LSTM_h_tanh_gate", outdim=odim, hasBias=hasBias); - - val lin5 = linear(in_i)(prefix+"LSTM_i_in_gate", outdim=odim, hasBias=hasBias); - val lin6 = linear(in_i)(prefix+"LSTM_i_out_gate", outdim=odim, hasBias=hasBias); - val lin7 = linear(in_i)(prefix+"LSTM_i_forget_gate", outdim=odim, hasBias=hasBias); - val lin8 = linear(in_i)(prefix+"LSTM_i_tanh_gate", outdim=odim, hasBias=hasBias); - - val sum1 = lin1 + lin5; - val sum2 = lin2 + lin6; - val sum3 = lin3 + lin7; - val sum4 = lin4 + lin8; - - val in_gate = σ(sum1); - val out_gate = σ(sum2); - val forget_gate = σ(sum3); - val in_sat = tanh(sum4); - - val in_prod = in_gate ∘ in_sat; - val f_prod = forget_gate ∘ in_c; - val out_c = in_prod + f_prod; - - val out_tanh = tanh(out_c); - val out_h = out_gate ∘ out_tanh; - - grid = (in_h on in_c on in_i on null) \ (lin1 \ lin5 \ sum1 \ in_gate \ in_prod \ out_tanh on - lin2 \ lin6 \ sum2 \ out_gate \ f_prod \ out_h on - lin3 \ lin7 \ sum3 \ forget_gate \ out_c \ null on - lin4 \ lin8 \ sum4 \ in_sat \ null \ null); - - val lopts = grid.data; - lopts.map((x:Node) => if (x != null) x.parent = this); - outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); - } - - // LSTM with 4 linear layers, with h and i stacked as inputs - // | Wi_in Wh_in | | i | + | Wi_out Wh_out | | i | + | Wi_forget Wh_forget | | i | + | Wi_g Wh_g | | i | - // | h | | h | - - def constructGraph1 = { - import BIDMach.networks.layers.Node._ - val odim = dim; - - val in_h = copy; - val in_c = copy; - val in_i = copy; - val h_over_i = in_h over in_i; - - val lin1 = linear(h_over_i)(prefix+"LSTM_in_gate", outdim=odim, hasBias=hasBias); - val lin2 = linear(h_over_i)(prefix+"LSTM_out_gate", outdim=odim, hasBias=hasBias); - val lin3 = linear(h_over_i)(prefix+"LSTM_forget_gate", outdim=odim, hasBias=hasBias); - val lin4 = linear(h_over_i)(prefix+"LSTM_tanh_gate", outdim=odim, hasBias=hasBias); - - val in_gate = σ(lin1); - val out_gate = σ(lin2); - val forget_gate = σ(lin3); - val in_sat = tanh(lin4); - - val in_prod = in_gate ∘ in_sat; - val f_prod = forget_gate ∘ in_c; - val out_c = in_prod + f_prod; - - val out_tanh = tanh(out_c); - val out_h = out_gate ∘ out_tanh; - - grid = in_h \ lin1 \ in_gate \ in_prod \ out_tanh on - in_c \ lin2 \ out_gate \ f_prod \ out_h on - in_i \ lin3 \ forget_gate \ out_c \ null on - h_over_i \ lin4 \ in_sat \ null \ null; - - val lopts = grid.data; - lopts.map((x:Node) => if (x != null) x.parent = this); - outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); - - } - - // LSTM with 1 linear layer, with h and i stacked as inputs, and all 4 output stacked - // Observed not very stable to train - // - // | Wi_in Wh_in | - // | Wi_out Wh_out | | i | - // | Wi_forget Wh_forget | | h | - // | Wi_g Wh_g | - - def constructGraph2 = { - import BIDMach.networks.layers.Node._ - val odim = dim; - val in_h = copy; - val in_c = copy; - val in_i = copy; - val h_over_i = in_h over in_i; - - val lin = linear(h_over_i)(prefix+"LSTM_all", outdim=4*odim, hasBias=hasBias); - val sp = splitvert(lin, 4); - - val in_gate = σ(sp(0)); - val out_gate = σ(sp(1)); - val forget_gate = σ(sp(2)); - val in_sat = tanh(sp(3)); - - val in_prod = in_gate ∘ in_sat; - val f_prod = forget_gate ∘ in_c; - val out_c = in_prod + f_prod; - - val out_tanh = tanh(out_c); - val out_h = out_gate ∘ out_tanh; - - grid = in_h \ lin \ in_gate \ in_prod \ out_tanh on - in_c \ sp \ out_gate \ f_prod \ out_h on - in_i \ null \ forget_gate \ out_c \ null on - h_over_i \ null \ in_sat \ null \ null; - - val lopts = grid.data; - lopts.map((x:Node) => if (x != null) x.parent = this); - outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) - } - - // LSTM with two sets of layers, paired outputs. More stable to train than the single linlayer network - // | Wi_in Wh_in | | i | + | Wi_forget Wh_forget | | i | - // | Wi_out Wh_out | | h | | Wi_g Wh_g | | h | - - def constructGraph3 = { - import BIDMach.networks.layers.Node._ - val odim = dim; - val in_h = copy; - val in_c = copy; - val in_i = copy; - val h_over_i = in_h over in_i; - - val lin1 = linear(h_over_i)(prefix+"LSTM_in_out", outdim=2*odim, hasBias=hasBias); - val sp1 = splitvert(lin1, 2); - val lin2 = linear(h_over_i)(prefix+"LSTM_forget_tanh", outdim=2*odim, hasBias=hasBias); - val sp2 = splitvert(lin2, 2); - - val in_gate = σ(sp1(0)); - val out_gate = σ(sp1(1)); - val forget_gate = σ(sp2(0)); - val in_sat = tanh(sp2(1)); - - val in_prod = in_gate ∘ in_sat; - val f_prod = forget_gate ∘ in_c; - val out_c = in_prod + f_prod; - - val out_tanh = tanh(out_c); - val out_h = out_gate ∘ out_tanh; - - grid = in_h \ lin1 \ in_gate \ in_prod \ out_tanh on - in_c \ sp1 \ out_gate \ f_prod \ out_h on - in_i \ lin2 \ forget_gate \ out_c \ null on - h_over_i \ sp2 \ in_sat \ null \ null; - - val lopts = grid.data; - lopts.map((x:Node) => if (x != null) x.parent = this); - outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) - } - - // LSTM with 2 linear layers from h and i respectively - - def constructGraph4 = { - import BIDMach.networks.layers.Node._ - val odim = dim; - val in_h = copy; - val in_c = copy; - val in_i = copy; - - val linh = linear(in_h)(prefix+"LSTM_h", outdim=4*odim, hasBias=hasBias); - val sph = splitvert(linh, 4); - val lini = linear(in_i)(prefix+"LSTM_i", outdim=4*odim, hasBias=hasBias); - val spi = splitvert(lini, 4); - - val lin1 = sph(0) + spi(0); - val lin2 = sph(1) + spi(1); - val lin3 = sph(2) + spi(2); - val lin4 = sph(3) + spi(3); - - val in_gate = σ(lin1); - val out_gate = σ(lin2); - val forget_gate = σ(lin3); - val in_sat = tanh(lin4); - - val in_prod = in_gate ∘ in_sat; - val f_prod = forget_gate ∘ in_c; - val out_c = in_prod + f_prod; - - val out_tanh = tanh(out_c); - val out_h = out_gate ∘ out_tanh; - - grid = (in_h on in_c on in_i on null) \ (linh \ lin1 \ in_gate \ in_prod \ out_tanh on - sph \ lin2 \ out_gate \ f_prod \ out_h on - lini \ lin3 \ forget_gate \ out_c \ null on - spi \ lin4 \ in_sat \ null \ null); - - val lopts = grid.data; - lopts.map((x:Node) => if (x != null) x.parent = this); - outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) - } - - override def clone:LSTMNode = { - copyTo(new LSTMNode).asInstanceOf[LSTMNode]; - } - - override def create(net:Net):LSTMLayer = { - LSTMLayer(net, this); - } - - override def toString = { - "LSTM@"+Integer.toHexString(hashCode % 0x10000).toString - } - - def h = apply(0); - - def c = apply(1); - } - - - -object LSTMNode { - - final val gridTypeNoOutput = 0; - final val gridTypeSoftmaxOutput = 1; - final val gridTypeNegsampOutput = 2; - - def apply() = { - val n = new LSTMNode; - n.constructGraph; - n - } - - def apply(opts:LSTMNodeOpts) = { - val n = new LSTMNode; - opts.copyOpts(n); - n.constructGraph; - n - } - - class GridOpts extends LSTMNodeOpts {var netType = 0; var bylevel = true}; - - def grid(nrows:Int, ncols:Int, opts:GridOpts):NodeMat = { - import BIDMach.networks.layers.Node._ - val nlin = 2; - val odim = opts.outdim; - val idim = opts.dim; - val nsoft = opts.netType match { - case `gridTypeNoOutput` => 0; - case `gridTypeNegsampOutput` => 1; - case `gridTypeSoftmaxOutput` => 2; - } - val gr = NodeMat(nrows + nlin + nsoft, ncols); - - for (k <- 0 until ncols) { - gr(0, k) = input - } - - val modelName = opts.modelName; - - for (k <- 0 until ncols) { - gr(1, k) = linear(gr(0, k))((modelName format 0) +"_bottom", outdim=idim, hasBias = opts.hasBias) - } - - for (k <- 0 until ncols) { - for (j <- nlin until nrows + nlin) { - val modelName = if (opts.bylevel) (opts.modelName format j-nlin) else (opts.modelName format 0) - val below = gr(j-1, k); - if (k > 0) { - val left = gr(j, k-1).asInstanceOf[LSTMNode] - gr(j, k) = lstm(h=left.h, c=left.c, i=below, m=modelName)(opts); - } else { - gr(j, k) = lstm(h=null, c=null, i=below, m=modelName)(opts); - } - } - } - - opts.netType match { - case `gridTypeNoOutput` => {} - case `gridTypeSoftmaxOutput` => { - for (k <- 0 until ncols) { - gr(nrows + nlin, k) = linear(gr(nrows + nlin - 1, k))(name=opts.modelName+"_top", outdim=odim, hasBias = opts.hasBias) - gr(nrows + nlin + 1, k) = softmaxout(gr(nrows + nlin, k))(opts.scoreType); - } - } - case `gridTypeNegsampOutput` => { - for (k <- 0 until ncols) { - gr(nrows + nlin, k) = negsamp(gr(nrows + nlin - 1, k))(name=opts.modelName+"_top", outdim=odim, hasBias=opts.hasBias, scoreType=opts.scoreType) - } - } - } - gr - } -} - -object LSTMLayer { - - def apply(net:Net) = new LSTMLayer(net, new LSTMNode); - - def apply(net:Net, opts:LSTMNode) = { - val x = new LSTMLayer(net, opts); - x.construct; - x; - } - - def grid(net:Net, nrows:Int, ncols:Int, opts:LSTMNode.GridOpts):LayerMat = { - val nodeGrid = LSTMNode.grid(nrows, ncols, opts); - LayerMat(nodeGrid, net); - } -} + */ + +class LSTMLayer(override val net:Net, override val opts:LSTMNode = new LSTMNode) extends CompoundLayer(net, opts) { + override val _inputs = new Array[LayerTerm](3); + override val _outputs = new Array[ND](2); + override val _derivs = new Array[ND](2); + + override def toString = { + "LSTM@"+Integer.toHexString(hashCode % 0x10000).toString + } +} + +trait LSTMNodeOpts extends CompoundNodeOpts { + var dim = 0; + var kind = 1; + var hasBias = false; + var scoreType = 0; + var outdim = 0; + + def copyOpts(opts:LSTMNodeOpts):LSTMNodeOpts = { + super.copyOpts(opts); + opts.dim = dim; + opts.kind = kind; + opts.hasBias = hasBias; + opts.scoreType = scoreType; + opts.outdim = outdim; + opts; + } +} + +class LSTMNode extends CompoundNode with LSTMNodeOpts { + + override val inputs:Array[NodeTerm] = Array(null, null, null); +// override val inputTerminals:Array[Int] = Array(0,0,0); + + def constructGraph = { + kind match { + case 0 => constructGraph0 + case 1 => constructGraph1 + case 2 => constructGraph2 + case 3 => constructGraph3 + case 4 => constructGraph4 + case _ => throw new RuntimeException("LSTMLayer type %d not recognized" format kind); + } + } + + // Basic LSTM topology with 8 linear layers + + def constructGraph0 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + + val in_h = copy; + val in_c = copy; + val in_i = copy; + + val lin1 = linear(in_h)(prefix+"LSTM_h_in_gate", outdim=odim, hasBias=hasBias); + val lin2 = linear(in_h)(prefix+"LSTM_h_out_gate", outdim=odim, hasBias=hasBias); + val lin3 = linear(in_h)(prefix+"LSTM_h_forget_gate", outdim=odim, hasBias=hasBias); + val lin4 = linear(in_h)(prefix+"LSTM_h_tanh_gate", outdim=odim, hasBias=hasBias); + + val lin5 = linear(in_i)(prefix+"LSTM_i_in_gate", outdim=odim, hasBias=hasBias); + val lin6 = linear(in_i)(prefix+"LSTM_i_out_gate", outdim=odim, hasBias=hasBias); + val lin7 = linear(in_i)(prefix+"LSTM_i_forget_gate", outdim=odim, hasBias=hasBias); + val lin8 = linear(in_i)(prefix+"LSTM_i_tanh_gate", outdim=odim, hasBias=hasBias); + + val sum1 = lin1 + lin5; + val sum2 = lin2 + lin6; + val sum3 = lin3 + lin7; + val sum4 = lin4 + lin8; + + val in_gate = σ(sum1); + val out_gate = σ(sum2); + val forget_gate = σ(sum3); + val in_sat = tanh(sum4); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = (in_h on in_c on in_i on null) \ (lin1 \ lin5 \ sum1 \ in_gate \ in_prod \ out_tanh on + lin2 \ lin6 \ sum2 \ out_gate \ f_prod \ out_h on + lin3 \ lin7 \ sum3 \ forget_gate \ out_c \ null on + lin4 \ lin8 \ sum4 \ in_sat \ null \ null); + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); + } + + // LSTM with 4 linear layers, with h and i stacked as inputs + + def constructGraph1 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + + val in_h = copy; + val in_c = copy; + val in_i = copy; + val h_over_i = in_h over in_i; + + val lin1 = linear(h_over_i)(prefix+"LSTM_in_gate", outdim=odim, hasBias=hasBias); + val lin2 = linear(h_over_i)(prefix+"LSTM_out_gate", outdim=odim, hasBias=hasBias); + val lin3 = linear(h_over_i)(prefix+"LSTM_forget_gate", outdim=odim, hasBias=hasBias); + val lin4 = linear(h_over_i)(prefix+"LSTM_tanh_gate", outdim=odim, hasBias=hasBias); + + val in_gate = σ(lin1); + val out_gate = σ(lin2); + val forget_gate = σ(lin3); + val in_sat = tanh(lin4); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = in_h \ lin1 \ in_gate \ in_prod \ out_tanh on + in_c \ lin2 \ out_gate \ f_prod \ out_h on + in_i \ lin3 \ forget_gate \ out_c \ null on + h_over_i \ lin4 \ in_sat \ null \ null; + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); + + } + + // LSTM with 1 linear layer, with h and i stacked as inputs, and all 4 output stacked + + def constructGraph2 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + val in_h = copy; + val in_c = copy; + val in_i = copy; + val h_over_i = in_h over in_i; + + val lin = linear(h_over_i)(prefix+"LSTM_all", outdim=4*odim, hasBias=hasBias); + val sp = splitvert(lin, 4); + + val in_gate = σ(sp(0)); + val out_gate = σ(sp(1)); + val forget_gate = σ(sp(2)); + val in_sat = tanh(sp(3)); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = in_h \ lin \ in_gate \ in_prod \ out_tanh on + in_c \ sp \ out_gate \ f_prod \ out_h on + in_i \ null \ forget_gate \ out_c \ null on + h_over_i \ null \ in_sat \ null \ null; + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) + } + + // LSTM with two sets of layers, paired outputs. More stable to train than the single linlayer network + + def constructGraph3 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + val in_h = copy; + val in_c = copy; + val in_i = copy; + val h_over_i = in_h over in_i; + + val lin1 = linear(h_over_i)(prefix+"LSTM_in_out", outdim=2*odim, hasBias=hasBias); + val sp1 = splitvert(lin1, 2); + val lin2 = linear(h_over_i)(prefix+"LSTM_forget_tanh", outdim=2*odim, hasBias=hasBias); + val sp2 = splitvert(lin2, 2); + + val in_gate = σ(sp1(0)); + val out_gate = σ(sp1(1)); + val forget_gate = σ(sp2(0)); + val in_sat = tanh(sp2(1)); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = in_h \ lin1 \ in_gate \ in_prod \ out_tanh on + in_c \ sp1 \ out_gate \ f_prod \ out_h on + in_i \ lin2 \ forget_gate \ out_c \ null on + h_over_i \ sp2 \ in_sat \ null \ null; + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) + } + + // LSTM with 2 linear layers from h and i respectively + + def constructGraph4 = { + import BIDMach.networks.layers.Node._ + val odim = dim; + val in_h = copy; + val in_c = copy; + val in_i = copy; + + val linh = linear(in_h)(prefix+"LSTM_h", outdim=4*odim, hasBias=hasBias); + val sph = splitvert(linh, 4); + val lini = linear(in_i)(prefix+"LSTM_i", outdim=4*odim, hasBias=hasBias); + val spi = splitvert(lini, 4); + + val lin1 = sph(0) + spi(0); + val lin2 = sph(1) + spi(1); + val lin3 = sph(2) + spi(2); + val lin4 = sph(3) + spi(3); + + val in_gate = σ(lin1); + val out_gate = σ(lin2); + val forget_gate = σ(lin3); + val in_sat = tanh(lin4); + + val in_prod = in_gate ∘ in_sat; + val f_prod = forget_gate ∘ in_c; + val out_c = in_prod + f_prod; + + val out_tanh = tanh(out_c); + val out_h = out_gate ∘ out_tanh; + + grid = (in_h on in_c on in_i on null) \ (linh \ lin1 \ in_gate \ in_prod \ out_tanh on + sph \ lin2 \ out_gate \ f_prod \ out_h on + lini \ lin3 \ forget_gate \ out_c \ null on + spi \ lin4 \ in_sat \ null \ null); + + val lopts = grid.data; + lopts.map((x:Node) => if (x != null) x.parent = this); + outputNumbers = Array(lopts.indexOf(out_h), lopts.indexOf(out_c)); // Specifies the output layer numbers (next_h and next_c) + } + + override def clone:LSTMNode = { + copyTo(new LSTMNode).asInstanceOf[LSTMNode]; + } + + override def create(net:Net):LSTMLayer = { + LSTMLayer(net, this); + } + + override def toString = { + "LSTM@"+Integer.toHexString(hashCode % 0x10000).toString + } + + def h = apply(0); + + def c = apply(1); + } + + + +object LSTMNode { + + final val gridTypeNoOutput = 0; + final val gridTypeSoftmaxOutput = 1; + final val gridTypeNegsampOutput = 2; + + def apply() = { + val n = new LSTMNode; + n.constructGraph; + n + } + + def apply(opts:LSTMNodeOpts) = { + val n = new LSTMNode; + opts.copyOpts(n); + n.constructGraph; + n + } + + class GridOpts extends LSTMNodeOpts {var netType = 0; var bylevel = true}; + + def grid(nrows:Int, ncols:Int, opts:GridOpts):NodeMat = { + import BIDMach.networks.layers.Node._ + val nlin = 2; + val odim = opts.outdim; + val idim = opts.dim; + val nsoft = opts.netType match { + case `gridTypeNoOutput` => 0; + case `gridTypeNegsampOutput` => 1; + case `gridTypeSoftmaxOutput` => 2; + } + val gr = NodeMat(nrows + nlin + nsoft, ncols); + + for (k <- 0 until ncols) { + gr(0, k) = input + } + + val modelName = opts.modelName; + + for (k <- 0 until ncols) { + gr(1, k) = linear(gr(0, k))((modelName format 0) +"_bottom", outdim=idim, hasBias = opts.hasBias) + } + + for (k <- 0 until ncols) { + for (j <- nlin until nrows + nlin) { + val modelName = if (opts.bylevel) (opts.modelName format j-nlin) else (opts.modelName format 0) + val below = gr(j-1, k); + if (k > 0) { + val left = gr(j, k-1).asInstanceOf[LSTMNode] + gr(j, k) = lstm(h=left.h, c=left.c, i=below, m=modelName)(opts); + } else { + gr(j, k) = lstm(h=null, c=null, i=below, m=modelName)(opts); + } + } + } + + opts.netType match { + case `gridTypeNoOutput` => {} + case `gridTypeSoftmaxOutput` => { + for (k <- 0 until ncols) { + gr(nrows + nlin, k) = linear(gr(nrows + nlin - 1, k))(name=opts.modelName+"_top", outdim=odim, hasBias = opts.hasBias) + gr(nrows + nlin + 1, k) = softmaxout(gr(nrows + nlin, k))(opts.scoreType); + } + } + case `gridTypeNegsampOutput` => { + for (k <- 0 until ncols) { + gr(nrows + nlin, k) = negsamp(gr(nrows + nlin - 1, k))(name=opts.modelName+"_top", outdim=odim, hasBias=opts.hasBias, scoreType=opts.scoreType) + } + } + } + gr + } +} + +object LSTMLayer { + + def apply(net:Net) = new LSTMLayer(net, new LSTMNode); + + def apply(net:Net, opts:LSTMNode) = { + val x = new LSTMLayer(net, opts); + x.construct; + x; + } + + def grid(net:Net, nrows:Int, ncols:Int, opts:LSTMNode.GridOpts):LayerMat = { + val nodeGrid = LSTMNode.grid(nrows, ncols, opts); + LayerMat(nodeGrid, net); + } +} From e783c86e2474b0e316dead114c66f7db3175e4c4 Mon Sep 17 00:00:00 2001 From: Bobby Jaros Date: Tue, 8 Mar 2016 17:10:02 -0800 Subject: [PATCH 6/6] Bad merge, fixing these two files (3rd time) --- src/main/scala/BIDMach/networks/layers/LSTM.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/scala/BIDMach/networks/layers/LSTM.scala b/src/main/scala/BIDMach/networks/layers/LSTM.scala index bc4a3d27..ed40d867 100644 --- a/src/main/scala/BIDMach/networks/layers/LSTM.scala +++ b/src/main/scala/BIDMach/networks/layers/LSTM.scala @@ -108,6 +108,9 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { } // LSTM with 4 linear layers, with h and i stacked as inputs + // + // | Wh_in Wi_in | | h | + | Wh_out Wi_out | | h | + | Wh_forget Wi_forget | | h | + | Wh_g Wi_g | | h | + // | i | | i | | i | | i | def constructGraph1 = { import BIDMach.networks.layers.Node._ @@ -147,6 +150,12 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { } // LSTM with 1 linear layer, with h and i stacked as inputs, and all 4 output stacked + // Observed not very stable to train + // + // | Wh_in Wi_in | + // | Wh_out Wi_out | | h | + // | Wh_forget Wi_forget | | i | + // | Wh_g Wi_g | def constructGraph2 = { import BIDMach.networks.layers.Node._ @@ -182,6 +191,9 @@ class LSTMNode extends CompoundNode with LSTMNodeOpts { } // LSTM with two sets of layers, paired outputs. More stable to train than the single linlayer network + // + // | Wh_in Wi_in | | h | + | Wh_forget Wi_forget | | h | + // | Wh_out Wi_out | | i | | Wh_g Wi_g | | i | def constructGraph3 = { import BIDMach.networks.layers.Node._