adding replaymaxrows for optional row count on recovery (#688)

Author: zanbuchanan

code/processes/wdb.q

@@ -552,11 +552,15 @@ replayupd:{[f;t;d]
     /- execute the supplied function
         f . (t;d);
     /- if the data count is greater than the threshold, then flush data to disk
-    if[(rpc:count[value t]) > lmt:maxrows[t];
-        .lg.o[`replayupd;"row limit (",string[lmt],") exceeded for ",string[t],". Table count is : ",string[rpc],". Flushing table to disk..."];
-        savetables[savedir;getpartition[];0b;t]]
+    replaymaxrowcheck[t;replaymaxrows[t]];
     }[upd];
 
+replaymaxrowcheck:{[t;lmt]
+    if[(rpc:count[value t]) > lmt;
+        .lg.o[`replayupd;"row limit (",string[lmt],") exceeded for ",string[t],". Table count is : ",string[rpc],". Flushing table to disk..."];
+        savetables[savedir;getpartition[];0b;t]];
+    };
+
 / - if there is data in the wdb directory for the partition remove it before replay
 / - is only for wdb processes that are saving data to disk
 clearwdbdata:{[]

code/wdb/origstartup.q

@@ -16,6 +16,11 @@ startup:{[]
        subscribe[];
        /- add missing tables to partitions in case an IDB process wants to connect. Only applicable for partbyenum writedown mode
        if[.wdb.writedownmode in `default`partbyenum;initmissingtables[currentpartition]];
+       // if for replay table maxrows were customised, we want to check row count for each table, save and gc where needed
+       if[(not .wdb.numtab~.wdb.replaynumtab)or .wdb.numrows<>.wdb.replaynumrows;
+          tabs:exec table from .sub.SUBSCRIPTIONS;
+          tabmaxrowpairs:{(x;.wdb.maxrows[x])}each tabs;
+          {replaymaxrowcheck[first x;last x]}each tabmaxrowpairs];
       ];
     @[`.; `upd; :; .wdb.upd];
  }

code/wdb/writedown.q

@@ -11,6 +11,7 @@ numtab:@[value;`numtab;`quote`trade!10000 50000];                          /-spe
 
 
 maxrows:{[tabname] numrows^numtab[tabname]};                               /- extract user defined row counts
+replaymaxrows:{[tabname] replaynumrows^replaynumtab[tabname]};
 
 partitiontype:@[value;`partitiontype;`date];                               /-set type of partition (defaults to `date)
 

config/settings/wdb.q

@@ -14,6 +14,8 @@ subsyms:`
 savedir:hsym`$getenv[`TORQHOME],"/wdbhdb"                                                   // location to save wdb data
 numrows:100000                                                                              // default number of rows
 numtab:`quote`trade!10000 50000                                                             // specify number of rows per table
+replaynumrows:numrows                                                                       // 0W                       for replaying all messages at once then flushing
+replaynumtab:numtab                                                                         // enlist[`]!enlist 0W      for replaying all messages at once then flushing
 mode:`save                                                                                  // the wdb process can operate in three modes	
                                                                                             // 1. saveandsort:     the process will subscribe for data,
                                                                                             //                     periodically write data to disk and at EOD it will flush