Introducing Word Weighting based on parent-child word relationship and recently used words [Proof of Concept]

Source/Includes/Conversions.ahk

@@ -97,10 +97,16 @@ RebuildDatabase()
 
 	CreateWordIndex()
 
+	CreateWordReplacementIndex()
+
 	CreateLastStateTable()
 
 	CreateWordlistsTable()
 
+	CreateWordRelationTable()
+
+	CreateWordRelationView()
+
 	SetDbVersion()
 	g_WordListDB.EndTransaction()
 
@@ -249,7 +255,7 @@ CreateWordsTable(WordsTableName:="Words")
 {
 	global g_WordListDB
 
-	IF not g_WordListDB.Query("CREATE TABLE " . WordsTableName . " (wordindexed TEXT NOT NULL, word TEXT NOT NULL, count INTEGER, worddescription TEXT, wordreplacement TEXT NOT NULL, PRIMARY KEY (word, wordreplacement) );")
+	IF not g_WordListDB.Query("CREATE TABLE " . WordsTableName . " (ID INTEGER PRIMARY KEY, wordindexed TEXT NOT NULL, word TEXT NOT NULL, count INTEGER, worddescription TEXT, wordreplacement TEXT NOT NULL, lastused INTEGER DEFAULT (0), UNIQUE (word, wordreplacement));")
 	{
 		ErrMsg := g_WordListDB.ErrMsg()
 		ErrCode := g_WordListDB.ErrCode()
@@ -271,6 +277,19 @@ CreateWordIndex()
 	}
 }
 
+CreateWordReplacementIndex()
+{
+	global g_WordListDB
+
+	IF not g_WordListDB.Query("CREATE INDEX WordReplacementIndexed ON Words (word, wordreplacement);")
+	{
+		ErrMsg := g_WordListDB.ErrMsg()
+		ErrCode := g_WordListDB.ErrCode()
+		msgbox Cannot Create WordReplacementIndexed Index - fatal error: %ErrCode% - %ErrMsg%
+		ExitApp
+	}
+}
+
 CreateWordlistsTable()
 {
 	global g_WordListDB
@@ -282,4 +301,30 @@ CreateWordlistsTable()
 		msgbox Cannot Create Wordlists Table - fatal error: %ErrCode% - %ErrMsg%
 		ExitApp
 	}
+}
+
+CreateWordRelationTable()
+{
+	global g_WordListDB
+
+	IF not g_WordListDB.Query("CREATE TABLE WordRelations ( word_minus1 INTEGER NOT NULL REFERENCES Words (ID) ON DELETE CASCADE, word INTEGER NOT NULL REFERENCES Words (ID) ON DELETE CASCADE, count INTEGER, lastused INTEGER DEFAULT (0), PRIMARY KEY ( word_minus1, word ) ); ")
+	{
+		ErrMsg := g_WordListDB.ErrMsg()
+		ErrCode := g_WordListDB.ErrCode()
+		msgbox Cannot Create WordRelations Table - fatal error: %ErrCode% - %ErrMsg%
+		ExitApp
+	}
+}
+
+CreateWordRelationView()
+{
+	global g_WordListDB
+
+	IF not g_WordListDB.Query("CREATE VIEW VW_WordRelations AS SELECT WordRelations.word_minus1 AS word_minus1_ID, Words_0.word AS word_minus1, WordRelations.word AS word_ID, Words.word AS word, WordRelations.count, WordRelations.lastused FROM words AS Words_0 INNER JOIN ( WordRelations INNER JOIN Words ON WordRelations.word = Words.ID ) ON Words_0.ID = WordRelations.word_minus1; ")
+	{
+		ErrMsg := g_WordListDB.ErrMsg()
+		ErrCode := g_WordListDB.ErrCode()
+		msgbox Cannot Create WordRelations View - fatal error: %ErrCode% - %ErrMsg%
+		ExitApp
+	}
 }

Source/Includes/Sending.ahk

@@ -33,6 +33,7 @@ SendWord(WordIndex)
    }
    ; Update Typed Count
    UpdateWordCount(sending,0)
+   UpdateWordHierarchy(sending)
    SendFull(sending, ForceBackspace)   
    ClearAllVars(true)
    Return
@@ -64,7 +65,7 @@ SendFull(SendValue,ForceBackspace=false)
          {
             Capitalize := true
          }
-      } else if ( RegExMatch(Substr(g_Word, 1, 1), "S)[A-Z�-��-�]") > 0 )
+      } else if ( RegExMatch(Substr(g_Word, 1, 1), "S)[A-ZÀ-ÖØ-ß]") > 0 )
       {
          Capitalize := true
       }
@@ -234,4 +235,4 @@ SendCompatible(SendValue,ForceSendForInput)
    Return
 }
 
-;------------------------------------------------------------------------
+;------------------------------------------------------------------------

Source/Includes/Wordlist.ahk

@@ -177,6 +177,7 @@ AddWordToList(AddWord,ForceCountNewOnly,ForceLearn=false, ByRef LearnedWordsCoun
    global g_WordListDB
 
    if !(LearnedWordsCount) {
+      ;This section handles the creation of new wordlist database from text files.
       StringSplit, SplitAddWord, AddWord, |
 
       IfEqual, SplitAddWord2, D
@@ -205,6 +206,7 @@ AddWordToList(AddWord,ForceCountNewOnly,ForceLearn=false, ByRef LearnedWordsCoun
 
    IfEqual, g_WordListDone, 0 ;if this is read from the wordlist
    {
+      ;If wordlist is not yet processed...
       IfNotEqual,LearnedWordsCount,  ;if this is a stored learned word, this will only have a value when LearnedWords are read in from the wordlist
       {
          ; must update wordreplacement since SQLLite3 considers nulls unique
@@ -237,13 +239,25 @@ AddWordToList(AddWord,ForceCountNewOnly,ForceLearn=false, ByRef LearnedWordsCoun
          IfNotEqual, ForceCountNewOnly, 1
          {
             IF (StrLen(AddWord) < prefs_LearnLength) ; don't add the word if it's not longer than the minimum length for learning if we aren't force learning it
+            {
+               ;Word not learned: Length less than LearnLength
+               ClearWordHierarchy() 
                Return
+            }
 
             if AddWord contains %prefs_ForceNewWordCharacters%
+            {
+               ;Word not learned: word contains character from prefs_ForceNewWordCharacters
+               ClearWordHierarchy() 
                Return
+            }
 
             if AddWord contains %prefs_DoNotLearnStrings%
+            {
+               ;Word not learned: word contains string from prefs_DoNotLearnStrings
+               ClearWordHierarchy() 
                Return
+            }
 
             CountValue = 1
 
@@ -265,12 +279,19 @@ AddWordToList(AddWord,ForceCountNewOnly,ForceLearn=false, ByRef LearnedWordsCoun
 
             IF ( CountValue < prefs_LearnCount )
             {
-               g_WordListDB.QUERY("UPDATE words SET count = ('" . prefs_LearnCount . "') WHERE word = '" . AddWordTransformed . "');")
+               ;Update the 'lastused' field as well
+               g_WordListDB.QUERY("UPDATE words SET count = ('" . prefs_LearnCount . "'), lastused = (select datetime(strftime('%s','now'), 'unixepoch')) WHERE word = '" . AddWordTransformed . "';")
             }
          } else {
             UpdateWordCount(AddWord,0) ;Increment the word count if it's already in the list and we aren't forcing it on
          }
       }
+
+      ;Since this section can only be reached by the above two conditions, this means that the word was either new and added; or it was
+      ;previous known and updated in the list.
+      ;Time to record the word hierarchy (parent-child relationship) information:
+
+      UpdateWordHierarchy(AddWordTransformed)
    }
 
    Return
@@ -306,7 +327,7 @@ CheckValid(Word,ForceLearn=false)
       {
          return
       }
-   } else if ( RegExMatch(Word, "S)[a-zA-Z�-��-��-��-�]") = 0 )
+   } else if ( RegExMatch(Word, "S)[a-zA-Zà-öø-ÿÀ-ÖØ-ß]") = 0 )
    {
       Return
    }
@@ -370,13 +391,34 @@ UpdateWordCount(word,SortOnly)
       Return
 
    StringReplace, wordEscaped, word, ', '', All
-   g_WordListDB.Query("UPDATE words SET count = count + 1 WHERE word = '" . wordEscaped . "';")
+   ;Update word count AND the lastused time for this word
+   g_WordListDB.Query("UPDATE words SET count = count + 1, lastused = (select datetime(strftime('%s','now'), 'unixepoch')) WHERE word = '" . wordEscaped . "';")
+
 
    Return
 }
 
 ;------------------------------------------------------------------------
 
+UpdateWordHierarchyCount(word)
+{
+   global prefs_LearnMode
+   global g_WordListDB
+   global g_Word_Minus1
+   ;Word = Word to increment count for
+
+   ;Should only be called when LearnMode is on  
+   IfEqual, prefs_LearnMode, Off
+      Return
+
+   StringReplace, wordEscaped, word, ', '', All
+   StringReplace, wordMinus1Escaped, g_Word_Minus1, ', '', All
+   g_WordListDB.Query("UPDATE WordRelations SET count = count + 1, lastused = (select datetime(strftime('%s','now'), 'unixepoch')) WHERE word = (select ID from words where word = '" . wordEscaped . "') and word_minus1 = (select ID from words where word = '" . wordMinus1Escaped . "');")   
+   Return
+}
+
+;------------------------------------------------------------------------
+
 CleanupWordList(LearnedWordsOnly := false)
 {
    ;Function cleans up all words that are less than the LearnCount threshold or have a NULL for count
@@ -386,10 +428,20 @@ CleanupWordList(LearnedWordsOnly := false)
    global prefs_LearnCount
    Progress, M, Please wait..., Cleaning wordlist, %g_ScriptTitle%
    if (LearnedWordsOnly) {
-      g_WordListDB.Query("DELETE FROM Words WHERE count < " . prefs_LearnCount . " AND count IS NOT NULL;")
+      g_WordListDB.Query("DELETE FROM Words WHERE count < " . prefs_LearnCount . " AND count IS NOT NULL and lastused < DATE('now','-7 day');")
    } else {
       g_WordListDB.Query("DELETE FROM Words WHERE count < " . prefs_LearnCount . " OR count IS NULL;")
    }
+
+   ;Zap all lastused values greater than a threshold (anything last used earlier than today) to reduce dataset size for lastused functionality
+   g_WordListDB.Query("UPDATE words SET lastused = 0 where lastused < DATE('now','-7 day') and lastused <> 0;")
+
+   ;Remove all word relationship that are under the threshold if older than 7 days.
+   g_WordListDB.Query("DELETE FROM WordRelations WHERE count < " . prefs_LearnCount . " AND lastused < DATE ('now','-7 day');")
+
+   ;Remove all words relationships that are not in the word list [not really required due to constraint on table]
+   g_WordListDB.Query("DELETE FROM WordRelations WHERE word NOT IN (SELECT ID FROM words) OR word_minus1 NOT IN (SELECT ID FROM words);")
+
    Progress, Off
 }
 
@@ -467,12 +519,42 @@ StrUnmark(string) {
    ; Remove combining marks and return result.
    string := RegExReplace(StrGet(&buf, len, "UTF-16"), "\pM")
 
-   StringReplace, string, string, �, ae, All
-   StringReplace, string, string, �, AE, All
-   StringReplace, string, string, �, oe, All
-   StringReplace, string, string, �, OE, All
-   StringReplace, string, string, �, ss, All   
+   StringReplace, string, string, æ, ae, All
+   StringReplace, string, string, Æ, AE, All
+   StringReplace, string, string, œ, oe, All
+   StringReplace, string, string, Œ, OE, All
+   StringReplace, string, string, ß, ss, All   
 
    return, string  
 
-}
+}
+;------------------------------------------------------------------------
+
+BulkLearnFromClipboard(textblock)
+{
+   global g_TerminatingCharactersParsed
+
+   ;Display progress bar window...
+   Progress, M, Please wait..., Bulk learning..., %g_ScriptTitle%
+
+   ;Count how many individual items there are, we need this number to display
+   ;an accurate progress bar.
+   Loop, Parse, textblock, %g_TerminatingCharactersParsed%`r`n%A_Tab%%A_Space%
+   {
+      ;Count the individual items
+      Counter++
+   }
+
+   Loop, Parse, textblock, %g_TerminatingCharactersParsed%`r`n%A_Tab%%A_Space%
+   {
+      ;Display words to show progress...
+      ProgressPercent := Round(A_Index/Counter * 100)
+      Progress, %ProgressPercent%, Please wait..., %A_LoopField%, %g_ScriptTitle%
+      AddWordToList(A_LoopField, 0,"ForceLearn")
+   }
+
+   ;Turn off progress bar window...
+   Progress, Off
+
+   return
+}