diff --git a/dal.py b/dal.py index f35ebb4..3f4cf42 100644 --- a/dal.py +++ b/dal.py @@ -60,8 +60,8 @@ def db_connect(db, host, read_default_file=DB_CONFIG_PATH): return connection def wiki_db_connect(lang): - wiki_db_name = self.lang + 'wiki_p' - wiki_db_host = self.lang + WIKI_DB_DOMAIN + wiki_db_name = lang + 'wiki_p' + wiki_db_host = lang + 'wiki.' + WIKI_DB_DOMAIN return db_connect(wiki_db_name, wiki_db_host) diff --git a/log.py b/log.py index d54b08c..d906abb 100644 --- a/log.py +++ b/log.py @@ -1,8 +1,7 @@ import os -from lithoxyl import Logger, SensibleSink, Formatter, StreamEmitter +from lithoxyl import Logger, SensibleSink, SensibleFormatter, StreamEmitter, SensibleFilter from lithoxyl.emitters import FileEmitter -from lithoxyl.filters import ThresholdFilter class FixedFileEmitter(FileEmitter): def __init__(self, filepath, encoding=None, **kwargs): @@ -11,12 +10,16 @@ def __init__(self, filepath, encoding=None, **kwargs): CUR_PATH = os.path.dirname(os.path.abspath(__file__)) LOG_FILE_PATH = os.path.join(CUR_PATH, 'logs', 'update_log.txt') +LOG_FOLDER_PATH = os.path.join(CUR_PATH, 'logs') + +if not os.path.isdir(LOG_FOLDER_PATH): + os.mkdir(LOG_FOLDER_PATH) tlog = Logger('toplog') -file_fmt = Formatter('{status_char}{end_local_iso8601_noms_notz} - {duration_secs}s - {record_name} - {message}') +file_fmt = SensibleFormatter('{status_char}{end_local_iso8601_noms_notz} - {duration_secs}s - {record_name} - {message}') file_emt = FixedFileEmitter(LOG_FILE_PATH) -file_filter = ThresholdFilter(success='critical', +file_filter = SensibleFilter(success='critical', failure='info', exception='debug') file_sink = SensibleSink(formatter=file_fmt, diff --git a/schema.sql b/schema.sql index 23cc2c2..5734b41 100644 --- a/schema.sql +++ b/schema.sql @@ -1,132 +1,107 @@ - --- begin run logs - -CREATE TABLE start_log (start_log_id integer not null auto_increment, - start_timestamp TIMESTAMP, - lang varchar(8), - command varchar(1024), - run_uuid varchar(36), - PRIMARY KEY (start_log_id), - KEY (start_timestamp), - KEY (lang)); - -CREATE TABLE complete_log (complete_log_id integer not null auto_increment, - complete_timestamp TIMESTAMP, - lang varchar(8), - output varchar(4096), - run_uuid varchar(36), - PRIMARY KEY (complete_log_id), - KEY (complete_timestamp), - KEY (lang)); - --- end run logs - -CREATE TABLE /*_*/hashtags ( - ht_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, - ht_text varbinary(767) NOT NULL default '', - ht_create_timestamp varbinary(14) NOT NULL default '', - ht_update_timestamp varbinary(14) NOT NULL default ''); - -CREATE INDEX /*i*/ht_text ON hashtags (ht_text); -CREATE INDEX /*i*/ht_create_timestamp ON hashtags (ht_create_timestamp); - -CREATE TABLE /*_*/mentions ( - mn_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, - mn_text varbinary(767) NOT NULL default '', - mn_create_timestamp varbinary(14) NOT NULL default '', - mn_update_timestamp varbinary(14) NOT NULL default ''); - -CREATE INDEX /*i*/mn_text ON mentions (mn_text); -CREATE INDEX /*i*/mn_create_timestamp ON mentions (mn_create_timestamp); - -CREATE TABLE /*_*/mention_recentchanges ( - mn_id int NOT NULL, - mnrc_id int NOT NULL); - -CREATE TABLE /*_*/hashtag_recentchanges ( - ht_id int NOT NULL, - htrc_id int NOT NULL); - -CREATE UNIQUE INDEX /*i*/htrc_hashtag_recentchanges ON /*_*/hashtag_recentchanges (ht_id, htrc_id); -CREATE INDEX /*i*/htrc_hashtag ON /*_*/hashtag_recentchanges (ht_id); - -CREATE TABLE /*_*/recentchanges ( - htrc_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, - htrc_lang varchar(32) NOT NULL default '', - rc_id int NOT NULL, - rc_timestamp varbinary(14) NOT NULL default '', - rc_cur_time varbinary(14) NOT NULL default '', - - -- As in revision - rc_user int unsigned NOT NULL default 0, - rc_user_text varchar(255) binary NOT NULL, - - -- When pages are renamed, their RC entries do _not_ change. - rc_namespace int NOT NULL default 0, - rc_title varchar(255) binary NOT NULL default '', - - -- as in revision... - rc_comment varbinary(767) NOT NULL default '', - rc_minor tinyint unsigned NOT NULL default 0, - - -- Edits by user accounts with the 'bot' rights key are - -- marked with a 1 here, and will be hidden from the - -- default view. - rc_bot tinyint unsigned NOT NULL default 0, - - -- Set if this change corresponds to a page creation - rc_new tinyint unsigned NOT NULL default 0, - - -- Key to page_id (was cur_id prior to 1.5). - -- This will keep links working after moves while - -- retaining the at-the-time name in the changes list. - rc_cur_id int unsigned NOT NULL default 0, - - -- rev_id of the given revision - rc_this_oldid int unsigned NOT NULL default 0, - - -- rev_id of the prior revision, for generating diff links. - rc_last_oldid int unsigned NOT NULL default 0, - - -- The type of change entry (RC_EDIT,RC_NEW,RC_LOG,RC_EXTERNAL) - rc_type tinyint unsigned NOT NULL default 0, - - -- The source of the change entry (replaces rc_type) - -- default of '' is temporary, needed for initial migration - rc_source varchar(16) binary not null default '', - - -- If the Recent Changes Patrol option is enabled, - -- users may mark edits as having been reviewed to - -- remove a warning flag on the RC list. - -- A value of 1 indicates the page has been reviewed. - rc_patrolled tinyint unsigned NOT NULL default 0, - - -- Recorded IP address the edit was made from, if the - -- $wgPutIPinRC option is enabled. - rc_ip varbinary(40), - - -- Text length in characters before - -- and after the edit - rc_old_len int, - rc_new_len int, - - -- Visibility of recent changes items, bitfield - rc_deleted tinyint unsigned NOT NULL default 0, - - -- Value corresponding to log_id, specific log entries - rc_logid int unsigned NOT NULL default 0, - -- Store log type info here, or null - rc_log_type varbinary(255) NULL default NULL, - -- Store log action or null - rc_log_action varbinary(255) NULL default NULL, - -- Log params - rc_params blob NULL -); - -CREATE INDEX /*i*/rc_timestamp ON /*_*/recentchanges (rc_timestamp); -CREATE INDEX /*i*/rc_lang_namespace_title ON /*_*/recentchanges (htrc_lang, rc_namespace, rc_title); -CREATE INDEX /*i*/rc_lang_cur_id ON /*_*/recentchanges (htrc_lang, rc_cur_id); -CREATE INDEX /*i*/lang_new_name_timestamp ON /*_*/recentchanges (htrc_lang, rc_new,rc_namespace,rc_timestamp); -CREATE INDEX /*i*/rc_ip ON /*_*/recentchanges (rc_ip); -CREATE INDEX /*i*/rc_lang_ns_usertext ON /*_*/recentchanges (htrc_lang, rc_namespace, rc_user_text); -CREATE INDEX /*i*/rc_lang_user_text ON /*_*/recentchanges (htrc_lang, rc_user_text, rc_timestamp); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `complete_log` ( + `complete_log_id` int(11) NOT NULL AUTO_INCREMENT, + `complete_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `lang` varbinary(8) DEFAULT NULL, + `output` varbinary(4096) DEFAULT NULL, + `run_uuid` varbinary(36) DEFAULT NULL, + PRIMARY KEY (`complete_log_id`), + KEY `complete_timestamp` (`complete_timestamp`), + KEY `lang` (`lang`) +) ENGINE=Aria AUTO_INCREMENT=1514514 DEFAULT CHARSET=binary PAGE_CHECKSUM=1; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `hashtag_recentchanges` ( + `ht_id` int(11) NOT NULL, + `htrc_id` int(11) NOT NULL, + UNIQUE KEY `htrc_hashtag_recentchanges` (`ht_id`,`htrc_id`), + KEY `htrc_hashtag` (`ht_id`) +) ENGINE=Aria DEFAULT CHARSET=binary PAGE_CHECKSUM=1; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `hashtags` ( + `ht_id` int(11) NOT NULL AUTO_INCREMENT, + `ht_text` varbinary(767) NOT NULL DEFAULT '', + `ht_create_timestamp` varbinary(14) NOT NULL DEFAULT '', + `ht_update_timestamp` varbinary(14) NOT NULL DEFAULT '', + PRIMARY KEY (`ht_id`), + KEY `ht_text` (`ht_text`), + KEY `ht_create_timestamp` (`ht_create_timestamp`) +) ENGINE=Aria AUTO_INCREMENT=36146 DEFAULT CHARSET=binary PAGE_CHECKSUM=1; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `mention_recentchanges` ( + `mn_id` int(11) NOT NULL, + `mnrc_id` int(11) NOT NULL +) ENGINE=Aria DEFAULT CHARSET=binary PAGE_CHECKSUM=1; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `mentions` ( + `mn_id` int(11) NOT NULL AUTO_INCREMENT, + `mn_text` varbinary(767) NOT NULL DEFAULT '', + `mn_create_timestamp` varbinary(14) NOT NULL DEFAULT '', + `mn_update_timestamp` varbinary(14) NOT NULL DEFAULT '', + PRIMARY KEY (`mn_id`), + KEY `mn_text` (`mn_text`), + KEY `mn_create_timestamp` (`mn_create_timestamp`) +) ENGINE=Aria AUTO_INCREMENT=129 DEFAULT CHARSET=binary PAGE_CHECKSUM=1; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `recentchanges` ( + `htrc_id` int(11) NOT NULL AUTO_INCREMENT, + `htrc_lang` varbinary(32) NOT NULL DEFAULT '', + `rc_id` int(11) NOT NULL, + `rc_timestamp` varbinary(14) NOT NULL DEFAULT '', + `rc_cur_time` varbinary(14) NOT NULL DEFAULT '', + `rc_user` int(10) unsigned NOT NULL DEFAULT '0', + `rc_user_text` varbinary(255) NOT NULL, + `rc_namespace` int(11) NOT NULL DEFAULT '0', + `rc_title` varbinary(255) NOT NULL DEFAULT '', + `rc_comment` varbinary(767) NOT NULL DEFAULT '', + `rc_minor` tinyint(3) unsigned NOT NULL DEFAULT '0', + `rc_bot` tinyint(3) unsigned NOT NULL DEFAULT '0', + `rc_new` tinyint(3) unsigned NOT NULL DEFAULT '0', + `rc_cur_id` int(10) unsigned NOT NULL DEFAULT '0', + `rc_this_oldid` int(10) unsigned NOT NULL DEFAULT '0', + `rc_last_oldid` int(10) unsigned NOT NULL DEFAULT '0', + `rc_type` tinyint(3) unsigned NOT NULL DEFAULT '0', + `rc_source` varbinary(16) NOT NULL DEFAULT '', + `rc_patrolled` tinyint(3) unsigned NOT NULL DEFAULT '0', + `rc_ip` varbinary(40) DEFAULT NULL, + `rc_old_len` int(11) DEFAULT NULL, + `rc_new_len` int(11) DEFAULT NULL, + `rc_deleted` tinyint(3) unsigned NOT NULL DEFAULT '0', + `rc_logid` int(10) unsigned NOT NULL DEFAULT '0', + `rc_log_type` varbinary(255) DEFAULT NULL, + `rc_log_action` varbinary(255) DEFAULT NULL, + `rc_params` blob, + PRIMARY KEY (`htrc_id`), + KEY `rc_timestamp` (`rc_timestamp`), + KEY `rc_lang_namespace_title` (`htrc_lang`,`rc_namespace`,`rc_title`), + KEY `rc_lang_cur_id` (`htrc_lang`,`rc_cur_id`), + KEY `lang_new_name_timestamp` (`htrc_lang`,`rc_new`,`rc_namespace`,`rc_timestamp`), + KEY `rc_ip` (`rc_ip`), + KEY `rc_lang_ns_usertext` (`htrc_lang`,`rc_namespace`,`rc_user_text`), + KEY `rc_lang_user_text` (`htrc_lang`,`rc_user_text`,`rc_timestamp`), + KEY `rc_id_htrc_lang` (`rc_id`,`htrc_lang`) COMMENT 'speed up add_recentchange 20160305' +) ENGINE=Aria AUTO_INCREMENT=5566734 DEFAULT CHARSET=binary PAGE_CHECKSUM=1; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `start_log` ( + `start_log_id` int(11) NOT NULL AUTO_INCREMENT, + `start_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `lang` varbinary(8) DEFAULT NULL, + `command` varbinary(1024) DEFAULT NULL, + `run_uuid` varbinary(36) DEFAULT NULL, + PRIMARY KEY (`start_log_id`), + KEY `start_timestamp` (`start_timestamp`), + KEY `lang` (`lang`) +) ENGINE=Aria AUTO_INCREMENT=1529298 DEFAULT CHARSET=binary PAGE_CHECKSUM=1; +/*!40101 SET character_set_client = @saved_cs_client */;