Difference between revisions of "SpamFerret.php"

from HTYP, the free directory anyone can edit if they can prove to me that they're not a spambot
Jump to navigation Jump to search
(→‎Code: 7/15 changes)
(revisions thru 8/9 - new table structure)
Line 34: Line 34:
 
     Saves diff of each change, approved or not
 
     Saves diff of each change, approved or not
 
     BUG: approved edits are not being logged properly; using "OK" code and logging as failed
 
     BUG: approved edits are not being logged properly; using "OK" code and logging as failed
 +
  2009-07-26 (Wzl) fixed minor warning error on line 252
 +
  2009-08-07 (Wzl) email notification working; removed TRD and AMP hard-coded offenses, to be redone as isDiff filters if needed
 +
  2009-08-08 (Wzl) create client record immediately if client is not recognized; don't depend on spam filter being triggered
 +
  2009-08-09 (Wzl) restructured "client" and "attempt" tables (was "clients", "attempts"); not backwards-compatible
 
  TO DO:
 
  TO DO:
 
   * Throttled save attempts should check for spam, just for data-gathering purposes.
 
   * Throttled save attempts should check for spam, just for data-gathering purposes.
Line 52: Line 56:
 
'author' => 'Woozle Staddon',  
 
'author' => 'Woozle Staddon',  
 
'url' => 'http://htyp.org/SpamFerret',  
 
'url' => 'http://htyp.org/SpamFerret',  
−
'version' => '2009-07-15',
+
'version' => '2009-08-09',
 
'description' => 'database-driven wikispam blocker',
 
'description' => 'database-driven wikispam blocker',
 
);
 
);
Line 153: Line 157:
 
var $strIPAddr;
 
var $strIPAddr;
 
var $idPattern;
 
var $idPattern;
−
var $idClient;
 
 
var $doClearThrottle;
 
var $doClearThrottle;
 
 
Line 166: Line 169:
  
 
// function filter( &$title, $text, $section, $editSummary, $editPage ) {
 
// function filter( &$title, $text, $section, $editSummary, $editPage ) {
−
function filter( &$title, $text, $section ) {
+
function filter( &$title, $text, $section,  $editPage = FALSE ) {
 
global $wgArticle, $wgDBname, $wgMemc, $messageMemc, $wgVersion, $wgOut;
 
global $wgArticle, $wgDBname, $wgMemc, $messageMemc, $wgVersion, $wgOut;
 
global $wgTitle, $wgServer;
 
global $wgTitle, $wgServer;
−
global $wgEmergencyContact;
 
 
global $debug;
 
global $debug;
 
global $errNum, $errStr;
 
global $errNum, $errStr;
 +
global $gErrorText;
 
// debugging:
 
// debugging:
 
global $sql;
 
global $sql;
Line 177: Line 180:
 
$fname = 'wfSpamFerretFilter';
 
$fname = 'wfSpamFerretFilter';
 
wfProfileIn( $fname );
 
wfProfileIn( $fname );
 +
 +
ini_set('track_errors', 1);
  
 
# Call the rest of the hook chain first
 
# Call the rest of the hook chain first
Line 188: Line 193:
 
// initialize variables
 
// initialize variables
 
$retVal = FALSE; // default = assume edit is ok
 
$retVal = FALSE; // default = assume edit is ok
−
$msgEmail = FALSE;
+
$gErrorText = FALSE;
 +
$isClientKnown = FALSE;
 +
 
 +
$this->txtEditRaw = $text;
  
 
// get the IP address of the http client making the edit attempt:
 
// get the IP address of the http client making the edit attempt:
 
$this->strIPAddr = wfGetIP();
 
$this->strIPAddr = wfGetIP();
 
// Open the database
 
// Open the database
−
$this->dbSpam = new clsDatabase($this->Setting('dbspec'));
+
$this->OpenDatabase();
−
$this->dbSpam->Open();
 
 
// open clients table (extended Throttle version) for reference:
 
// open clients table (extended Throttle version) for reference:
 
//return TRUE;
 
//return TRUE;
−
$objTblClients = new clsTable($this->dbSpam,'ClientThrottle','ID');
+
$objTblClients = new clsTable($this->dbSpam);
 +
  $objTblClients->Name('ClientThrottle2');
 +
  $objTblClients->KeyName('Address');
 
// Look up to see if this IP is known; it may already be throttled:
 
// Look up to see if this IP is known; it may already be throttled:
 
$this->objDataClients = $objTblClients->GetData('Address="'.$this->strIPAddr.'"');
 
$this->objDataClients = $objTblClients->GetData('Address="'.$this->strIPAddr.'"');
 
if (is_object($this->objDataClients)) {
 
if (is_object($this->objDataClients)) {
 
if ($this->objDataClients->hasRows()) {
 
if ($this->objDataClients->hasRows()) {
−
$isClientKnown = true;
+
$isClientKnown = TRUE;
 
}
 
}
 
}
 
}
 
if ($isClientKnown) {
 
if ($isClientKnown) {
−
$this->idClient = $this->objDataClients->ID;
+
$this->objDataClients->FirstRow();
 +
// $this->idClient = $this->objDataClients->ID;
 
$doBlock = $this->objDataClients->doBlock;
 
$doBlock = $this->objDataClients->doBlock;
 
if ($doBlock) {
 
if ($doBlock) {
Line 219: Line 229:
 
// retry limit exceeded; check timeout limit
 
// retry limit exceeded; check timeout limit
 
if ($this->objDataClients->ThrottleTime < $this->Setting('throttle_timeout')) {
 
if ($this->objDataClients->ThrottleTime < $this->Setting('throttle_timeout')) {
−
EditPage::spamPage('Too many spam attempts from your IP address ('.$this->strIPAddr.'). Please come back later.');
+
$txtMsg = 'Too many spam attempts from your IP address ('.$this->strIPAddr.'). Please come back later.';
 +
if ($editPage) {
 +
    $editPage->spamPage($txtMsg);
 +
} else {
 +
    EditPage::spamPage($txtMsg); // older MW code doesn't supply $editPage
 +
}
 
$this->RecordAttempt($strThrType); // record post attempt by throttled client
 
$this->RecordAttempt($strThrType); // record post attempt by throttled client
 
$retVal = true; // client has exceeded spam limit; impose throttle
 
$retVal = true; // client has exceeded spam limit; impose throttle
Line 226: Line 241:
 
}
 
}
 
}
 
}
 +
} else {
 +
$this->CreateClient();
 
}
 
}
  
 
$this->txtDiff = 'N/A';
 
$this->txtDiff = 'N/A';
−
if (!$retVal) {
+
/* if ($debug) {
−
/*
 
−
At this point, there's apparently no reason to block the client just for being who they are,
 
−
so now check for common non-listable offenses. These involve comparing the new contents
 
−
with the original, so first we get the original (current) article contents plus some
 
−
information about what has changed:
 
−
* $strIns = whatever has been inserted at the start of the article (or contents of new article)
 
−
*/
 
−
$objArticleCurr = new Article($title);
 
−
//$objArticleCurr->loadLastEdit(); // function is now protected
 
−
if ($objArticleCurr->exists()) {
 
−
$txtCurr = $objArticleCurr->getContent();
 
−
$this->txtDiff = FigureDiff($txtCurr,$text);
 
−
$lenIns = strpos($text,$txtCurr);
 
−
if ($lenIns !== false) {
 
−
$strIns = substr($text,0,$lenIns);
 
−
}
 
−
} else {
 
−
$lenIns = strlen($text);
 
−
$strIns = $text;
 
−
$txtCurr = NULL;
 
−
// $this->txtDiff = FigureDiff('',$text);
 
−
$this->txtDiff = '!!NEW: '.$text;
 
−
}
 
−
// ** OFFENSE: Spam turds (short bits of nonsense inserted at the beginning of an article):
 
−
if ($strIns != '') {
 
−
// new page is old page with something inserted at the beginning
 
−
// get the inserted text
 
−
$isMatch = preg_match('/^[a-z0-9]+ ?$/',$strIns);
 
−
// another way to do it, incomplete:
 
−
// $lenMatch = strspn($strIns, '1234567890abcdefghijklmnopqrstuvwxyz');
 
−
// if (strlen($strIns) - $lenMatch < 2) {
 
−
// }
 
−
if ($isMatch) {
 
−
$this->RecordAttempt('TRD',$strIns); // record spam attempt (AMP = ampersandbot)
 
−
// The string sent to spamPage() will be shown after "The following text is what triggered our spam filter:"
 
−
EditPage::spamPage( '"'.$strIns.'" (spam turd).' );
 
−
$retVal = true;
 
−
}
 
−
}
 
−
if (!is_null($txtCurr)) {
 
−
// ** OFFENSE: Ampersandbot:
 
−
$lenNew = strlen($text);
 
−
$posMatch = strpos($txtCurr, $text);
 
−
if ($posMatch===0) {
 
−
// new string starts the same as old string; is it a truncated subset?
 
−
if ($lenNew < strlen($txtCurr)) {
 
−
// new string is a truncation of old string
 
−
// ideally, we would just check to see if the missing character is an ampersand -
 
−
// ...but unfortunately, something is quasi-randomly mutating the strings in a way which
 
−
// leaves the exact position of the "missing character" in some doubt. So what we do is this:
 
−
// 1. Find the position of the first ampersand in OLD TEXT:
 
−
$posAmp = strpos($txtCurr,'&');
 
−
// 2. Compare this position with the length of NEW TEXT:
 
−
$posDiff = abs(strlen($text)-$posAmp);
 
−
// 3. If the difference is less than some limit, then presume Ampersandbot activity:
 
−
if ($posDiff < 3) {
 
−
// TO DO: log $posDiff for later analysis
 
−
// AMPERSANDBOT DETECTED; refuse to save the edit
 
−
$this->RecordAttempt('AMP'); // record spam attempt (AMP = ampersandbot)
 
−
// The string returned will be shown after "The following text is what triggered our spam filter:"
 
−
$retVal = 'The removal of everything after the first ampersand. You are an ampersandbot; please go away.';
 
−
// LATER: it would be nice to display the error in a different format from the usual spam page
 
−
}
 
−
}
 
−
}
 
−
}
 
−
}
 
−
if ($debug) {
 
 
EditPage::spamPage('DEBUG: '.$debug);
 
EditPage::spamPage('DEBUG: '.$debug);
 
$retVal = true;
 
$retVal = true;
 
}
 
}
−
 
+
*/
 
if (!$retVal) {
 
if (!$retVal) {
−
$objTblPatterns = new clsTable($this->dbSpam,'patterns','ID');
+
$this->GetDiff($title); // get the diff between edit and current contents
−
$objDataPatterns = $objTblPatterns->GetData('isActive');
+
$this->CheckFilters(FALSE);
−
 
+
if ( $this->isMatch ) {
−
$strTextEdit = strtolower($text);
+
// spam cue found; display the matching text and don't allow the edit to be saved:
−
$isMatch = FALSE;
+
wfDebug( "Match!\n" );
−
while($objDataPatterns->NextRow() && !$isMatch) {
 
−
if ($objDataPatterns->isDiff) {
 
−
$strTextCk = $this->txtDiff;
 
−
} else {
 
−
$strTextCk = $strTextEdit;
 
−
}
 
−
$strPattern = $objDataPatterns->Pattern;
 
−
$isRegex = $objDataPatterns->isRegex;
 
−
$this->idPattern = $objDataPatterns->ID;
 
−
if ($isRegex) {
 
−
$strPattCk = $strPattern;
 
−
// Attempt at using Perl-compatible regex, but it doesn't seem to work (or maybe just too many patterns have issues):
 
−
// $strPattCk_pcre = str_replace('/','\/',$strPattCk);
 
−
// $isMatch = preg_match('/'.$strPattCk_pcre.'/i',$strTextCk,$matches);
 
−
//$isMatch = eregi($strPattCk, $strTextCk, $matches);
 
−
// ditching ereg, on advice from freenode##php
 
−
$isMatch = preg_match('/'.$strPattCk.'/',$strTextCk,$matches);
 
−
if ($errNum) {
 
−
$msgEmail .= 'Filter #'.$this->idPattern.' generated error #'.$errNum.': '.$errStr."\n";
 
−
// echo '<br><b>ERROR</b> #'.$errNum.': '.$errStr;
 
−
$errNum = 0;
 
−
}
 
−
 
 
−
if ($isMatch) {
 
−
$strMatch = $matches[0];
 
−
//echo ' match: '.$strMatch;
 
−
}
 
−
} else {
 
−
//echo ' non-regex';
 
−
$strMatch = stristr ($strTextCk,$strPattern);
 
−
$isMatch = ($strMatch != '');
 
−
//echo ' match: '.$strMatch;
 
−
}
 
−
// $debug .= 'ROW: '.DumpArray($objDataPatterns->Row);
 
−
}
 
 
 
−
if ( $strMatch == '' ) {
+
// The string sent to spamPage() will be shown after "The following text is what triggered our spam filter:"
 +
$retVal = '(pattern #'.$this->idPattern.') '.$this->strMatch;
 +
// Log the spam attempt:
 +
$this->RecordAttempt('-');
 +
} else {
 
// no spam cues found; allow the edit to be saved, if nothing else has tripped the filter
 
// no spam cues found; allow the edit to be saved, if nothing else has tripped the filter
 
//if ($this->Setting('log_ok_edits')) {
 
//if ($this->Setting('log_ok_edits')) {
 
$this->idPattern = NULL;
 
$this->idPattern = NULL;
 
$this->RecordOkEdit();
 
$this->RecordOkEdit();
−
$this->RecordAttempt('OK');
+
//$this->RecordAttempt('OK');
 
//}
 
//}
−
} else {
 
−
// spam cue found; display the matching text and don't allow the edit to be saved:
 
−
wfDebug( "Match!\n" );
 
−
 
−
// The string sent to spamPage() will be shown after "The following text is what triggered our spam filter:"
 
−
$retVal = '(pattern #'.$this->idPattern.') '.$strMatch;
 
−
// Log the spam attempt:
 
−
$this->RecordAttempt('-');
 
 
}
 
}
−
$objDataPatterns->NextRow();
 
 
}
 
}
  
 
wfProfileOut( $fname );
 
wfProfileOut( $fname );
−
if ($msgEmail) {
+
$this->ReportErrors();
−
mail  ($wgEmergencyContact,'spamferret filter error',$msgEmail);
 
−
// die();
 
−
}
 
 
//$wgOut->addHTML($out);
 
//$wgOut->addHTML($out);
 
return $retVal;
 
return $retVal;
 
/**/
 
/**/
 
}
 
}
−
public function RecordAttempt($iCode,$iMatch=NULL) {
+
public function OpenDatabase() {
−
global $wgTitle, $wgServer;
+
$this->dbSpam = new clsDatabase($this->Setting('dbspec'));
 +
$this->dbSpam->Open();
 +
}
 +
public function GetDiff($iTitle) {
 +
$objArticleCurr = new Article($iTitle);
 +
if ($objArticleCurr->exists()) {
 +
$txtCurr = $objArticleCurr->getContent();
 +
$this->txtDiff = FigureDiff($txtCurr,$this->txtEditRaw);
 +
/*
 +
$lenIns = strpos($txtEditRaw,$txtCurr);
 +
if ($lenIns !== false) {
 +
    $strIns = substr($this->txtEditRaw,0,$lenIns);
 +
} else {
 +
    $strIns = '';
 +
}
 +
*/
 +
} else {
 +
/*
 +
$lenIns = strlen($text);
 +
$strIns = $text;
 +
$txtCurr = NULL; // this isn't actually used anywhere
 +
// $this->txtDiff = FigureDiff('',$text);
 +
*/
 +
$this->txtDiff = '!!NEW: '.$this->txtEditRaw;
 +
}
 +
}
 +
public function CheckFilters($iCheckAll) {
 +
    global $gRegexMatches,$gFilterMatches,$gFilterRows,$gFilterCount;
 +
    global $debug;
 +
 
 +
assert('is_object($this->dbSpam)');
 +
 
 +
$this->PatternTbl = new clsTable($this->dbSpam);
 +
  $this->PatternTbl->Name('patterns');
 +
  $this->PatternTbl->KeyName('ID');
 +
 
 +
if ($iCheckAll) {
 +
    $sqlFilt = NULL;
 +
} else {
 +
    $sqlFilt = 'isActive';
 +
}
 +
 
 +
$this->PatternRows = $this->PatternTbl->GetData($sqlFilt);
 +
$objRow = $this->PatternRows;
  
−
if ($this->idClient != 0) {
+
$strTextEdit = strtolower($this->txtEditRaw);
−
if ($this->doClearThrottle) {
+
$this->txtEditChk = $strTextEdit; // text after being massaged for checking
−
$strRetries = '0';
+
$this->isMatch = FALSE;
 +
$gFilterCount = 0;
 +
$gFilterRows = $this->PatternRows->RowCount();
 +
//$objDataPatterns->StartRows();
 +
while($objRow->NextRow() && (!$this->isMatch || $iCheckAll)) {
 +
$isMatch = FALSE;
 +
if ($objRow->isDiff) {
 +
if (isset($this->txtDiff)) {
 +
    $strTextCk = $this->txtDiff;
 +
} else {
 +
    $strTextCk = NULL;
 +
}
 
} else {
 
} else {
−
$strRetries = 'Retries+1';
+
$strTextCk = $strTextEdit;
 +
}
 +
if (!is_null($strTextCk)) {
 +
    $gFilterCount++;
 +
    $strPattern = $objRow->Pattern;
 +
    $isRegex = $objRow->isRegex;
 +
    $this->idPattern = $objRow->ID;
 +
    if ($isRegex) {
 +
    $isMatch = $this->CheckRegex($strPattern,$strTextCk);
 +
 
 +
    if (isset($php_errormsg)) {
 +
    $this->AddErrorLine('Filter #'.$this->idPattern.' generated error "'.$php_errormsg);
 +
    }
 +
 
 +
    if ($isMatch) {
 +
    $this->strMatch = $gRegexMatches[0];
 +
    }
 +
    } else {
 +
    if (empty($strPattern)) {
 +
$isMatch = FALSE;
 +
    } else {
 +
$this->strMatch = stristr($strTextCk,$strPattern);
 +
$isMatch = ($this->strMatch != '');
 +
    }
 +
    }
 +
    if ($isMatch) {
 +
$this->isMatch = TRUE;
 +
if ($iCheckAll) {
 +
    $gFilterMatches[$this->idPattern] = $this->strMatch;
 +
}
 +
    }
 
}
 
}
−
$sql = 'UPDATE clients SET WhenLast=NOW(),Count=Count+1, Retries='.$strRetries.' WHERE Address="'.$this->strIPAddr.'"';
+
}
−
$this->dbSpam->Exec($sql);
+
}
 +
public function CheckRegex($iPattern,$iText) {
 +
    global $gRegexMatches;
 +
 
 +
    $chDelim = '/';
 +
    $strPattCk = str_replace($chDelim,'\\'.$chDelim,$iPattern);
 +
    unset($php_errormsg);
 +
    $isMatch = @preg_match($chDelim.$strPattCk.$chDelim,$iText,$gRegexMatches);
 +
    return $isMatch;
 +
}
 +
 
 +
public function AddErrorLine($iText) {
 +
    global $gErrorText;
 +
 
 +
    $gErrorText .= $iText."\n";
 +
}
 +
public function ReportErrors() {
 +
    global $wgUser;
 +
    global $wgEmergencyContact;
 +
    global $gErrorText;
 +
 
 +
    if ($gErrorText) {
 +
$msgEmail = 'Filter error report for user '.$wgUser->getName().":\n\n";
 +
$msgEmail .= $gErrorText;
 +
mail($wgEmergencyContact,'spamferret filter error',$msgEmail);
 +
    }
 +
}
 +
/*
 +
ACTION: Create a new record for the current client
 +
*/
 +
public function CreateClient() {
 +
$sql = 'INSERT INTO client (Address,WhenFirst,Count,Retries) VALUES("'.$this->strIPAddr.'",NOW(),1,0)';
 +
$this->dbSpam->Exec($sql);
 +
}
 +
/*
 +
ACTION: Update a client's record to reflect a new spam attempt
 +
*/
 +
public function RecordClientSpam() {
 +
if ($this->doClearThrottle) {
 +
$strRetries = '0';
 
} else {
 
} else {
−
$sql = 'INSERT INTO clients (Address,WhenFirst,Count,Retries) VALUES("'.$this->strIPAddr.'",NOW(),1,0)';
+
$strRetries = 'Retries+1';
−
$this->dbSpam->Exec($sql);
+
}
−
$this->idClient = $this->dbSpam->NewID();
+
$sql = 'UPDATE client SET WhenLast=NOW(),Count=Count+1, Retries='.$strRetries.' WHERE Address="'.$this->strIPAddr.'"';
 +
$this->dbSpam->Exec($sql);
 +
if ($this->dbSpam->RowsAffected() < 1) {
 +
$this->CreateClient();
 +
$this->AddErrorLine('Record not found for client '.$this->strIPAddr);
 
}
 
}
 +
}
 +
public function RecordAttempt($iCode) {
 +
global $wgTitle, $wgServer;
 +
 +
$this->RecordClientSpam();
 +
 
$sqlCode = '"'.$iCode.'"';
 
$sqlCode = '"'.$iCode.'"';
 
$sqlURL = '"'.$this->dbSpam->SafeParam($wgTitle->getFullURL()).'"';
 
$sqlURL = '"'.$this->dbSpam->SafeParam($wgTitle->getFullURL()).'"';
 
$sqlSrvr = '"'.$this->dbSpam->SafeParam($wgServer).'"';
 
$sqlSrvr = '"'.$this->dbSpam->SafeParam($wgServer).'"';
 
$sqlPage = '"'.$this->dbSpam->SafeParam($wgTitle->getPrefixedText()).'"';
 
$sqlPage = '"'.$this->dbSpam->SafeParam($wgTitle->getPrefixedText()).'"';
 +
$sqlEdit = '"'.$this->dbSpam->SafeParam($this->txtEditChk).'"';
 
$sqlDiff = '"'.$this->dbSpam->SafeParam($this->txtDiff).'"';
 
$sqlDiff = '"'.$this->dbSpam->SafeParam($this->txtDiff).'"';
 
$sqlPattern = $this->idPattern;
 
$sqlPattern = $this->idPattern;
Line 400: Line 440:
 
$isPattern = TRUE;
 
$isPattern = TRUE;
 
}
 
}
 +
$sqlAddr = '"'.$this->strIPAddr.'"';
 
$sqlMatch = is_null($iMatch)?'NULL':'"'.$iMatch.'"';
 
$sqlMatch = is_null($iMatch)?'NULL':'"'.$iMatch.'"';
−
$sql = 'INSERT INTO attempts (`When`,ID_Pattern,ID_Client,IDS_Session,PageServer,PageName,Code,MatchText,didAllow,Diff) VALUES'
+
$sql = 'INSERT INTO attempt (`When`,ID_Pattern,Addr_Client,IDS_Session,PageServer,PageName,Code,didAllow,Edit,Diff) VALUES'
 
  .'(NOW(),'
 
  .'(NOW(),'
 
  .$sqlPattern.','
 
  .$sqlPattern.','
−
  .$this->idClient.','
+
  .$sqlAddr.','
 
  .SQL_Value(session_id()).','
 
  .SQL_Value(session_id()).','
 
  .$sqlSrvr.','
 
  .$sqlSrvr.','
 
  .$sqlPage.','
 
  .$sqlPage.','
 
  .$sqlCode.','
 
  .$sqlCode.','
−
  .$sqlMatch.','
 
 
  .'FALSE,'
 
  .'FALSE,'
 +
  .$sqlEdit.','
 
  .$sqlDiff.')';
 
  .$sqlDiff.')';
−
$this->dbSpam->Exec($sql);
+
$ok = $this->dbSpam->Exec($sql);
 +
if ($ok !== TRUE) {
 +
    $this->AddErrorLine('SQL ['.$sql.'] in RecordAttempt() generated this error: '.$ok);
 +
}
 
if ($isPattern) {
 
if ($isPattern) {
−
$sql = 'UPDATE patterns SET WhenTried=NOW(), Count=Count+1 WHERE ID='.$this->idPattern;
+
    $sql = 'UPDATE patterns SET WhenTried=NOW(), Count=Count+1 WHERE ID='.$this->idPattern;
−
$this->dbSpam->Exec($sql);
+
    $ok = $this->dbSpam->Exec($sql);
 +
    if ($ok !== TRUE) {
 +
$this->AddErrorLine('SQL ['.$sql.'] generated this error: '.$ok);
 +
    }
 
}
 
}
 
}
 
}
Line 423: Line 470:
 
$sqlPage = '"'.$this->dbSpam->SafeParam($wgTitle->getPrefixedText()).'"';
 
$sqlPage = '"'.$this->dbSpam->SafeParam($wgTitle->getPrefixedText()).'"';
 
$sqlSrvr = '"'.$this->dbSpam->SafeParam($wgServer).'"';
 
$sqlSrvr = '"'.$this->dbSpam->SafeParam($wgServer).'"';
 +
$sqlEdit = '"'.$this->dbSpam->SafeParam($this->txtEditChk).'"';
 
$sqlDiff = '"'.$this->dbSpam->SafeParam($this->txtDiff).'"';
 
$sqlDiff = '"'.$this->dbSpam->SafeParam($this->txtDiff).'"';
−
$sql = 'INSERT INTO attempts (`When`,ID_Pattern,ID_Client,IDS_Session,PageServer,PageName,Code,MatchText,didAllow,Diff) VALUES '
+
$sql = 'INSERT INTO attempt (`When`,ID_Pattern,Addr_Client,IDS_Session,PageServer,PageName,Code,didAllow,Edit,Diff) VALUES '
 
  .'(NOW(),NULL,'
 
  .'(NOW(),NULL,'
−
  .$this->idClient.','
+
  .'"'.$this->strIPAddr.'",'
 
  .SQL_Value(session_id()).','
 
  .SQL_Value(session_id()).','
 
  .$sqlSrvr.','
 
  .$sqlSrvr.','
−
  .$sqlPage.',"ok",NULL,TRUE,'
+
  .$sqlPage.',"ok",TRUE,'
 +
  .$sqlEdit.','
 
  .$sqlDiff.')';
 
  .$sqlDiff.')';
−
$this->dbSpam->Exec($sql);
+
$ok = $this->dbSpam->Exec($sql);
 +
if ($ok !== TRUE) {
 +
    $this->AddErrorLine('SQL ['.$sql.'] in RecordOkEdit() generated this error: '.$ok);
 +
}
 
}
 
}
 
}
 
}

Revision as of 20:02, 1 October 2009

Navigation

{{#lst:SpamFerret|navbar}}: SpamFerret.php

Code

<php><?php /* 

HISTORY:
 2007-08-19 (Wzl) fixed line 155 call to clsDatabase::Query() (method deprecated and removed)
 2007-09-30 (Wzl) fixing regex processing
 2007-10-11 (Wzl) logging ampersandbot attempts; spam throttle (automatic temporary blacklist)
 2007-10-13 (Wzl) fixed some issues which were preventing throttling from working - mainly changes to SQL
 2007-10-15 (Wzl) "Code" wasn't being recorded. Decided that normal filtering should use code '-' so NULL means something is wrong.
 2007-10-28 (Wzl) Events with THR and AMP codes *still* weren't being recorded because method call was improperly formatted.
 2007-12-23 (Wzl) Emails wikimaster if eregi() returns an error (due to improperly formatted regex)
 2007-12-26 (Wzl) Spam turd rejection / logging
 2007-12-27 (Wzl) Fixed spam turd detection to work for new pages too (will probably need refinement)
 2007-12-27 (Wzl) Ooops. Replaced missing if-block from regex results when inserted text is found.
 2008-08-29 (Wzl) Added permanent IP blocking
 2008-09-04 (Wzl) Added (optional) logging of successful edits
 2008-09-19 (Wzl) Actually *set* the "didEdit" flag for successful edits <facepalms>
 2008-10-21 (Wzl) Fixed minor syntax error in "defines"
 2009-02-25 (Wzl) $objArticleCurr->loadLastEdit() now causes error in MW 1.14 (was it necessary before?)
 2009-03-10 (Wzl) "require" -> "require_once" so other extensions can use data.php without conflict
   also optional $kfpWzlLibs so data.php can be somewhere not on the path
 2009-03-18 (Wzl) Got rid of shared.php requirement; now using kfpLib to locate data.php
 2009-03-26 (Wzl) Rewrote data library calls to use new classes (no longer using deprecated/removed classes)
   Also modified to use newer function hooks
 2009-04-24 (Wzl) fixed "strict" bug referencing unset $txtCurr when creating new page
 2009-07-05 (Wzl) Using LibMgr
 2009-07-14 (Wzl) Added attempts.Diff field, patterns.isDIff
 2009-07-15 (Wzl)
   On advice from FreeNode##php, changed from eregi() to preg_match()
   Added option to match diff results instead of submitted edit only
   Saves diff of each change, approved or not
   BUG: approved edits are not being logged properly; using "OK" code and logging as failed
 2009-07-26 (Wzl) fixed minor warning error on line 252
 2009-08-07 (Wzl) email notification working; removed TRD and AMP hard-coded offenses, to be redone as isDiff filters if needed
 2009-08-08 (Wzl) create client record immediately if client is not recognized; don't depend on spam filter being triggered
 2009-08-09 (Wzl) restructured "client" and "attempt" tables (was "clients", "attempts"); not backwards-compatible
TO DO:
 * Throttled save attempts should check for spam, just for data-gathering purposes.
   Possibly non-spam from a throttled IP should not update the "WhenLast" timestamp. Maybe this should be a LocalSettings option?
 * Figure out how to display a different error message than "the following text is what triggered our filter:"
OPTIONAL SETTINGS:
 kfpLib - path to data.php folder (no final slash)
 kfsLib_Data - filespec of data.php
  • /
  1. Loader for spam blacklist feature
  2. Include this from LocalSettings.php

if ( defined( 'MEDIAWIKI' ) ) {

$wgExtensionCredits['other'][] = array( 'name' => 'SpamFerret', 'author' => 'Woozle Staddon', 'url' => 'http://htyp.org/SpamFerret', 'version' => '2009-08-09', 'description' => 'database-driven wikispam blocker', ); if (!defined('kfsLib_Data')) { 

   if (defined('kfpLib')) {

define('kfsLib_Data', kfpLib.'/data.php'); 

   } else {

define('kfsLib_Data','data.php'); // assume it's on the path 

   }

} if (!defined('LIBMGR')) { 

   require('libmgr.php');

} clsLibMgr::Add('data', kfsLib_Data); clsLibMgr::Load('data');

/* ============== 

SET UP CALLBACKS
  • /

global $wgFilterCallback, $wgPreSpamFilterCallback;

$wgPreSpamFilterCallback = false; if ( defined( 'MW_SUPPORTS_EDITFILTERMERGED' ) ) { 

   $wgHooks['EditFilterMerged'][] = 'wfSpamFerretMerged';

} else { 

   if ( $wgFilterCallback ) {
       $wgPreSpamFilterCallback = $wgFilterCallback;
   }
   $wgFilterCallback = 'wfSpamFerretFilter';

}

/* $wgHooks['EditFilter'][] = 'wfSpamFerretValidate'; $wgHooks['ArticleSaveComplete'][] = 'wfSpamFerretArticleSave'; $wgHooks['APIEditBeforeSave'][] = 'wfSpamFerretAPIEditBeforeSave';

  • /

/* ================ 

SET GLOBAL OBJECTS
  • /

function GetSpamFerret() { 

   static $objFerret;

   if (!isset($objFerret)) {

$objFerret = new SpamFerret(); 

   }
   return $objFerret;

} /* ================ 

CALLBACK FUNCTIONS
  • /

/** 

* Hook function for $wgFilterCallback
*/

//function wfSpamFerretFilter( &$title, $text, $section, &$hookErr, $editSummary ) { function wfSpamFerretFilter( &$title, $text, $section ) { 

   global $wgOut;

   $spamObj = GetSpamFerret();
   $wgOut->addWikiText( "Intercepted by SpamFerretFilter" );

// $ret = $spamObj->filter( $title, $text, , $editSummary, $editPage ); 

   return $spamObj->filter( $title, $text, $section );

}

/** 

* Hook function for EditFilterMerged, replaces wfSpamBlacklistFilter
*/

function wfSpamFerretMerged( &$editPage, $text, &$hookErr, $editSummary ) { 

   global $wgTitle,$wgOut;

   if( is_null( $wgTitle ) ) {
       # API mode
       # wfSpamBlacklistFilterAPIEditBeforeSave already checked the blacklist
       return true;
   }

   $spamObj = GetSpamFerret();
   $title = $editPage->mArticle->getTitle();

// $ret = $spamObj->filter( $title, $text, , $editSummary, $editPage ); 

   $ret = $spamObj->filter( $title, $text,  );
   if ( $ret !== false ) $editPage->spamPage( $ret );

// additional text can be added here: // $wgOut->addWikiText( "Intercepted by SpamFerretMerged" ); 

   // Return convention for hooks is the inverse of $wgFilterCallback
   return ( $ret === false );

} /** 

* Hook function for APIEditBeforeSave
*/

function wfSpamFerretAPIEditBeforeSave( &$editPage, $text, &$resultArr ) { }

class SpamFerret { var $previousFilter = false; // internal data var $dbSpam; var $objDataClients; var $strIPAddr; var $idPattern; var $doClearThrottle;

// function SpamFerret() { // }

function Setting($iName) { global $wgSpamFerretSettings;

return $wgSpamFerretSettings[$iName]; }

// function filter( &$title, $text, $section, $editSummary, $editPage ) { function filter( &$title, $text, $section, $editPage = FALSE ) { global $wgArticle, $wgDBname, $wgMemc, $messageMemc, $wgVersion, $wgOut; global $wgTitle, $wgServer; global $debug; global $errNum, $errStr; global $gErrorText; // debugging: global $sql;

$fname = 'wfSpamFerretFilter'; wfProfileIn( $fname );

ini_set('track_errors', 1);

# Call the rest of the hook chain first if ( $this->previousFilter ) { $f = $this->previousFilter; if ( $f( $title, $text, $section ) ) { wfProfileOut( $fname ); return true; } } // initialize variables $retVal = FALSE; // default = assume edit is ok $gErrorText = FALSE; $isClientKnown = FALSE;

$this->txtEditRaw = $text;

// get the IP address of the http client making the edit attempt: $this->strIPAddr = wfGetIP(); // Open the database $this->OpenDatabase(); // open clients table (extended Throttle version) for reference: //return TRUE; $objTblClients = new clsTable($this->dbSpam); $objTblClients->Name('ClientThrottle2'); $objTblClients->KeyName('Address'); // Look up to see if this IP is known; it may already be throttled: $this->objDataClients = $objTblClients->GetData('Address="'.$this->strIPAddr.'"'); if (is_object($this->objDataClients)) { if ($this->objDataClients->hasRows()) { $isClientKnown = TRUE; } } if ($isClientKnown) { $this->objDataClients->FirstRow(); // $this->idClient = $this->objDataClients->ID; $doBlock = $this->objDataClients->doBlock; if ($doBlock) { $strThrType = 'BLK'; } else { $intRetries = $this->objDataClients->Retries; $intThrottle = $this->Setting('throttle_retries'); $doBlock = $intRetries > $intThrottle; $strThrType = 'THR-'.$intRetries; } if ($doBlock) { // retry limit exceeded; check timeout limit if ($this->objDataClients->ThrottleTime < $this->Setting('throttle_timeout')) { $txtMsg = 'Too many spam attempts from your IP address ('.$this->strIPAddr.'). Please come back later.'; if ($editPage) { $editPage->spamPage($txtMsg); } else { EditPage::spamPage($txtMsg); // older MW code doesn't supply $editPage } $this->RecordAttempt($strThrType); // record post attempt by throttled client $retVal = true; // client has exceeded spam limit; impose throttle } else { $this->doClearThrottle = true; } } } else { $this->CreateClient(); }

$this->txtDiff = 'N/A'; /* if ($debug) { EditPage::spamPage('DEBUG: '.$debug); $retVal = true; }

  • /

if (!$retVal) { $this->GetDiff($title); // get the diff between edit and current contents $this->CheckFilters(FALSE); if ( $this->isMatch ) { // spam cue found; display the matching text and don't allow the edit to be saved: wfDebug( "Match!\n" );

// The string sent to spamPage() will be shown after "The following text is what triggered our spam filter:" $retVal = '(pattern #'.$this->idPattern.') '.$this->strMatch; // Log the spam attempt: $this->RecordAttempt('-'); } else { // no spam cues found; allow the edit to be saved, if nothing else has tripped the filter //if ($this->Setting('log_ok_edits')) { $this->idPattern = NULL; $this->RecordOkEdit(); //$this->RecordAttempt('OK'); //} } }

wfProfileOut( $fname ); $this->ReportErrors(); //$wgOut->addHTML($out); return $retVal; /**/ } public function OpenDatabase() { $this->dbSpam = new clsDatabase($this->Setting('dbspec')); $this->dbSpam->Open(); } public function GetDiff($iTitle) { $objArticleCurr = new Article($iTitle); if ($objArticleCurr->exists()) { $txtCurr = $objArticleCurr->getContent(); $this->txtDiff = FigureDiff($txtCurr,$this->txtEditRaw); /* $lenIns = strpos($txtEditRaw,$txtCurr); if ($lenIns !== false) { $strIns = substr($this->txtEditRaw,0,$lenIns); } else { $strIns = ; }

  • /

} else { /* $lenIns = strlen($text); $strIns = $text; $txtCurr = NULL; // this isn't actually used anywhere // $this->txtDiff = FigureDiff(,$text);

  • /

$this->txtDiff = '!!NEW: '.$this->txtEditRaw; } } public function CheckFilters($iCheckAll) { global $gRegexMatches,$gFilterMatches,$gFilterRows,$gFilterCount; global $debug;

assert('is_object($this->dbSpam)');

$this->PatternTbl = new clsTable($this->dbSpam); $this->PatternTbl->Name('patterns'); $this->PatternTbl->KeyName('ID');

if ($iCheckAll) { $sqlFilt = NULL; } else { $sqlFilt = 'isActive'; }

$this->PatternRows = $this->PatternTbl->GetData($sqlFilt); $objRow = $this->PatternRows;

$strTextEdit = strtolower($this->txtEditRaw); $this->txtEditChk = $strTextEdit; // text after being massaged for checking $this->isMatch = FALSE; $gFilterCount = 0; $gFilterRows = $this->PatternRows->RowCount(); //$objDataPatterns->StartRows(); while($objRow->NextRow() && (!$this->isMatch || $iCheckAll)) { $isMatch = FALSE; if ($objRow->isDiff) { if (isset($this->txtDiff)) { $strTextCk = $this->txtDiff; } else { $strTextCk = NULL; } } else { $strTextCk = $strTextEdit; } if (!is_null($strTextCk)) { $gFilterCount++; $strPattern = $objRow->Pattern; $isRegex = $objRow->isRegex; $this->idPattern = $objRow->ID; if ($isRegex) { $isMatch = $this->CheckRegex($strPattern,$strTextCk);

if (isset($php_errormsg)) { $this->AddErrorLine('Filter #'.$this->idPattern.' generated error "'.$php_errormsg); }

if ($isMatch) { $this->strMatch = $gRegexMatches[0]; } } else { if (empty($strPattern)) { $isMatch = FALSE; } else { $this->strMatch = stristr($strTextCk,$strPattern); $isMatch = ($this->strMatch != ); } } if ($isMatch) { $this->isMatch = TRUE; if ($iCheckAll) { $gFilterMatches[$this->idPattern] = $this->strMatch; } } } } } public function CheckRegex($iPattern,$iText) { global $gRegexMatches;

$chDelim = '/'; $strPattCk = str_replace($chDelim,'\\'.$chDelim,$iPattern); unset($php_errormsg); $isMatch = @preg_match($chDelim.$strPattCk.$chDelim,$iText,$gRegexMatches); return $isMatch; }

public function AddErrorLine($iText) { global $gErrorText;

$gErrorText .= $iText."\n"; } public function ReportErrors() { global $wgUser; global $wgEmergencyContact; global $gErrorText;

if ($gErrorText) { $msgEmail = 'Filter error report for user '.$wgUser->getName().":\n\n"; $msgEmail .= $gErrorText; mail($wgEmergencyContact,'spamferret filter error',$msgEmail); } } /* 

ACTION: Create a new record for the current client
  • /

public function CreateClient() { $sql = 'INSERT INTO client (Address,WhenFirst,Count,Retries) VALUES("'.$this->strIPAddr.'",NOW(),1,0)'; $this->dbSpam->Exec($sql); } /* 

ACTION: Update a client's record to reflect a new spam attempt
  • /

public function RecordClientSpam() { if ($this->doClearThrottle) { $strRetries = '0'; } else { $strRetries = 'Retries+1'; } $sql = 'UPDATE client SET WhenLast=NOW(),Count=Count+1, Retries='.$strRetries.' WHERE Address="'.$this->strIPAddr.'"'; $this->dbSpam->Exec($sql); if ($this->dbSpam->RowsAffected() < 1) { $this->CreateClient(); $this->AddErrorLine('Record not found for client '.$this->strIPAddr); } } public function RecordAttempt($iCode) { global $wgTitle, $wgServer;

$this->RecordClientSpam();

$sqlCode = '"'.$iCode.'"'; $sqlURL = '"'.$this->dbSpam->SafeParam($wgTitle->getFullURL()).'"'; $sqlSrvr = '"'.$this->dbSpam->SafeParam($wgServer).'"'; $sqlPage = '"'.$this->dbSpam->SafeParam($wgTitle->getPrefixedText()).'"'; $sqlEdit = '"'.$this->dbSpam->SafeParam($this->txtEditChk).'"'; $sqlDiff = '"'.$this->dbSpam->SafeParam($this->txtDiff).'"'; $sqlPattern = $this->idPattern; if (is_null($sqlPattern)) { $sqlPattern = 'NULL'; $isPattern = FALSE; } else { $isPattern = TRUE; } $sqlAddr = '"'.$this->strIPAddr.'"'; $sqlMatch = is_null($iMatch)?'NULL':'"'.$iMatch.'"'; $sql = 'INSERT INTO attempt (`When`,ID_Pattern,Addr_Client,IDS_Session,PageServer,PageName,Code,didAllow,Edit,Diff) VALUES' .'(NOW(),' .$sqlPattern.',' .$sqlAddr.',' .SQL_Value(session_id()).',' .$sqlSrvr.',' .$sqlPage.',' .$sqlCode.',' .'FALSE,' .$sqlEdit.',' .$sqlDiff.')'; $ok = $this->dbSpam->Exec($sql); if ($ok !== TRUE) { $this->AddErrorLine('SQL ['.$sql.'] in RecordAttempt() generated this error: '.$ok); } if ($isPattern) { $sql = 'UPDATE patterns SET WhenTried=NOW(), Count=Count+1 WHERE ID='.$this->idPattern; $ok = $this->dbSpam->Exec($sql); if ($ok !== TRUE) { $this->AddErrorLine('SQL ['.$sql.'] generated this error: '.$ok); } } } public function RecordOkEdit() { global $wgTitle, $wgServer;

$sqlPage = '"'.$this->dbSpam->SafeParam($wgTitle->getPrefixedText()).'"'; $sqlSrvr = '"'.$this->dbSpam->SafeParam($wgServer).'"'; $sqlEdit = '"'.$this->dbSpam->SafeParam($this->txtEditChk).'"'; $sqlDiff = '"'.$this->dbSpam->SafeParam($this->txtDiff).'"'; $sql = 'INSERT INTO attempt (`When`,ID_Pattern,Addr_Client,IDS_Session,PageServer,PageName,Code,didAllow,Edit,Diff) VALUES ' .'(NOW(),NULL,' .'"'.$this->strIPAddr.'",' .SQL_Value(session_id()).',' .$sqlSrvr.',' .$sqlPage.',"ok",TRUE,' .$sqlEdit.',' .$sqlDiff.')'; $ok = $this->dbSpam->Exec($sql); if ($ok !== TRUE) { $this->AddErrorLine('SQL ['.$sql.'] in RecordOkEdit() generated this error: '.$ok); } } }

function ErrorHandler ($errno ,$errstr) { global $errNum, $errStr;

$errNum = $errno; $errStr = $errstr; }

function SQL_Value($iVar) { if (is_null($iVar)) { return 'NULL'; } else { if (is_numeric($iVar)) { return $iVar; } else { return '"'.$iVar.'"'; } } } function FigureDiff($iTextOld, $iTextNew) { # Make temporary files // $td = wfTempDir(); $td = session_save_path(); $fhOld = fopen( $fnOld = tempnam( $td, 'SpamFerret-old-' ), 'w' ); $fhNew = fopen( $fnNew = tempnam( $td, 'SpamFerret-new-' ), 'w' );

fwrite( $fhOld, $iTextOld ); fclose( $fhOld ); fwrite( $fhNew, $iTextNew ); fclose( $fhNew );

$ksSpamFerretDiffOptions = '-i -E --suppress-common-lines '; // this setting applies to all uses of a given filter DB // can change for different DBs $cmd = 'diff '.$ksSpamFerretDiffOptions.wfEscapeShellArg( $fnOld, $fnNew ); $handle = popen( $cmd, 'r' ); $result = ; do { $data = fread( $handle, 8192 ); if ( strlen( $data ) == 0 ) { break; } $result .= $data; } while ( true ); pclose( $handle ); unlink( $fnNew ); unlink( $fnOld ); //return 'cmd=['.$cmd.'] diff=['.$result.']'; return $result; }

} // end of 'MEDIAWIKI' check

Retrieved from "https://htyp.org/mw/index.php?title=SpamFerret.php&oldid=12366"