User:AnomieBOT/source/tasks/BrokenRedirectDeleter.pm
 | BRFA approved 2014-02-26 Wikipedia:Bots/Requests for approval/AnomieBOT III |
| First supplemental BFRA approved 2017-05-22 Wikipedia:Bots/Requests for approval/AnomieBOT III 4 |
package tasks::BrokenRedirectDeleter;
=pod
=begin metadata
Bot: AnomieBOT III
Task: BrokenRedirectDeleter
BRFA: Wikipedia:Bots/Requests for approval/AnomieBOT III
Status: Approved 2014-02-26
+BRFA: Wikipedia:Bots/Requests for approval/AnomieBOT III 4
+Status: Approved 2017-05-22
Created: 2014-01-06
Cleans up broken redirects:
* Attempted interwiki redirects are replaced with {{tl|soft redirect}}
* Redirects broken due to a move of the target without leaving a redirect are
updated, if possible.
* Other broken redirects are deleted if the following are true:
** The redirect has only 1 revision OR was created more than 4 days ago
** The redirect is not in the User or User talk namespaces
** The target page has no log entries less than 12 hours ago
** The redirect has 10 or fewer incoming links
** The bot is not excluded, e.g. with {{tl|nobots}}
* When a broken redirect is deleted, any subpages, and the talk page (if any)
and its subpages will be deleted, unless:
** The page is a talk page with an existing subject page
** The page is tagged with {{tl|G8-exempt}}
** The page has more than 10 incoming links
** The bot is excluded, e.g. with {{tl|nobots}}
** The parent subpage wasn't deleted
* Skipped redirects will be reported to [[User:AnomieBOT III/Broken redirects]]
=end metadata
=cut
use utf8;
use strict;
use AnomieBOT::Task qw/ISO2timestamp/;
use Data::Dumper;
use URI::Escape;
use vars qw/@ISA/;
@ISA=qw/AnomieBOT::Task/;
my ($screwup, $ns_subpages, %ns, %rns);
my %ignore = (
'Wikipedia:Example of a broken redirect' => 1,
);
sub new {
my $class=shift;
my $self=$class->SUPER::new();
bless $self, $class;
return $self;
}
=pod
=for info
BRFA approved 2014-02-26<br />[[Wikipedia:Bots/Requests for approval/AnomieBOT III]]
=for info
First supplemental BFRA approved 2017-05-22<br />[[Wikipedia:Bots/Requests for approval/AnomieBOT III 4]]
=cut
sub approved {
return 500;
}
sub run {
my ($self, $api)=@_;
my $res;
$screwup='If this bot is malfunctioning, please report it at [[User:'.$api->user.'/shutoff/BrokenRedirectDeleter]]';
$api->task('BrokenRedirectDeleter', 0, 10, qw/d::IWNS d::Redirects/);
%ns = $api->namespace_map();
%rns = $api->namespace_reverse_map();
$ns_subpages = $api->cache->get( 'BrokenRedirectDeleter:ns_subpages' );
if ( !$ns_subpages ) {
my $res = $api->query(
meta => 'siteinfo',
siprop => 'namespaces',
);
if ( $res->{'code'} ne 'success' ) {
$api->warn( "Failed to fetch namespace info: " . $res->{'error'} . "\n" );
return 60;
}
$ns_subpages = {};
for my $ns (values %{$res->{'query'}{'namespaces'}}) {
$ns_subpages->{$ns->{'id'}} = exists($ns->{'subpages'});
}
$api->cache->set( 'BrokenRedirectDeleter:ns_subpages', $ns_subpages, 86400 );
}
my ($dbh);
eval {
($dbh) = $api->connectToReplica( 'enwiki', 'analytics' );
};
if ( $@ ) {
$api->warn( "Error connecting to replica: $@\n" );
return 300;
}
my $from = $self->{'dbfrom'} // 0;
my $report = $self->{'report'} // {};
if ( $from == 0 && %$report ) {
$res = $self->do_report( $api, $report );
return $res if $res;
$self->{'report'} = $report = {};
}
# Ensure bot is logged in.
$res = $api->login();
if ( $res->{'code'} ne 'success' ) {
$api->warn( "Not logged in (WTF?): " . $res->{'error'} . "\n" );
return 300;
}
# Spend a max of 5 minutes on this task before restarting
my $endtime=time()+300;
my $dbmax = (@{ $dbh->selectcol_arrayref('SELECT MAX(rd_from) FROM redirect') })[0];
while ( $from <= $dbmax ) {
return 0 if $api->halting;
# Load the list of redirects needing deletion
my @rows;
my $to = $from + 1e6;
$api->debug(1, "Selecting broken redirects $from-$to");
eval {
@rows = @{ $dbh->selectall_arrayref( qq{
SELECT rd_from, p1.page_namespace, p1.page_title, rd_interwiki, rd_namespace, rd_title, rd_fragment, last_updated
FROM redirect
JOIN page AS p1 ON(rd_from = p1.page_id)
LEFT JOIN page AS p2 ON(rd_namespace=p2.page_namespace AND rd_title=p2.page_title)
JOIN heartbeat_p.heartbeat ON(shard = 's1')
WHERE rd_namespace >= 0
AND (p2.page_id IS NULL OR rd_interwiki != '' AND rd_interwiki IS NOT NULL)
AND rd_from > $from AND rd_from <= $to
}, { Slice => {} } ) };
};
if ( $@ ) {
$api->warn( "Error fetching page list from replica: $@\n" );
return 300;
}
@rows = sort { $a->{'rd_from'} <=> $b->{'rd_from'} } @rows;
for my $row (@rows) {
return 0 if $api->halting;
$row->{'rd_interwiki'} //= '';
utf8::decode( $row->{'rd_interwiki'} ); # Data from database is binary
utf8::decode( $row->{'page_title'} ); # Data from database is binary
my $from = $row->{'page_title'};
$from = $rns{$row->{'page_namespace'}} . ':' . $from if $row->{'page_namespace'} != 0;
$from =~ s/_/ /g;
utf8::decode( $row->{'rd_title'} ); # Data from database is binary
my $to = $row->{'rd_title'};
$to = $rns{$row->{'rd_namespace'}} . ':' . $to if $row->{'rd_namespace'} != 0;
$to = $row->{'rd_interwiki'} . ':' . $to if $row->{'rd_interwiki'} ne '';
$to =~ s/_/ /g;
utf8::decode( $row->{'rd_fragment'} ); # Data from database is binary
my $to_f = $row->{'rd_fragment'} ? "$to#$row->{rd_fragment}" : $to;
if ( defined( $ignore{$from} ) ) {
#$api->log( "Ignoring $from, it's on the ignore list." );
goto done;
}
#$api->log( "Checking [[$from]] → [[$to_f]]" );
if ( $row->{'page_namespace'} == $ns{'User'} || $row->{'page_namespace'} == $ns{'User talk'} ) {
# Always skip User and User talk namespaces
push @{$report->{'user'}}, "[[:$from]] → [[:$to_f]]";
goto done;
} elsif ( $row->{'rd_interwiki'} ne '' ) {
my $tok = $api->edittoken( $from, EditRedir => 1 );
if ( $tok->{'code'} eq 'shutoff' ) {
$api->warn( "Task disabled: " . $tok->{'content'} . "\n" );
return 300;
}
if ( $tok->{'code'} eq 'botexcluded' ) {
$api->warn( "Bot excluded from $from: " . $tok->{'error'} . "\n" );
push @{$report->{'skip'}}, "[[:$from]] → [[:$to_f]] (bot excluded)";
goto done;
}
if ( $tok->{'code'} ne 'success' ) {
$api->warn( "Failed to get edit token for $from: " . $tok->{'error'} . "\n" );
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] (couldn't fetch edit token)";
goto failed;
}
if ( exists( $tok->{'missing'} ) ) {
#$api->log("$from no longer exists, skipping");
goto done;
}
if ( $tok->{'revisions'}[0]{'timestamp'} gt $row->{'last_updated'} ) {
$api->log("Excessive replag for $from, skipping ($tok->{'revisions'}[0]{'timestamp'} > $row->{'last_updated'})");
goto done;
}
my $txt = $tok->{'revisions'}[0]{'slots'}{'main'}{'*'};
$txt=~s/^\s*.*?\]\]/{{soft redirect|1=$to_f}}\n/;
my $summary = "Changing interwiki redirect to [[:$to_f]] into a soft redirect";
$api->log( "$summary in $from" );
my $r = $api->edit( $tok, $txt, "$summary. $screwup", 0, 1 );
if ( $r->{'code'} ne 'success' ) {
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] (edit failed)";
goto failed;
}
goto done;
} else {
# Make sure $to didn't get recreated
my $pg2 = undef;
if ( $to ne '' ) {
$res = $api->query( titles => $to, prop => 'imageinfo' );
if ( $res->{'code'} ne 'success' ) {
$api->warn( "Failed to get existence for $to: " . $res->{'error'} . "\n" );
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] (couldn't fetch target existence)";
goto failed;
}
$pg2 = (values %{$res->{'query'}{'pages'}})[0];
if ( exists( $pg2->{'pageid'} ) ) {
#$api->log("$to_f still exists, skipping");
goto done;
}
$res = $api->query( titles => $from, redirects => 1 );
if ( $res->{'code'} ne 'success' ) {
$api->warn( "Failed to resolve redirects $from: " . $res->{'error'} . "\n" );
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] (couldn't check redirect)";
goto failed;
}
my $pg3 = (values %{$res->{'query'}{'pages'}})[0];
if ( !exists( $pg3->{'missing'} ) && $pg3->{'title'} ne $to ) {
$api->log("$from is no longer a redirect to $to_f, skipping");
goto done;
}
}
my %q = (
titles => $from,
prop => 'info|revisions|imageinfo',
inprop => 'talkid',
rvprop => 'timestamp',
rvlimit => 2,
iiprop => 'canonicaltitle',
);
if ( $to ne '' ) {
%q = ( %q,
list => 'logevents',
letitle => $to,
lelimit => 1,
);
}
$res = $api->query( %q );
if ( $res->{'code'} ne 'success' ) {
$api->warn( "Failed to get info for $from and $to: " . $res->{'error'} . "\n" );
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] (couldn't fetch page info)";
goto failed;
}
my @le = @{$res->{'query'}{'logevents'} // []};
my $pg = (values %{$res->{'query'}{'pages'}})[0];
if ( exists( $pg->{'missing'} ) ) {
#$api->log("$from no longer exists, skipping");
goto done;
}
if ( !exists( $pg->{'redirect'} ) ) {
$api->log("$from is no longer a redirect, skipping");
goto done;
}
if ( ($pg->{'imagerepository'}//'') eq 'local' && $pg->{'imageinfo'}[0]{'canonicaltitle'} eq $pg->{'title'} ) {
#$api->log("$from has a local file version");
push @{$report->{'skip'}}, "[[:$from]] → [[:$to_f]] (page is a redirect, but a local file exists at the title)";
goto done;
}
if ( $pg2 && ($pg2->{'imagerepository'}//'') eq 'local' ) {
#$api->log("$to has a file");
push @{$report->{'skip'}}, "[[:$from]] → [[:$to_f]] (a file exists at the target title)";
goto done;
}
if ( $pg2 && ($pg2->{'imagerepository'}//'') ne '' ) {
#$api->log("$to has a file");
push @{$report->{'skip'}}, "[[:$from]] → [[:$to_f]] (a non-local file exists at the target title; while this doesn't actually work, people would rather fix it manually)";
goto done;
}
if ( @{$pg->{'revisions'}} > 1 && ISO2timestamp( $pg->{'revisions'}[0]{'timestamp'} ) > time() - 4*86400 ) {
#$api->log("$from has more that one revision and was edited recently");
push @{$report->{'wait'}}, "[[:$from]] → [[:$to_f]] (redirect has more than 1 revision and was edited less than 4 days ago)";
goto done;
}
if ( @le ) {
my $le = $le[0];
my ( $err, $ts, $newTo ) = $self->checkLogEntry( $api, $le );
if ( $err ) {
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] (couldn't check log entries)";
goto failed;
}
if ( ISO2timestamp( $ts ) > time() - 43200 ) {
#$api->log("$to (or something in the chain of moves after it) has recent log entries");
if ( $newTo && $newTo ne $to ) {
push @{$report->{'wait'}}, "[[:$from]] → [[:$to_f]] → [[:$newTo]] (intermediate or eventual target has recent log entries)";
} else {
push @{$report->{'wait'}}, "[[:$from]] → [[:$to_f]] (intermediate or eventual target has recent log entries)";
}
goto done;
}
if ( $newTo && $newTo ne $to ) {
my $newTo_f = $newTo;
$newTo_f .= $1 if $to_f =~ /(#.*)$/;
my $tok = $api->edittoken( $from, EditRedir => 1, NoExclusion => 1 );
if ( $tok->{'code'} eq 'shutoff' ) {
$api->warn( "Task disabled: " . $tok->{'content'} . "\n" );
return 300;
}
if ( $tok->{'code'} ne 'success' ) {
$api->warn( "Failed to get edit token for $from: " . $tok->{'error'} . "\n" );
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] → [[:$newTo]] (couldn't fetch edit token)";
goto failed;
}
my $re = $api->redirect_regex();
my $txt = $tok->{'revisions'}[0]{'slots'}{'main'}{'*'} // '';
unless ( $txt=~s/($re)\[\[[^]]+\]\]/$1\[\[$newTo_f\]\]/ ) {
$api->warn( "Failed to replace #REDIRECT in $from\n" );
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] → [[:$newTo]] (couldn't replace #REDIRECT)";
goto failed;
}
my $summary = "Redirecting [[:$from]] to [[:$newTo_f]] following a move-without-redirect of [[:$to]]";
$api->log( $summary );
my $r = $api->edit( $tok, $txt, "$summary. $screwup", 0, 1 );
if ( $r->{'code'} ne 'success' ) {
push @{$report->{'fail'}}, "[[:$from]] → [[:$to_f]] → [[:$newTo]] (edit failed)";
goto failed;
}
goto done;
}
}
my ($key, $reason);
($res, $key, $reason) = $self->do_delete( $api, $row->{'page_namespace'}, $row->{'page_title'}, "[[WP:CSD#G8|G8]]: Broken redirect to [[:$to_f]]", 1 );
if ( $res ) {
return $res if $res > 0;
push @{$report->{$key}}, "[[:$from]] → [[:$to_f]] ($reason)";
goto failed if $key eq 'failed';
goto done;
}
$self->do_delete( $api, $row->{'page_namespace'}+1, $row->{'page_title'}, "[[WP:CSD#G8|G8]]: Talk page of deleted page", 1 ) if exists($pg->{'talkid'});
}
done:
$self->{'dbfrom'} = $row->{'rd_from'};
failed:
$self->{'report'} = $report;
# If we've been at it long enough, let another task have a go.
return 0 if time()>=$endtime;
}
$self->{'dbfrom'} = $from = $to;
}
$self->{'dbfrom'} = 0;
my $wait = exists($report->{'fail'}) ? 60 : 21600;
$res = $self->do_report( $api, $report );
return $res if $res;
$self->{'report'} = {};
return $wait;
}
sub do_delete {
my ($self, $api, $ns, $title, $reason, $subpages) = @_;
$title =~ s/_/ /g;
my $page = $ns ? $rns{$ns} . ':' . $title : $title;
my $tok=$api->gettoken('csrf', Title => $page, EditRedir => 1, templates => { templates => 'Template:G8-exempt' } );
if ( $tok->{'code'} eq 'shutoff' ) {
$api->warn( "Task disabled: " . $tok->{'content'} . "\n" );
return (300, undef);
}
if ( $tok->{'code'} eq 'botexcluded' ) {
$api->warn( "Bot excluded from $page: " . $tok->{'error'} . "\n" );
return (-1, 'skip', 'bot excluded');
}
if ( $tok->{'code'} ne 'success' ) {
$api->warn( "Failed to get delete token for $page: " . $tok->{'error'} . "\n" );
return (-1, 'fail', "couldn't fetch delete token");
}
if ( exists( $tok->{'missing'} ) ) {
#$api->log("$page no longer exists, skipping");
return (0, 'ok', "no longer exists");
}
if ( @{$tok->{'templates'} // []} ) {
#$api->log("$page is G8-exempt");
return (-1, 'skip', "marked G8-exempt");
}
my $res = $api->query(
list => 'backlinks',
bltitle => $page,
bllimit => 10,
);
if ( exists( $res->{'query-continue'}{'backlinks'} ) ) {
#$api->log("$page has too many backlinks");
return (-1, 'skip', "too many incoming links");
}
$api->log( "Deleting $page: $reason" );
$res = $api->action( $tok,
action => 'delete',
title => $page,
reason => "$reason. $screwup",
);
if ( $res->{'code'} ne 'success' ) {
$api->warn( "Failed to delete $page: " . $res->{'error'} . "\n" );
return (-1, 'fail', 'delete failed');
}
if ( $subpages && $ns_subpages->{$ns} ) {
# Delete subpages of deleted page
my $iter = $api->iterator(
generator => 'allpages',
gapnamespace => $ns,
gapprefix => "$title/",
gaplimit => 'max',
prop => 'info',
inprop => 'subjectid',
);
my %skip = ();
ITER: while( my $p = $iter->next ) {
last unless $p->{'_ok_'};
my @parts = split( m!/!, $p->{'title'} );
for ( my $i = 0; $i < @parts; $i++ ) {
my $t = join( '/', @parts[0..$i] );
next ITER if exists( $skip{$t} );
}
if ( exists( $p->{'subjectid'} ) || @{$p->{'templates'} // []} ) {
$skip{$p->{'title'}} = 1;
next ITER;
}
$p->{'title'}=~s/^[^:]*:// if $ns;
my ($res, $key, $reason) = $self->do_delete( $api, $ns, $p->{'title'}, "[[WP:CSD#G8|G8]]: Subpage of a deleted page", 0 );
$skip{$p->{'title'}} = 1 if $res;
}
}
}
sub do_report {
my ($self, $api, $report) = @_;
my $txt;
$txt = "<noinclude>This page reports on redirects that AnomieBOT\'s BrokenRedirectDeleter will not clean up because they are in the User or User talk namespace. This page was last updated {{#time:Y-m-d H:i:s|{{REVISIONTIMESTAMP}}}} (UTC).";
if ( exists( $report->{'user'} ) ) {
$txt .= "</noinclude>\n";
$txt .= "== User space ==\nThese redirects are in the User or User talk namespaces, and will '''not''' be automatically cleaned up.\n<includeonly>{{collapse top|title=Userspace redirects}}</includeonly>\n";
$txt .= "* " . join( "\n* ", @{$report->{'user'}} ) . "\n<includeonly>{{collapse bottom}}</includeonly>";
} else {
$txt .= "\n\nThere are no unhandled redirects at this time.</noinclude>";
}
$txt =~ s/(?<=^\* )\[\[:(.*?)\]\]/{{no redirect|1=$1}}/gm;
$txt =~ s/\s*$/\n/;
my $title = "User:" . $api->user . "/Broken redirects/Userspace";
my $tok = $api->edittoken( $title, EditRedir => 1, NoExclusion => 1 );
if ( $tok->{'code'} eq 'shutoff' ) {
$api->warn( "Task disabled: " . $tok->{'content'} . "\n" );
return 300;
}
if ( $tok->{'code'} ne 'success' ) {
$api->warn( "Failed to get edit token for $title: " . $tok->{'error'} . "\n" );
return 60;
}
my $intxt = $tok->{'revisions'}[0]{'slots'}{'main'}{'*'} // '';
$intxt =~ s/\s*$/\n/;
if ( $txt ne $intxt ) {
$api->log( "Updating $title" );
my $r = $api->edit( $tok, $txt, "Updating broken redirects list", 0, 1 );
if ( $r->{'code'} ne 'success' ) {
return 60;
}
}
$txt = "This page reports on redirects that AnomieBOT\'s BrokenRedirectDeleter cannot clean up. This page was last updated {{#time:Y-m-d H:i:s|{{REVISIONTIMESTAMP}}}} (UTC).\n\n";
unless ( %$report ) {
$txt .= "There are no unhandled redirects at this time.";
}
if ( exists( $report->{'skip'} ) ) {
$txt .= "== Skipped ==\nThese redirects were skipped by the bot, and will '''not''' be automatically cleaned up as long as the indicated issues apply.\n";
$txt .= "* " . join( "\n* ", @{$report->{'skip'}} ) . "\n\n";
}
if ( exists( $report->{'user'} ) ) {
$txt =~ s/\s*$/\n/;
$txt .= "{{ {{FULLPAGENAME}}/Userspace }}\n";
}
if ( exists( $report->{'wait'} ) ) {
$txt .= "== Recently changed ==\nThese redirects were created recently, or their targets have recent log entries. The bot will process them after the appropriate waiting period.\n";
$txt .= "* " . join( "\n* ", @{$report->{'wait'}} ) . "\n\n";
}
if ( exists( $report->{'fail'} ) ) {
$txt .= "== Failed ==\nThe most recent attempt to edit or delete these redirects failed. The bot will retry on its next run.\n";
$txt .= "* " . join( "\n* ", @{$report->{'fail'}} ) . "\n\n";
}
$txt =~ s/(?<=^\* )\[\[:(.*?)\]\]/{{no redirect|1=$1}}/gm;
$txt =~ s/\s*$/\n/;
$title = "User:" . $api->user . "/Broken redirects";
$tok = $api->edittoken( $title, EditRedir => 1, NoExclusion => 1 );
if ( $tok->{'code'} eq 'shutoff' ) {
$api->warn( "Task disabled: " . $tok->{'content'} . "\n" );
return 300;
}
if ( $tok->{'code'} ne 'success' ) {
$api->warn( "Failed to get edit token for $title" . $tok->{'error'} . "\n" );
return 60;
}
$intxt = $tok->{'revisions'}[0]{'slots'}{'main'}{'*'} // '';
$intxt =~ s/\s*$/\n/;
if ( $txt ne $intxt ) {
$api->log( "Updating $title" );
my $r = $api->edit( $tok, $txt, "Updating broken redirects list", 0, 1 );
if ( $r->{'code'} ne 'success' ) {
return 60;
}
}
return 0;
}
sub checkLogEntry {
my ( $self, $api, $le ) = @_;
my $ts = $le->{'timestamp'};
return ( undef, $ts, undef ) unless ( $le->{'type'} eq 'move' && exists( $le->{'params'}{'suppressredirect'} ) && $le->{'params'}{'target_ns'} == $le->{'ns'} );
# Find the new target
my $target = $le->{'params'}{'target_title'};
# Bypass double redirects
my $res = $api->query( titles => $target, redirects => 1 );
if ( $res->{'code'} ne 'success' ) {
$api->warn( "Failed to retrieve info for $target: " . $res->{'error'} . "\n" );
return ( 'fail' );
}
my %map = ();
if ( exists( $res->{'query'}{'normalized'} ) ) {
$map{$_->{'from'}} = $_->{'to'} foreach @{$res->{'query'}{'normalized'}};
}
if ( exists($res->{'query'}{'redirects'} ) ) {
$map{$_->{'from'}} = $_->{'to'} foreach @{$res->{'query'}{'redirects'}};
}
$target = $map{$target} if exists( $map{$target} );
# Does the final target exist?
my %exists = ();
if ( exists( $res->{'query'}{'pages'} ) ) {
for my $p (values %{$res->{'query'}{'pages'}}) {
$exists{$p->{'title'}} = $p->{'ns'} if $p->{'pageid'}//0;
}
}
if ( exists( $exists{$target} ) ) {
return ( undef, $ts, ($exists{$target} == $le->{'ns'} ? $target : undef ) );
}
# No, check if it in turn was moved-without-redirect.
$res = $api->query( list => 'logevents', letitle => $target, lelimit => 1 );
if ( $res->{'code'} ne 'success' ) {
$api->warn( "Failed to retrieve log events for $target: " . $res->{'error'} . "\n" );
return ( 'fail' );
}
$target = undef; # If not, just return no target.
my @le = @{$res->{'query'}{'logevents'} // []};
if ( @le ) {
my ( $err, $ts2 );
( $err, $ts2, $target ) = $self->checkLogEntry( $api, $le[0] );
return ( $err ) if $err;
$ts = $ts2 if defined( $ts2 ) && $ts2 lt $ts;
}
return ( undef, $ts, $target );
}
1;