User:Polygnotus/DuplicateReferences.js
Appearance
Code that you insert on this page could contain malicious content capable of compromising your account. If you import a script from another page with "importScript", "mw.loader.load", "iusc", or "lusc", take note that this causes you to dynamically load a remote script, which could be changed by others. Editors are responsible for all edits and actions they perform, including by scripts. User scripts are not centrally supported and may malfunction or become inoperable due to software changes. A guide to help you find broken scripts is available. If you are unsure whether code you are adding to this page is safe, you can ask at the appropriate village pump. This code will be executed when previewing this page. |
![]() | This user script seems to have a documentation page at User:Polygnotus/DuplicateReferences. |
//Testpage: https://en.wikipedia.org/wiki/User:Polygnotus/DuplicateReferencesTest
// <nowiki>
mw.loader.using(['mediawiki.util'], function () {
$(document).ready(function () {
const DEBUG = false;
function debug(...args) {
if (DEBUG) {
console.log('[DuplicateReferences]', ...args);
}
}
if ((mw.config.get('wgNamespaceNumber') !== 0 && mw.config.get('wgPageName') !== 'User:Polygnotus/DuplicateReferencesTest') || mw.config.get('wgAction') !== 'view') {
debug("Not the correct page or action, script terminated");
return;
}
debug("Page title:", document.title);
debug("URL:", window.location.href);
function findNextReflistDiv(element) {
let nextElement = element.nextElementSibling;
while (nextElement) {
if (nextElement.tagName.toLowerCase() === 'div' &&
(nextElement.classList.contains('reflist') || nextElement.classList.contains('mw-references-wrap'))) {
return nextElement;
}
nextElement = nextElement.nextElementSibling;
}
return null;
}
const referencesHeader = document.querySelector("h2#References");
if (!referencesHeader) {
debug("References heading not found, script terminated");
return;
}
const containerDiv = referencesHeader.closest("div");
if (!containerDiv) {
debug("Container div not found, script terminated");
return;
}
const reflistDiv = findNextReflistDiv(containerDiv);
if (!reflistDiv) {
debug("Reflist div not found, script terminated");
return;
}
const referencesList = reflistDiv.querySelector('ol.references');
if (!referencesList) {
debug("ol.references not found within reflist div");
return;
}
const style = document.createElement('style');
style.textContent = `
li:target { border: 1px dotted red; padding: 2px; background-color: #ffcccc !important;}
.duplicate-citation-highlight { background-color: #e1eeff; }
.duplicate-citation-hover { background-color: #cce0ff; border: 1px dotted blue; }
.duplicate-citation-clicked { border: 1px dotted red; padding: 2px; background-color: #ffe6e6; }
.mw-collapsible-toggle { font-weight: normal; float: right; }
.duplicate-references-table { width: 100%; }
@media only screen and (max-width: 768px) {
.duplicate-references-table { display: none; }
}
`;
document.head.appendChild(style);
function addDuplicateCitationsTemplate() {
debug("Adding duplicate citations template");
var api = new mw.Api();
var pageTitle = mw.config.get('wgPageName');
let duplicateInfo = getDuplicateInfo();
// Get current date
const currentDate = new Date();
const monthNames = ["January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December"
];
const currentMonth = monthNames[currentDate.getMonth()];
const currentYear = currentDate.getFullYear();
const dateParam = `|date=${currentMonth} ${currentYear}`;
api.get({
action: 'query',
prop: 'revisions',
titles: pageTitle,
rvprop: 'content',
rvslots: 'main',
formatversion: 2
}).then(function(data) {
var page = data.query.pages[0];
var content = page.revisions[0].slots.main.content;
// Define the templates to check for
const templatesToCheck = [
'{{short description',
'{{DISPLAYTITLE',
'{{Lowercase title',
'{{Italic title',
'{{about',
'{{redirect',
'{{Distinguish',
'{{for'
];
// Find the position to insert the new template
let insertPosition = 0;
let lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
let line = lines[i].trim().toLowerCase();
if (templatesToCheck.some(template => line.startsWith(template.toLowerCase()))) {
insertPosition = i + 1;
} else if (line && !line.startsWith('{{') && !line.startsWith('__')) {
break;
}
}
// Create the reason string
let reason = '[[User:Polygnotus/DuplicateReferences|DuplicateReferences]] detected:<br>\n';
if (duplicateInfo.length > 0) {
duplicateInfo.forEach((info) => {
reason += `* ${info.url} (refs: ${info.refs.map(r => r.number).join(', ')})<br>\n`;
});
}
// Insert the new template with the reason parameter
lines.splice(insertPosition, 0, `{{Duplicated citations|reason=${reason}${dateParam}}}`);
var newContent = lines.join('\n');
let summary = `[[User:Polygnotus/DuplicateReferences|DuplicateReferences]] +{{Duplicated citations|reason=${reason}${dateParam}}}`;
return api.postWithToken('csrf', {
action: 'edit',
title: pageTitle,
text: newContent,
summary: summary
});
}).then(function() {
mw.notify('Successfully added the Duplicate citations template!');
location.reload();
}).catch(function(error) {
console.error('Error:', error);
mw.notify('Failed to add the template. See console for details.', {type: 'error'});
});
}
function getVisibleText(element) {
// Recursively get the visible text content of an element
let text = '';
for (let node of element.childNodes) {
if (node.nodeType === Node.TEXT_NODE) {
text += node.textContent.trim() + ' ';
} else if (node.nodeType === Node.ELEMENT_NODE) {
// Skip hidden elements
const style = window.getComputedStyle(node);
if (style.display !== 'none' && style.visibility !== 'hidden') {
text += getVisibleText(node) + ' ';
}
}
}
return text.trim();
}
function calculateLevenshteinDistance(a, b) {
debug("Comparing:");
debug("Text 1:", a);
debug("Text 2:", b);
if (a.length === 0) return b.length;
if (b.length === 0) return a.length;
const matrix = [];
// Increment along the first column of each row
for (let i = 0; i <= b.length; i++) {
matrix[i] = [i];
}
// Increment each column in the first row
for (let j = 0; j <= a.length; j++) {
matrix[0][j] = j;
}
// Fill in the rest of the matrix
for (let i = 1; i <= b.length; i++) {
for (let j = 1; j <= a.length; j++) {
if (b.charAt(i - 1) === a.charAt(j - 1)) {
matrix[i][j] = matrix[i - 1][j - 1];
} else {
matrix[i][j] = Math.min(
matrix[i - 1][j - 1] + 1, // substitution
Math.min(
matrix[i][j - 1] + 1, // insertion
matrix[i - 1][j] + 1 // deletion
)
);
}
}
}
debug("Levenshtein distance:", matrix[b.length][a.length]);
return matrix[b.length][a.length];
}
function calculateSimilarityPercentage(distance, maxLength) {
const similarity = ((maxLength - distance) / maxLength) * 100;
debug("Similarity percentage:", similarity.toFixed(2) + "%");
return Math.round(similarity) + '%';
}
function getDuplicateInfo() {
debug("Getting duplicate info");
const duplicates = [];
const urlMap = new Map();
const referenceItems = Array.from(referencesList.children);
debug("Number of reference items:", referenceItems.length);
referenceItems.forEach((item, index) => {
if (item.tagName.toLowerCase() === 'li') {
const refId = item.id;
const refNumber = index + 1;
debug(`Processing reference item ${refNumber} (${refId})`);
// Get the visible text of the entire reference item
const refText = getVisibleText(item);
debug(` Reference text: ${refText}`);
// Find the first valid link in the reference
const links = item.querySelectorAll('a');
let validLink = null;
for (let link of links) {
const url = link.href;
// Skip this reference if the URL doesn't contain 'http'
if (!url.includes('http')) {
debug(` Skipping reference ${refNumber} - URL does not contain 'http'`);
return; // This 'return' is equivalent to 'continue' in a regular for loop
}
const linkText = link.textContent.trim();
if (
// (!url.includes("wikipedia.org/wiki/") || url.includes("Special:BookSources")) &&
linkText !== "Archived" &&
!url.includes("wikipedia.org") &&
!url.includes("_(identifier)") && // Templates like ISBN and ISSN and OCLC and S2CID contain (identifier)
!url.startsWith("https://www.bbc.co.uk/news/live/") && // BBC live articles get frequent updates
!url.startsWith("https://books.google.com/") && //may be 2 different pages of the same book
!url.startsWith("https://archive.org/details/isbn_")
) {
validLink = link;
debug(` Valid link found: ${url}`);
break;
}
}
if (validLink) {
const url = validLink.href;
if (urlMap.has(url)) {
urlMap.get(url).push({id: refId, number: refNumber, text: refText});
debug(` Duplicate found for URL: ${url}`);
} else {
urlMap.set(url, [{id: refId, number: refNumber, text: refText}]);
debug(` New URL added to map: ${url}`);
}
} else {
debug(` No valid link found in this item`);
}
}
});
urlMap.forEach((refs, url) => {
if (refs.length > 1) {
// Calculate Levenshtein distance for each pair of refs
for (let i = 0; i < refs.length - 1; i++) {
for (let j = i + 1; j < refs.length; j++) {
debug(`Comparing references ${refs[i].number} and ${refs[j].number}:`);
const distance = calculateLevenshteinDistance(refs[i].text, refs[j].text);
const maxLength = Math.max(refs[i].text.length, refs[j].text.length);
const similarity = calculateSimilarityPercentage(distance, maxLength);
refs[i].similarity = refs[i].similarity || {};
refs[i].similarity[refs[j].id] = similarity;
}
}
duplicates.push({ url, refs });
}
});
debug("Number of duplicate sets found:", duplicates.length);
debug("Duplicate sets:", duplicates);
return duplicates;
}
function createCollapsibleTable(duplicateInfo) {
const table = document.createElement('table');
table.className = 'wikitable mw-collapsible duplicate-references-table';
table.setAttribute('role', 'presentation');
const tbody = document.createElement('tbody');
table.appendChild(tbody);
const headerRow = document.createElement('tr');
const headerCell = document.createElement('td');
headerCell.innerHTML = '<strong>Duplicate References</strong>';
const toggleSpan = document.createElement('span');
toggleSpan.className = 'mw-collapsible-toggle';
toggleSpan.innerHTML = '[<a href="#" class="mw-collapsible-text">hide</a>]';
headerCell.appendChild(toggleSpan);
headerRow.appendChild(headerCell);
tbody.appendChild(headerRow);
const pageTitle = mw.config.get('wgPageName').replace(/_/g, ' ');
duplicateInfo.forEach(({ url, refs }) => {
const row = document.createElement('tr');
const cell = document.createElement('td');
// Create report icon
const reportIcon = document.createElement('a');
reportIcon.href = `https://en.wikipedia.org/wiki/User_talk:Polygnotus?action=edit§ion=new&preloadtitle=Reporting%20%5B%5BUser%3APolygnotus%2FDuplicateReferences%7CDuplicateReferences%5D%5D%20false-positive&preload=User:Polygnotus/$1&preloadparams%5b%5d=${encodeURIComponent(`[[${pageTitle}]] ${url}`)}%20~~~~`;
reportIcon.innerHTML = '<img src="https://upload.wikimedia.org/wikipedia/commons/thumb/e/ef/Cross_CSS_Red.svg/15px-Cross_CSS_Red.svg.png" width="15" height="15" alt="Report false positive" title="Report false positive" />';
reportIcon.style.marginRight = '5px';
cell.appendChild(reportIcon);
let urlLink = document.createElement('a');
urlLink.href = url;
urlLink.textContent = url;
urlLink.target = "_blank";
urlLink.rel = "noopener noreferrer";
cell.appendChild(urlLink);
cell.appendChild(document.createTextNode(' in refs: '));
const originalRef = refs[0];
refs.forEach((ref, index) => {
let link = document.createElement('a');
link.href = `#${ref.id}`;
link.textContent = ref.number;
cell.appendChild(link);
// Add similarity information
if (index > 0) {
const similarity = calculateSimilarityPercentage(
calculateLevenshteinDistance(originalRef.text, ref.text),
Math.max(originalRef.text.length, ref.text.length)
);
let similarityInfo = document.createElement('span');
similarityInfo.textContent = ` (${similarity})`;
cell.appendChild(similarityInfo);
}
link.addEventListener('mouseover', () => {
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
if (r.id === ref.id) {
citationElement.classList.add('duplicate-citation-hover');
} else {
citationElement.classList.add('duplicate-citation-highlight');
}
}
});
});
link.addEventListener('mouseout', () => {
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
citationElement.classList.remove('duplicate-citation-hover');
citationElement.classList.remove('duplicate-citation-highlight');
}
});
});
link.addEventListener('click', () => {
document.querySelectorAll('.duplicate-citation-clicked').forEach(el => {
el.classList.remove('duplicate-citation-clicked');
});
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
citationElement.classList.add('duplicate-citation-clicked');
}
});
});
if (index < refs.length - 1) {
cell.appendChild(document.createTextNode(', '));
}
});
row.appendChild(cell);
tbody.appendChild(row);
});
return table;
}
function checkDuplicateReferenceLinks() {
debug("Checking for duplicate reference links");
const duplicateInfo = getDuplicateInfo();
if (duplicateInfo.length > 0) {
debug("Duplicates found, creating collapsible table");
if (document.querySelector('table.box-Duplicated_citations') === null) {
const editSections = containerDiv.querySelectorAll('span.mw-editsection');
editSections.forEach(editSection => {
let spanBefore = document.createElement('span');
spanBefore.className = 'mw-editsection-bracket';
spanBefore.textContent = '[';
let addTemplateLink = document.createElement('a');
addTemplateLink.textContent = ' add {{duplicated citations}} ';
addTemplateLink.href = '#';
addTemplateLink.addEventListener('click', function(e) {
e.preventDefault();
addDuplicateCitationsTemplate();
});
let spanAfter = document.createElement('span');
spanAfter.className = 'mw-editsection-bracket';
spanAfter.textContent = ']';
editSection.appendChild(spanBefore);
editSection.appendChild(addTemplateLink);
editSection.appendChild(spanAfter);
});
}
const table = createCollapsibleTable(duplicateInfo);
containerDiv.after(table);
// Set up collapsible functionality
const toggleLink = table.querySelector('.mw-collapsible-toggle a');
const tableBody = $(table).find('tr:not(:first-child)');
const storageKey = 'duplicateReferencesTableState';
function setTableState(isCollapsed) {
if (isCollapsed) {
tableBody.hide();
toggleLink.textContent = 'show';
} else {
tableBody.show();
toggleLink.textContent = 'hide';
}
localStorage.setItem(storageKey, isCollapsed);
}
// Initialize state from localStorage
const initialState = localStorage.getItem(storageKey) === 'true';
setTableState(initialState);
toggleLink.addEventListener('click', function(e) {
e.preventDefault();
const isCurrentlyCollapsed = tableBody.is(':hidden');
setTableState(!isCurrentlyCollapsed);
});
} else {
debug("No duplicates found");
}
}
checkDuplicateReferenceLinks();
debug("Script execution completed");
});
});
// </nowiki>