User:RoboJeb/Filters
From Kerbal Space Program Wiki
- These are scripts RoboJeb runs every 5 minutes against any matching pages.
- To disable a script, enclose it in triple quotation marks (
"""this would be disabled"""or"""Line one)
Line two""" - If you want a script added, talk to N3X15 (talk)!
Contents
File filters
Mark all -t (small) images as not needed
def markThumbnailedImages(content, article, **kwargs):
title = u(article.title).lower()
# Strip file extension
end=-4
if title[-5:]==u'.jpeg':
end=-5
ext = file[end:]
toosmall=False
if ext in ('.jpg','.png','.gif','.jpeg'):
filePage = wikitools.wikifile.File(wiki(), article.title)
tempFile = getTempFilename(extension=ext[1:])
filePage.download(location=tempFile)
im = Image.open(tempFile)
width, height = img.size
if width <= 65 and height <= 65:
toosmall=True
deleteFile(tempFile)
if toosmall or title[:end][-2:]==u'-t':
template=u'{{NoThumbnailsPlease}}'
if template in content:
return content # Already messed with this one.
else:
return content+template
return content # We aren't needed here.
addFileFilter(markThumbnailedImages)
Crush all PNG/JPG images
Borrowed from User:CrushBot on TF2Wiki.
Changes made from the original code:
- Removed urlQuery from File.download, as RoboJeb uses wikitools trunk.
- Use File.upload instead of uploadFile. uploadFile doesn't work on this version of MediaWiki.
- Report errors to console.
class imageCrushFilter:
def __init__(self):
self.minRatio = 10 # Compression ratio threshold
self.minByteDiff = 2048 # Byte difference threshold
self.jpgScanMap = u'0: 0 0 0 0 ;1 2: 0 0 0 0 ;0: 1 8 0 2 ;1: 1 8 0 0 ;2: 1 8 0 0 ;0: 9 63 0 2 ;0: 1 63 2 1 ;0: 1 63 1 0 ;1: 9 63 0 0 ;2: 9 63 0 0 ;'.replace(u';', u';\n')
self.filterName = 'Saved crush information'
self.extractHash = compileRegex(r'\{\{(?:png)?crush\s*\|\s*(\w+?)\s*\|\s*(\w+?)\s*}}')
# pngout has problems sometimes, but it works, at least.
if programExists('pngout'):
self.pngenabled = 'pngout'
elif programExists('pngcrush'):
self.pngenabled = 'pngcrush'
else:
print 'Warning: PNGOut and PNGCrush are not installed or are not in $PATH'
self.pngenabled = None
self.jpgenabled = programExists('jpegtran')
if not self.jpgenabled:
print 'Warning: jpegtran is not installed or not in $PATH'
def __call__(self, content, article, **kwargs):
title = u(article.title).lower()
if title[-4:] == '.png':
isPNG = True
if self.pngenabled is None:
return content
elif title[-5:] == '.jpeg' or title[-4:] == '.jpg':
isPNG = False
if not self.jpgenabled:
return content
else:
return content
try: # This is a high-risk filter, lots of I/O, so wrap it in a big try
filePage = wikitools.wikifile.File(wiki(), article.title)
hashes = [u, u]
hashResult = self.extractHash.search(content)
hashes = None
hashTemplate = None
if hashResult:
hashes = (u(hashResult.group(1)).lower(), u(hashResult.group(2)).lower())
hashTemplate = u'{{crush|' + hashes[0] + u'|' + hashes[1] + u'}}'
tempFile = getTempFilename(extension='png')
filePage.download(location=tempFile) #, urlQuery=u(getRandBits()))
oldHash = getFileHash(tempFile)
if hashes is not None and oldHash in hashes:
return content # Already worked on that one
hashTemplate = u'{{crush|' + oldHash + u'|None}}'
tempOutput = getTempFilename(extension='png')
if isPNG:
if self.pngenabled == 'pngout':
shutil.copyfile(tempFile, tempOutput)
result = subprocess.call(['pngout', '-b256', '-y', tempOutput])
if result == 0:
possibleBlocks = ['0', '64', '128', '192', '256', '512', '1024', '2048']
currentUpperBlock = 4
currentLowerBlock = 4
tryUpper = True
tryLower = True
while tryUpper or tryLower:
if tryUpper:
currentUpperBlock += 1
upperResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentUpperBlock], '-y', tempOutput])
if upperResult != 0 or currentUpperBlock >= len(possibleBlocks) - 1:
tryUpper = False
if tryLower:
currentLowerBlock -= 1
lowerResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentLowerBlock], '-y', tempOutput])
if lowerResult != 0 or currentLowerBlock <= 0:
tryLower = False
elif self.pngenabled == 'pngcrush':
result = subprocess.call(['pngcrush', '-q', '-l', '9', '-reduce', '-rem', 'gAMA', '-rem', 'cHRM', '-rem', 'iCCP', '-rem', 'sRGB'] + [i for l in [('-m', str(i)) for i in range(138)] for i in l] + [tempFile, tempOutput])
else:
result = subprocess.call(['jpegtran', '-o', '-copy', 'none', '-progressive', '-outfile', tempOutput, tempFile])
oldSize = os.path.getsize(tempFile)
newSize = os.path.getsize(tempOutput)
deleteFile(tempFile)
if not result and oldSize > newSize:
# Ready to upload... or are we?
ratio = int(round(100 * (1.0 - float(newSize) / float(oldSize))))
if ratio >= self.minRatio or oldSize - newSize >= self.minByteDiff:
newHash = getFileHash(tempOutput)
if hashes is not None and newHash in hashes:
deleteFile(tempOutput)
return content # Already got that result, no need to reupload
hashTemplate = u'{{crush|' + oldHash + u'|' + newHash + u'}}'
content2 = wikitools.wikifile.File(wiki(), article.title).getWikiText()
hashResult2 = self.extractHash.search(content2)
hashes2 = None
if hashResult2:
hashes2 = (u(hashResult2.group(1)), u(hashResult2.group(2)))
if hashes == hashes2:
with open(tempOutput, 'rb') as tempOutputHandle:
wikitools.wikifile.File(wiki(), article.title).upload(tempOutputHandle, u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', ignorewarnings=True)
#uploadFile(tempOutput, u(article.title), u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', overwrite=True, reupload=True)
hashes = (oldHash, newHash)
if hashResult:
content = content[:hashResult.start()] + hashTemplate + content[hashResult.end():]
else:
content = content.strip() + u'\n\n' + hashTemplate
deleteFile(tempOutput)
except Exception as e:
print('Error: {0}'.format(str(e)))
pass # Well, that didn't work
return content
addFileFilter(imageCrushFilter())
Replacements
Moved Links
- Suggest links at Project:Request Move!
# Borrowed from WindBOT
def movedLinks(link, **kwargs):
movedLinks = {
# Original Replacement
u'S.A.S': u'SAS',
u'ASAS': u'SAS',
u'TT-38K Radial Decoupler (Flat)': u'Hydraulic Detachment Manifold',
u'Basic Rocket Design': u'Tutorial:Basic Rocket Design',
u'Illuminator mk1': u'Illuminator Mk1',
u'Illuminator mk2': u'Illuminator Mk2',
u'OX-STAT Photovoltaic panels': u'OX-STAT Photovoltaic Panels',
u'OX-STAT Photovolatic panels/Box': u'OX-STAT Photovoltaic Panels/Box',
u'OX-4B Photovoltaic Panels': u'OX-4L 1x6 Photovoltaic Panels',
u'OX-4 Photovoltaic Panels': u'OX-4W 2x3 Photovoltaic Panels'
}
if link.getType() == u'internal' and link.getLink() in movedLinks:
link.setLink(movedLinks[link.getLink()])
return link
addLinkFilter(movedLinks)
Moved URLs
def movedURLs(link, **kwargs):
movedLinks = {
'Forums moved to new subdomain': [compileRegex(r'^http://kerbalspaceprogram.com/forum(.*)$'),u'http://forum.kerbalspaceprogram.com\g<1>'],
'Wiki moved to new subdomain (a while ago)': [compileRegex(r'^http://kspwiki.nexisonline.net/wiki(.*)$'),u'http://wiki.kerbalspaceprogram.com/wiki\g<1>']
}
if link.getType() == u'external':
for reason in movedLinks:
if movedLinks[reason][0].search(link.getLink()):
link.setLink(movedLinks[reason][0].sub(movedLinks[reason][1],link.getLink()))
return link
addLinkFilter(movedURLs)
