User:RoboJeb/Filters
From Kerbal Space Program Wiki
Contents
File filters
Mark all -t (small) images as not needed
def imageThumbnailedImages(content, article, **kwargs):
title = u(t.title).lower()
# Strip file extension
end=-4
if title[-5:]==u'.jpeg':
end=-5
if(title[:end][-2:]==u'-t'):
template=u'{{NoThumbnailsPlease}}'
if template in content:
return content # Already messed with this one.
else:
return content+template
return content # We aren't needed here.
Crush all PNG/JPG images
# Borrowed from User:CrushBot on TF2Wiki.
class imageCrushFilter:
def __init__(self):
self.minRatio = 10 # Compression ratio threshold
self.minByteDiff = 2048 # Byte difference threshold
self.jpgScanMap = u'0: 0 0 0 0 ;1 2: 0 0 0 0 ;0: 1 8 0 2 ;1: 1 8 0 0 ;2: 1 8 0 0 ;0: 9 63 0 2 ;0: 1 63 2 1 ;0: 1 63 1 0 ;1: 9 63 0 0 ;2: 9 63 0 0 ;'.replace(u';', u';\n')
self.filterName = 'Saved crush information'
self.extractHash = compileRegex(r'\{\{(?:png)?crush\s*\|\s*(\w+?)\s*\|\s*(\w+?)\s*}}')
if programExists('pngout'):
self.pngenabled = 'pngout'
elif programExists('pngcrush'):
self.pngenabled = 'pngcrush'
else:
print 'Warning: PNGOut and PNGCrush are not installed or are not in $PATH'
self.pngenabled = None
self.jpgenabled = programExists('jpegtran')
if not self.jpgenabled:
print 'Warning: jpegtran is not installed or not in $PATH'
def __call__(self, content, article, **kwargs):
title = u(article.title).lower()
if title[-4:] == '.png':
isPNG = True
if self.pngenabled is None:
return content
elif title[-5:] == '.jpeg' or title[-4:] == '.jpg':
isPNG = False
if not self.jpgenabled:
return content
else:
return content
try: # This is a high-risk filter, lots of I/O, so wrap it in a big try
filePage = wikitools.wikifile.File(wiki(), article.title)
hashes = [u, u]
hashResult = self.extractHash.search(content)
hashes = None
hashTemplate = None
if hashResult:
hashes = (u(hashResult.group(1)).lower(), u(hashResult.group(2)).lower())
hashTemplate = u'{{crush|' + hashes[0] + u'|' + hashes[1] + u'}}'
tempFile = getTempFilename(extension='png')
filePage.download(location=tempFile, urlQuery=u(getRandBits()))
oldHash = getFileHash(tempFile)
if hashes is not None and oldHash in hashes:
return content # Already worked on that one
hashTemplate = u'{{crush|' + oldHash + u'|None}}'
tempOutput = getTempFilename(extension='png')
if isPNG:
if self.pngenabled == 'pngout':
shutil.copyfile(tempFile, tempOutput)
result = subprocess.call(['pngout', '-b256', '-y', tempOutput])
if result == 0:
possibleBlocks = ['0', '64', '128', '192', '256', '512', '1024', '2048']
currentUpperBlock = 4
currentLowerBlock = 4
tryUpper = True
tryLower = True
while tryUpper or tryLower:
if tryUpper:
currentUpperBlock += 1
upperResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentUpperBlock], '-y', tempOutput])
if upperResult != 0 or currentUpperBlock >= len(possibleBlocks) - 1:
tryUpper = False
if tryLower:
currentLowerBlock -= 1
lowerResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentLowerBlock], '-y', tempOutput])
if lowerResult != 0 or currentLowerBlock <= 0:
tryLower = False
elif self.pngenabled == 'pngcrush':
result = subprocess.call(['pngcrush', '-q', '-l', '9', '-reduce', '-rem', 'gAMA', '-rem', 'cHRM', '-rem', 'iCCP', '-rem', 'sRGB'] + [i for l in [('-m', str(i)) for i in range(138)] for i in l] + [tempFile, tempOutput])
else:
result = subprocess.call(['jpegtran', '-o', '-copy', 'none', '-progressive', '-outfile', tempOutput, tempFile])
oldSize = os.path.getsize(tempFile)
newSize = os.path.getsize(tempOutput)
deleteFile(tempFile)
if not result and oldSize > newSize:
# Ready to upload... or are we?
ratio = int(round(100 * (1.0 - float(newSize) / float(oldSize))))
if ratio >= self.minRatio or oldSize - newSize >= self.minByteDiff:
newHash = getFileHash(tempOutput)
if hashes is not None and newHash in hashes:
deleteFile(tempOutput)
return content # Already got that result, no need to reupload
hashTemplate = u'{{crush|' + oldHash + u'|' + newHash + u'}}'
content2 = wikitools.wikifile.File(wiki(), article.title).getWikiText()
hashResult2 = self.extractHash.search(content2)
hashes2 = None
if hashResult2:
hashes2 = (u(hashResult2.group(1)), u(hashResult2.group(2)))
if hashes == hashes2:
uploadFile(tempOutput, u(article.title), u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', overwrite=True, reupload=True)
hashes = (oldHash, newHash)
if hashResult:
content = content[:hashResult.start()] + hashTemplate + content[hashResult.end():]
else:
content = content.strip() + u'\n\n' + hashTemplate
deleteFile(tempOutput)
except:
pass # Well, that didn't work
return content
addFileFilter(imageCrushFilter())
Replacements
Moved Links
# Borrowed from WindBOT
def movedLinks(link, **kwargs):
movedLinks = {
# Original Original
u'S.A.S': u'SAS',
u'ASAS': u'SAS'
}
if link.getType() == u'internal' and link.getLink() in movedLinks:
link.setLink(movedLinks[link.getLink()])
return link
addLinkFilter(movedLinks)
