Difference between revisions of "User:RoboJeb/Filters"

Revision as of 04:29, 20 October 2012

Filters	Talk	Blacklist	Requests	Documentation	Source	Edits

• These are scripts RoboJeb runs every 5 minutes against any matching pages.
• To disable a script, enclose it in triple quotation marks ("""this would be disabled""" or """Line one
Line two""")
• If you want a script added, talk to N3X15 (talk)!

File filters

Mark all -t (small) images as not needed

def markThumbnailedImages(content, article, **kwargs):
    title = u(article.title).lower()
    # Strip file extension
    end=-4
    if title[-5:]==u'.jpeg':
        end=-5
    if(title[:end][-2:]==u'-t'):
        template=u'{{NoThumbnailsPlease}}'
        if template in content:
            return content # Already messed with this one.
        else:
            return content+template
    return content # We aren't needed here.
addFileFilter(markThumbnailedImages)

Crush all PNG/JPG images

Borrowed from User:CrushBot on TF2Wiki.

Changes made from the original code:

• Removed urlQuery from File.download, as RoboJeb uses wikitools trunk.
• Use File.upload instead of uploadFile. uploadFile doesn't work on this version of MediaWiki.
• Report errors to console.

class imageCrushFilter:
    def __init__(self):
        self.minRatio = 10 # Compression ratio threshold
        self.minByteDiff = 2048 # Byte difference threshold
        self.jpgScanMap = u'0:   0  0 0 0 ;1 2: 0  0 0 0 ;0:   1  8 0 2 ;1:   1  8 0 0 ;2:   1  8 0 0 ;0:   9 63 0 2 ;0:   1 63 2 1 ;0:   1 63 1 0 ;1:   9 63 0 0 ;2:   9 63 0 0 ;'.replace(u';', u';\n')
        self.filterName = 'Saved crush information'
        self.extractHash = compileRegex(r'\{\{(?:png)?crush\s*\|\s*(\w+?)\s*\|\s*(\w+?)\s*}}')
        # pngout has problems sometimes, but it works, at least.
        if programExists('pngout'):
            self.pngenabled = 'pngout'
        elif programExists('pngcrush'):
            self.pngenabled = 'pngcrush'
        else:
            print 'Warning: PNGOut and PNGCrush are not installed or are not in $PATH'
            self.pngenabled = None
        self.jpgenabled = programExists('jpegtran')
        if not self.jpgenabled:
            print 'Warning: jpegtran is not installed or not in $PATH'
    def __call__(self, content, article, **kwargs):
        title = u(article.title).lower()
        if title[-4:] == '.png':
            isPNG = True
            if self.pngenabled is None:
                return content
        elif title[-5:] == '.jpeg' or title[-4:] == '.jpg':
            isPNG = False
            if not self.jpgenabled:
                return content
        else:
            return content
        try: # This is a high-risk filter, lots of I/O, so wrap it in a big try
            filePage = wikitools.wikifile.File(wiki(), article.title)
            hashes = [u, u]
            hashResult = self.extractHash.search(content)
            hashes = None
            hashTemplate = None
            if hashResult:
                hashes = (u(hashResult.group(1)).lower(), u(hashResult.group(2)).lower())
                hashTemplate = u'{{crush|' + hashes[0] + u'|' + hashes[1] + u'}}'
            tempFile = getTempFilename(extension='png')
            filePage.download(location=tempFile) #, urlQuery=u(getRandBits()))
            oldHash = getFileHash(tempFile)
            if hashes is not None and oldHash in hashes:
                return content # Already worked on that one
            hashTemplate = u'{{crush|' + oldHash + u'|None}}'
            tempOutput = getTempFilename(extension='png')
            if isPNG:
                if self.pngenabled == 'pngout':
                    shutil.copyfile(tempFile, tempOutput)
                    result = subprocess.call(['pngout', '-b256', '-y', tempOutput])
                    if result == 0:
                        possibleBlocks = ['0', '64', '128', '192', '256', '512', '1024', '2048']
                        currentUpperBlock = 4
                        currentLowerBlock = 4
                        tryUpper = True
                        tryLower = True
                        while tryUpper or tryLower:
                            if tryUpper:
                                currentUpperBlock += 1
                                upperResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentUpperBlock], '-y', tempOutput])
                                if upperResult != 0 or currentUpperBlock >= len(possibleBlocks) - 1:
                                    tryUpper = False
                            if tryLower:
                                currentLowerBlock -= 1
                                lowerResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentLowerBlock], '-y', tempOutput])
                                if lowerResult != 0 or currentLowerBlock <= 0:
                                    tryLower = False
                elif self.pngenabled == 'pngcrush':
                    result = subprocess.call(['pngcrush', '-q', '-l', '9', '-reduce', '-rem', 'gAMA', '-rem', 'cHRM', '-rem', 'iCCP', '-rem', 'sRGB'] + [i for l in [('-m', str(i)) for i in range(138)] for i in l] + [tempFile, tempOutput])
            else:
                result = subprocess.call(['jpegtran', '-o', '-copy', 'none', '-progressive', '-outfile', tempOutput, tempFile])
            oldSize = os.path.getsize(tempFile)
            newSize = os.path.getsize(tempOutput)
            deleteFile(tempFile)
            if not result and oldSize > newSize:
                # Ready to upload... or are we?
                ratio = int(round(100 * (1.0 - float(newSize) / float(oldSize))))
                if ratio >= self.minRatio or oldSize - newSize >= self.minByteDiff:
                    newHash = getFileHash(tempOutput)
                    if hashes is not None and newHash in hashes:
                        deleteFile(tempOutput)
                        return content # Already got that result, no need to reupload
                    hashTemplate = u'{{crush|' + oldHash + u'|' + newHash + u'}}'
                    content2 = wikitools.wikifile.File(wiki(), article.title).getWikiText()
                    hashResult2 = self.extractHash.search(content2)
                    hashes2 = None
                    if hashResult2:
                        hashes2 = (u(hashResult2.group(1)), u(hashResult2.group(2)))
                    if hashes == hashes2:
                        with open(tempOutput, 'rb') as tempOutputHandle:
                            wikitools.wikifile.File(wiki(), article.title).upload(tempOutputHandle, u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', ignorewarnings=True)
                            #uploadFile(tempOutput, u(article.title), u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', overwrite=True, reupload=True)
                    hashes = (oldHash, newHash)
            if hashResult:
                content = content[:hashResult.start()] + hashTemplate + content[hashResult.end():]
            else:
                content = content.strip() + u'\n\n' + hashTemplate
            deleteFile(tempOutput)
        except Exception as e:
            print('Error: {0}'.format(str(e)))
            pass # Well, that didn't work
        return content
addFileFilter(imageCrushFilter())

Replacements

Moved Links

• Suggest links at Project:Request Move!

# Borrowed from WindBOT
def movedLinks(link, **kwargs):
   movedLinks = {
       # Original       Replacement
       u'S.A.S':        u'SAS',
       u'ASAS':         u'SAS',
       u'TT-38K Radial Decoupler (Flat)': u'Hydraulic Detachment Manifold',
       u'Basic Rocket Design': u'Tutorial:Basic Rocket Design',
   }
   if link.getType() == u'internal' and link.getLink() in movedLinks:
       link.setLink(movedLinks[link.getLink()])
   return link
addLinkFilter(movedLinks)

Moved URLs

def movedURLs(link, **kwargs):
    movedLinks = {
        'Forums moved to new subdomain': [compileRegex(r'^http://kerbalspaceprogram.com/forum(.*)$'),u'http://forum.kerbalspaceprogram.com\g<1>']
    }
    if link.getType() == u'external':
        for reason in movedLinks:
            if movedLinks[reason][0].search(link.getLink()):
                link.setLink(movedLinks[reason][0].sub(movedLinks[reason][1],link.getLink()))
    return link
addLinkFilter(movedURLs)