Difference between revisions of "User:RoboJeb/Filters"

From Kerbal Space Program Wiki
Jump to: navigation, search
m (Mark all -t (small) images as not needed)
(Mark all -t (small) images as not needed)
 
(23 intermediate revisions by the same user not shown)
Line 1: Line 1:
 
{{User:RoboJeb/Header}}
 
{{User:RoboJeb/Header}}
== What? ==
+
 
 
* These are scripts RoboJeb runs every 5 minutes against any matching pages.
 
* These are scripts RoboJeb runs every 5 minutes against any matching pages.
 
* To disable a script, enclose it in triple quotation marks (<code>"""this would be disabled"""</code> or <code>"""Line one<br />Line two"""</code>)
 
* To disable a script, enclose it in triple quotation marks (<code>"""this would be disabled"""</code> or <code>"""Line one<br />Line two"""</code>)
Line 9: Line 9:
 
=== Mark all -t (small) images as not needed ===
 
=== Mark all -t (small) images as not needed ===
 
  def markThumbnailedImages(content, article, **kwargs):
 
  def markThumbnailedImages(content, article, **kwargs):
     title = u(t.title).lower()
+
     title = u(article.title).lower()
 
     # Strip file extension
 
     # Strip file extension
 
     end=-4
 
     end=-4
 
     if title[-5:]==u'.jpeg':
 
     if title[-5:]==u'.jpeg':
 
         end=-5
 
         end=-5
     if(title[:end][-2:]==u'-t'):
+
     ext = file[end:]
 +
    toosmall=False
 +
    if ext in ('.jpg','.png','.gif','.jpeg'):
 +
        try:
 +
            filePage = wikitools.wikifile.File(wiki(), article.title)
 +
            tempFile = getTempFilename(extension=ext[1:])
 +
            filePage.download(location=tempFile)
 +
            im = Image.open(tempFile)
 +
            width, height = img.size
 +
            if width <= 65 and height <= 65:
 +
                toosmall=True
 +
            deleteFile(tempFile)
 +
        except Exception as e:
 +
            print('Error: {0}'.format(str(e)))
 +
            pass # Well, that didn't work
 +
    if toosmall or title[:end][-2:]==u'-t':
 
         template=<nowiki>u'{{NoThumbnailsPlease}}'</nowiki>
 
         template=<nowiki>u'{{NoThumbnailsPlease}}'</nowiki>
 
         if template in content:
 
         if template in content:
Line 24: Line 39:
  
 
=== Crush all PNG/JPG images ===
 
=== Crush all PNG/JPG images ===
# Borrowed from User:CrushBot on TF2Wiki.
+
Borrowed from User:CrushBot on TF2Wiki.
 +
 
 +
Changes made from the original code:
 +
* Removed urlQuery from File.download, as RoboJeb uses wikitools trunk.
 +
* Use File.upload instead of uploadFile. uploadFile doesn't work on this version of MediaWiki.
 +
* Report errors to console.
 
  class imageCrushFilter:
 
  class imageCrushFilter:
 
     def __init__(self):
 
     def __init__(self):
Line 32: Line 52:
 
         self.filterName = 'Saved crush information'
 
         self.filterName = 'Saved crush information'
 
         self.extractHash = compileRegex(r'\{\{(?:png)?crush\s*\|\s*(\w+?)\s*\|\s*(\w+?)\s*}}')
 
         self.extractHash = compileRegex(r'\{\{(?:png)?crush\s*\|\s*(\w+?)\s*\|\s*(\w+?)\s*}}')
 +
        # pngout has problems sometimes, but it works, at least.
 
         if programExists('pngout'):
 
         if programExists('pngout'):
 
             self.pngenabled = 'pngout'
 
             self.pngenabled = 'pngout'
Line 64: Line 85:
 
                 hashTemplate = <nowiki>u'{{crush|' + hashes[0] + u'|' + hashes[1] + u'}}'</nowiki>
 
                 hashTemplate = <nowiki>u'{{crush|' + hashes[0] + u'|' + hashes[1] + u'}}'</nowiki>
 
             tempFile = getTempFilename(extension='png')
 
             tempFile = getTempFilename(extension='png')
             filePage.download(location=tempFile, urlQuery=u(getRandBits()))
+
             filePage.download(location=tempFile) #, urlQuery=u(getRandBits()))
 
             oldHash = getFileHash(tempFile)
 
             oldHash = getFileHash(tempFile)
 
             if hashes is not None and oldHash in hashes:
 
             if hashes is not None and oldHash in hashes:
Line 113: Line 134:
 
                         hashes2 = (u(hashResult2.group(1)), u(hashResult2.group(2)))
 
                         hashes2 = (u(hashResult2.group(1)), u(hashResult2.group(2)))
 
                     if hashes == hashes2:
 
                     if hashes == hashes2:
                         uploadFile(tempOutput, u(article.title), u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', overwrite=True, reupload=True)
+
                         with open(tempOutput, 'rb') as tempOutputHandle:
 +
                            wikitools.wikifile.File(wiki(), article.title).upload(tempOutputHandle, u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', ignorewarnings=True)
 +
                            #uploadFile(tempOutput, u(article.title), u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', overwrite=True, reupload=True)
 
                     hashes = (oldHash, newHash)
 
                     hashes = (oldHash, newHash)
 
             if hashResult:
 
             if hashResult:
Line 120: Line 143:
 
                 content = content.strip() + u'\n\n' + hashTemplate
 
                 content = content.strip() + u'\n\n' + hashTemplate
 
             deleteFile(tempOutput)
 
             deleteFile(tempOutput)
         except:
+
         except Exception as e:
 +
            print('Error: {0}'.format(str(e)))
 
             pass # Well, that didn't work
 
             pass # Well, that didn't work
 
         return content
 
         return content
Line 127: Line 151:
 
== Replacements ==
 
== Replacements ==
 
=== Moved Links ===
 
=== Moved Links ===
 +
* Suggest links at [[Project:Request Move]]!
 +
 
  # Borrowed from WindBOT
 
  # Borrowed from WindBOT
 
  def movedLinks(link, **kwargs):
 
  def movedLinks(link, **kwargs):
 
     movedLinks = {
 
     movedLinks = {
         # Original      Original
+
         # Original      Replacement
 
         u'S.A.S':        u'SAS',
 
         u'S.A.S':        u'SAS',
         u'ASAS':        u'SAS'
+
         u'ASAS':        u'SAS',
 +
        u'TT-38K Radial Decoupler (Flat)': u'Hydraulic Detachment Manifold',
 +
        u'Basic Rocket Design': u'Tutorial:Basic Rocket Design',
 +
        u'Illuminator mk1': u'Illuminator Mk1',
 +
        u'Illuminator mk2': u'Illuminator Mk2',
 +
        u'OX-STAT Photovoltaic panels': u'OX-STAT Photovoltaic Panels',
 +
        u'OX-STAT Photovolatic panels/Box': u'OX-STAT Photovoltaic Panels/Box',
 +
        u'OX-4B Photovoltaic Panels': u'OX-4L 1x6 Photovoltaic Panels',
 +
        u'OX-4 Photovoltaic Panels': u'OX-4W 2x3 Photovoltaic Panels'
 
     }
 
     }
 
     if link.getType() == u'internal' and link.getLink() in movedLinks:
 
     if link.getType() == u'internal' and link.getLink() in movedLinks:
Line 138: Line 172:
 
     return link
 
     return link
 
  addLinkFilter(movedLinks)
 
  addLinkFilter(movedLinks)
 +
 +
=== Moved URLs ===
 +
def movedURLs(link, **kwargs):
 +
    movedLinks = {
 +
        'Forums moved to new subdomain': [compileRegex(r'^http:<nowiki>//</nowiki>kerbalspaceprogram.com/forum(.*)$'),u'http:<nowiki>//</nowiki>forum.kerbalspaceprogram.com\g<1>'],
 +
        'Wiki moved to new subdomain (a while ago)': [compileRegex(r'^http:<nowiki>//</nowiki>kspwiki.nexisonline.net/wiki(.*)$'),u'http:<nowiki>//</nowiki>wiki.kerbalspaceprogram.com/wiki\g<1>']
 +
    }
 +
    if link.getType() == u'external':
 +
        for reason in movedLinks:
 +
            if movedLinks[reason][0].search(link.getLink()):
 +
                link.setLink(movedLinks[reason][0].sub(movedLinks[reason][1],link.getLink()))
 +
    return link
 +
addLinkFilter(movedURLs)

Latest revision as of 13:20, 24 August 2013

  • These are scripts RoboJeb runs every 5 minutes against any matching pages.
  • To disable a script, enclose it in triple quotation marks ("""this would be disabled""" or """Line one
    Line two"""
    )
  • If you want a script added, talk to N3X15 (talk)!

File filters

Mark all -t (small) images as not needed

def markThumbnailedImages(content, article, **kwargs):
    title = u(article.title).lower()
    # Strip file extension
    end=-4
    if title[-5:]==u'.jpeg':
        end=-5
    ext = file[end:]
    toosmall=False
    if ext in ('.jpg','.png','.gif','.jpeg'):
        try:
            filePage = wikitools.wikifile.File(wiki(), article.title)
            tempFile = getTempFilename(extension=ext[1:])
            filePage.download(location=tempFile)
            im = Image.open(tempFile)
            width, height = img.size
            if width <= 65 and height <= 65:
                toosmall=True
            deleteFile(tempFile)
        except Exception as e:
            print('Error: {0}'.format(str(e)))
            pass # Well, that didn't work
    if toosmall or title[:end][-2:]==u'-t':
        template=u'{{NoThumbnailsPlease}}'
        if template in content:
            return content # Already messed with this one.
        else:
            return content+template
    return content # We aren't needed here.
addFileFilter(markThumbnailedImages)

Crush all PNG/JPG images

Borrowed from User:CrushBot on TF2Wiki.

Changes made from the original code:

  • Removed urlQuery from File.download, as RoboJeb uses wikitools trunk.
  • Use File.upload instead of uploadFile. uploadFile doesn't work on this version of MediaWiki.
  • Report errors to console.
class imageCrushFilter:
    def __init__(self):
        self.minRatio = 10 # Compression ratio threshold
        self.minByteDiff = 2048 # Byte difference threshold
        self.jpgScanMap = u'0:   0  0 0 0 ;1 2: 0  0 0 0 ;0:   1  8 0 2 ;1:   1  8 0 0 ;2:   1  8 0 0 ;0:   9 63 0 2 ;0:   1 63 2 1 ;0:   1 63 1 0 ;1:   9 63 0 0 ;2:   9 63 0 0 ;'.replace(u';', u';\n')
        self.filterName = 'Saved crush information'
        self.extractHash = compileRegex(r'\{\{(?:png)?crush\s*\|\s*(\w+?)\s*\|\s*(\w+?)\s*}}')
        # pngout has problems sometimes, but it works, at least.
        if programExists('pngout'):
            self.pngenabled = 'pngout'
        elif programExists('pngcrush'):
            self.pngenabled = 'pngcrush'
        else:
            print 'Warning: PNGOut and PNGCrush are not installed or are not in $PATH'
            self.pngenabled = None
        self.jpgenabled = programExists('jpegtran')
        if not self.jpgenabled:
            print 'Warning: jpegtran is not installed or not in $PATH'
    def __call__(self, content, article, **kwargs):
        title = u(article.title).lower()
        if title[-4:] == '.png':
            isPNG = True
            if self.pngenabled is None:
                return content
        elif title[-5:] == '.jpeg' or title[-4:] == '.jpg':
            isPNG = False
            if not self.jpgenabled:
                return content
        else:
            return content
        try: # This is a high-risk filter, lots of I/O, so wrap it in a big try
            filePage = wikitools.wikifile.File(wiki(), article.title)
            hashes = [u, u]
            hashResult = self.extractHash.search(content)
            hashes = None
            hashTemplate = None
            if hashResult:
                hashes = (u(hashResult.group(1)).lower(), u(hashResult.group(2)).lower())
                hashTemplate = u'{{crush|' + hashes[0] + u'|' + hashes[1] + u'}}'
            tempFile = getTempFilename(extension='png')
            filePage.download(location=tempFile) #, urlQuery=u(getRandBits()))
            oldHash = getFileHash(tempFile)
            if hashes is not None and oldHash in hashes:
                return content # Already worked on that one
            hashTemplate = u'{{crush|' + oldHash + u'|None}}'
            tempOutput = getTempFilename(extension='png')
            if isPNG:
                if self.pngenabled == 'pngout':
                    shutil.copyfile(tempFile, tempOutput)
                    result = subprocess.call(['pngout', '-b256', '-y', tempOutput])
                    if result == 0:
                        possibleBlocks = ['0', '64', '128', '192', '256', '512', '1024', '2048']
                        currentUpperBlock = 4
                        currentLowerBlock = 4
                        tryUpper = True
                        tryLower = True
                        while tryUpper or tryLower:
                            if tryUpper:
                                currentUpperBlock += 1
                                upperResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentUpperBlock], '-y', tempOutput])
                                if upperResult != 0 or currentUpperBlock >= len(possibleBlocks) - 1:
                                    tryUpper = False
                            if tryLower:
                                currentLowerBlock -= 1
                                lowerResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentLowerBlock], '-y', tempOutput])
                                if lowerResult != 0 or currentLowerBlock <= 0:
                                    tryLower = False
                elif self.pngenabled == 'pngcrush':
                    result = subprocess.call(['pngcrush', '-q', '-l', '9', '-reduce', '-rem', 'gAMA', '-rem', 'cHRM', '-rem', 'iCCP', '-rem', 'sRGB'] + [i for l in [('-m', str(i)) for i in range(138)] for i in l] + [tempFile, tempOutput])
            else:
                result = subprocess.call(['jpegtran', '-o', '-copy', 'none', '-progressive', '-outfile', tempOutput, tempFile])
            oldSize = os.path.getsize(tempFile)
            newSize = os.path.getsize(tempOutput)
            deleteFile(tempFile)
            if not result and oldSize > newSize:
                # Ready to upload... or are we?
                ratio = int(round(100 * (1.0 - float(newSize) / float(oldSize))))
                if ratio >= self.minRatio or oldSize - newSize >= self.minByteDiff:
                    newHash = getFileHash(tempOutput)
                    if hashes is not None and newHash in hashes:
                        deleteFile(tempOutput)
                        return content # Already got that result, no need to reupload
                    hashTemplate = u'{{crush|' + oldHash + u'|' + newHash + u'}}'
                    content2 = wikitools.wikifile.File(wiki(), article.title).getWikiText()
                    hashResult2 = self.extractHash.search(content2)
                    hashes2 = None
                    if hashResult2:
                        hashes2 = (u(hashResult2.group(1)), u(hashResult2.group(2)))
                    if hashes == hashes2:
                        with open(tempOutput, 'rb') as tempOutputHandle:
                            wikitools.wikifile.File(wiki(), article.title).upload(tempOutputHandle, u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', ignorewarnings=True)
                            #uploadFile(tempOutput, u(article.title), u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', overwrite=True, reupload=True)
                    hashes = (oldHash, newHash)
            if hashResult:
                content = content[:hashResult.start()] + hashTemplate + content[hashResult.end():]
            else:
                content = content.strip() + u'\n\n' + hashTemplate
            deleteFile(tempOutput)
        except Exception as e:
            print('Error: {0}'.format(str(e)))
            pass # Well, that didn't work
        return content
addFileFilter(imageCrushFilter())

Replacements

Moved Links

# Borrowed from WindBOT
def movedLinks(link, **kwargs):
   movedLinks = {
       # Original       Replacement
       u'S.A.S':        u'SAS',
       u'ASAS':         u'SAS',
       u'TT-38K Radial Decoupler (Flat)': u'Hydraulic Detachment Manifold',
       u'Basic Rocket Design': u'Tutorial:Basic Rocket Design',
       u'Illuminator mk1': u'Illuminator Mk1',
       u'Illuminator mk2': u'Illuminator Mk2',
       u'OX-STAT Photovoltaic panels': u'OX-STAT Photovoltaic Panels',
       u'OX-STAT Photovolatic panels/Box': u'OX-STAT Photovoltaic Panels/Box',
       u'OX-4B Photovoltaic Panels': u'OX-4L 1x6 Photovoltaic Panels',
       u'OX-4 Photovoltaic Panels': u'OX-4W 2x3 Photovoltaic Panels'
   }
   if link.getType() == u'internal' and link.getLink() in movedLinks:
       link.setLink(movedLinks[link.getLink()])
   return link
addLinkFilter(movedLinks)

Moved URLs

def movedURLs(link, **kwargs):
    movedLinks = {
        'Forums moved to new subdomain': [compileRegex(r'^http://kerbalspaceprogram.com/forum(.*)$'),u'http://forum.kerbalspaceprogram.com\g<1>'],
        'Wiki moved to new subdomain (a while ago)': [compileRegex(r'^http://kspwiki.nexisonline.net/wiki(.*)$'),u'http://wiki.kerbalspaceprogram.com/wiki\g<1>']
    }
    if link.getType() == u'external':
        for reason in movedLinks:
            if movedLinks[reason][0].search(link.getLink()):
                link.setLink(movedLinks[reason][0].sub(movedLinks[reason][1],link.getLink()))
    return link
addLinkFilter(movedURLs)