mdk/plugins.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 Moodle Development Kit
   6
   7 Copyright (c) 2013 Frédéric Massart - FMCorz.net
   8
   9 This program is free software: you can redistribute it and/or modify
  10 it under the terms of the GNU General Public License as published by
  11 the Free Software Foundation, either version 3 of the License, or
  12 (at your option) any later version.
  13
  14 This program is distributed in the hope that it will be useful,
  15 but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 GNU General Public License for more details.
  18
  19 You should have received a copy of the GNU General Public License
  20 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 http://github.com/FMCorz/mdk
  23 """
  24
  25 import os
  26 import json
  27 import httplib
  28 from urllib import urlencode, urlretrieve
  29 import logging
  30 import zipfile
  31 import re
  32 import shutil
  33 from tempfile import gettempdir
  34 from .config import Conf
  35 from . import tools
  36
  37 C = Conf()
  38
  39
  40 class PluginManager(object):
  41
  42     _subSystems = {
  43         'admin': '/{admin}',
  44         'auth': '/auth',
  45         'availability': '/availability',
  46         'backup': '/backup/util/ui',
  47         'badges': '/badges',
  48         'block': '/blocks',
  49         'blog': '/blog',
  50         'cache': '/cache',
  51         'calendar': '/calendar',
  52         'cohort': '/cohort',
  53         'competency': '/competency',
  54         'course': '/course',
  55         'editor': '/lib/editor',
  56         'enrol': '/enrol',
  57         'files': '/files',
  58         'form': '/lib/form',
  59         'grades': '/grade',
  60         'grading': '/grade/grading',
  61         'group': '/group',
  62         'message': '/message',
  63         'mnet': '/mnet',
  64         'my': '/my',
  65         'notes': '/notes',
  66         'plagiarism': '/plagiarism',
  67         'portfolio': '/portfolio',
  68         'publish': '/course/publish',
  69         'question': '/question',
  70         'rating': '/rating',
  71         'register': '/{admin}/registration',
  72         'repository': '/repository',
  73         'rss': '/rss',
  74         'role': '/{admin}/roles',
  75         'search': '/search',
  76         'tag': '/tag',
  77         'user': '/user',
  78         'webservice': '/webservice'
  79     }
  80
  81     _pluginTypesPath = {
  82         'antivirus': '/lib/antivirus',
  83         'availability': '/availability/condition',
  84         'qtype': '/question/type',
  85         'mod': '/mod',
  86         'auth': '/auth',
  87         'calendartype': '/calendar/type',
  88         'enrol': '/enrol',
  89         'message': '/message/output',
  90         'block': '/blocks',
  91         'filter': '/filter',
  92         'editor': '/lib/editor',
  93         'format': '/course/format',
  94         'profilefield': '/user/profile/field',
  95         'report': '/report',
  96         'coursereport': '/course/report',  # Must be after system reports.
  97         'gradeexport': '/grade/export',
  98         'gradeimport': '/grade/import',
  99         'gradereport': '/grade/report',
 100         'gradingform': '/grade/grading/form',
 101         'mnetservice': '/mnet/service',
 102         'webservice': '/webservice',
 103         'repository': '/repository',
 104         'portfolio': '/portfolio',
 105         'search': '/search/engine',
 106         'qbehaviour': '/question/behaviour',
 107         'qformat': '/question/format',
 108         'plagiarism': '/plagiarism',
 109         'tool': '/{admin}/tool',
 110         'cachestore': '/cache/stores',
 111         'cachelock': '/cache/locks',
 112
 113         'theme': '/theme',
 114         'local': '/local'
 115     }
 116     _supportSubtypes = ['mod', 'editor', 'local', 'tool']
 117
 118     @classmethod
 119     def extract(cls, f, plugin, M, override=False):
 120         """Extract a plugin zip file to the plugin directory of M"""
 121
 122         if type(plugin) != PluginObject:
 123             raise ValueError('PluginObject expected')
 124
 125         if not override and cls.hasPlugin(plugin, M):
 126             raise Exception('Plugin directory already exists')
 127
 128         if not cls.validateZipFile(f, plugin.name):
 129             raise Exception('Invalid zip file')
 130
 131         zp = zipfile.ZipFile(f)
 132         try:
 133             logging.info('Extracting plugin...')
 134             rootDir = os.path.commonprefix(zp.namelist())
 135             extractIn = cls.getTypeDirectory(plugin.t, M)
 136             zp.extractall(extractIn)
 137             if plugin.name != rootDir.rstrip('/'):
 138                 orig = os.path.join(extractIn, rootDir).rstrip('/')
 139                 dest = os.path.join(extractIn, plugin.name).rstrip('/')
 140
 141                 # Merge directories
 142                 for src_dir, dirs, files in os.walk(orig):
 143                     dst_dir = src_dir.replace(orig, dest)
 144                     if not os.path.exists(dst_dir):
 145                         os.mkdir(dst_dir)
 146                     for file_ in files:
 147                         src_file = os.path.join(src_dir, file_)
 148                         dst_file = os.path.join(dst_dir, file_)
 149                         if os.path.exists(dst_file):
 150                             os.remove(dst_file)
 151                         shutil.move(src_file, dst_dir)
 152
 153                 shutil.rmtree(orig)
 154
 155         except OSError:
 156             raise Exception('Error while extracting the files')
 157
 158     @classmethod
 159     def getTypeAndName(cls, plugin):
 160         """Accepts a full plugin name 'mod_book' and returns the type and plugin name"""
 161
 162         if plugin == 'moodle' or plugin == 'core' or plugin == '':
 163             return ('core', None)
 164
 165         if not '_' in plugin:
 166             t = 'mod'
 167             name = plugin
 168
 169         else:
 170             (t, name) = plugin.split('_', 1)
 171             if t == 'moodle':
 172                 t = 'core'
 173
 174         return (t, name)
 175
 176     @classmethod
 177     def getSubsystems(cls):
 178         """Return the list of subsytems and their relative directory"""
 179         return cls._subSystems
 180
 181     @classmethod
 182     def getSubsystemDirectory(cls, subsystem, M=None):
 183         """Return the subsystem directory, absolute if M is passed"""
 184         path = cls._subSystems.get(subsystem)
 185         if not path:
 186             raise ValueError('Unknown subsystem')
 187
 188         if M:
 189             path = path.replace('{admin}', M.get('admin', 'admin'))
 190             path = os.path.join(M.get('path'), path.strip('/'))
 191
 192         return path
 193
 194     @classmethod
 195     def getSubsystemOrPluginFromPath(cls, path, M=None):
 196         """Get a subsystem from a path. Path should be relative to dirroot or M should be passed.
 197
 198         This returns a tuple containing the name of the subsystem or plugin type, and the plugin name
 199         if we could resolve one.
 200         """
 201
 202         subtypes = {}
 203         path = os.path.realpath(os.path.abspath(path))
 204         if M:
 205             path = '/' + path.replace(M.get('path'), '').strip('/')
 206             admindir = M.get('admin', 'admin')
 207             if path.startswith('/' + admindir):
 208                 path = re.sub(r'^/%s' % admindir, '/{admin}', path)
 209             subtypes = cls.getSubtypes(M)
 210         path = '/' + path.lstrip('/')
 211
 212         pluginOrSubsystem = None
 213         pluginName = None
 214         candidate = path
 215         head = True
 216         tail = None
 217         while head and head != '/' and not pluginOrSubsystem:
 218             # Check plugin types.
 219             if not pluginOrSubsystem:
 220                 for k, v in cls._pluginTypesPath.iteritems():
 221                     if v == candidate:
 222                         pluginOrSubsystem = k
 223                         pluginName = tail
 224                         break
 225
 226             # Check sub plugin types.
 227             if not pluginOrSubsystem:
 228                 for k, v in subtypes.iteritems():
 229                     if v == candidate:
 230                         pluginOrSubsystem = k
 231                         pluginName = tail
 232                         break
 233
 234             # Check subsystems.
 235             for k, v in cls._subSystems.iteritems():
 236                 if v == candidate:
 237                     pluginOrSubsystem = k
 238                     break
 239
 240             (head, tail) = os.path.split(candidate)
 241             candidate = head
 242
 243         return (pluginOrSubsystem, pluginName)
 244
 245     @classmethod
 246     def getSubtypes(cls, M):
 247         """Get the sub plugins declared in an instance"""
 248         regex = re.compile(r'\s*(?P<brackets>[\'"])(.*?)(?P=brackets)\s*=>\s*(?P=brackets)(.*?)(?P=brackets)')
 249         subtypes = {}
 250         for t in cls._supportSubtypes:
 251             path = cls.getTypeDirectory(t, M)
 252             dirs = os.listdir(path)
 253             for d in dirs:
 254                 if not os.path.isdir(os.path.join(path, d)):
 255                     continue
 256                 subpluginsfile = os.path.join(path, d, 'db', 'subplugins.php')
 257                 if not os.path.isfile(subpluginsfile):
 258                     continue
 259
 260                 searchOpen = False
 261                 f = open(subpluginsfile, 'r')
 262                 for line in f:
 263                     if '$subplugins' in line:
 264                         searchOpen = True
 265
 266                     if searchOpen:
 267                         search = regex.findall(line)
 268                         if search:
 269                             for match in search:
 270                                 subtypes[match[1]] = '/' + match[2].replace('admin/', '{admin}/').lstrip('/')
 271
 272                     # Exit when we find a semi-colon.
 273                     if searchOpen and ';' in line:
 274                         break
 275
 276         return subtypes
 277
 278     @classmethod
 279     def getTypeDirectory(cls, t, M=None):
 280         """Returns the path to the plugin type directory. If M is passed, the full path is returned."""
 281         path = cls._pluginTypesPath.get(t, False)
 282         if not path:
 283             if M:
 284                 subtypes = cls.getSubtypes(M)
 285                 path = subtypes.get(t, False)
 286             if not path:
 287                 raise ValueError('Unknown plugin or subplugin type')
 288
 289         if M:
 290             if t == 'theme':
 291                 themedir = M.get('themedir', None)
 292                 if themedir != None:
 293                     return themedir
 294
 295             path = path.replace('{admin}', M.get('admin', 'admin'))
 296             path = os.path.join(M.get('path'), path.strip('/'))
 297
 298         return path
 299
 300     @classmethod
 301     def hasPlugin(cls, plugin, M):
 302         path = cls.getTypeDirectory(plugin.t, M)
 303         target = os.path.join(path, plugin.name)
 304         return os.path.isdir(target)
 305
 306     @classmethod
 307     def validateZipFile(cls, f, name):
 308         zp = zipfile.ZipFile(f, 'r')
 309
 310         # Checking that the content is all contained in one single directory
 311         rootDir = os.path.commonprefix(zp.namelist())
 312         if rootDir == '':
 313             return False
 314         return True
 315
 316
 317 class PluginObject(object):
 318
 319     component = None
 320     t = None
 321     name = None
 322
 323     def __init__(self, component):
 324         self.component = component
 325         (self.t, self.name) = PluginManager.getTypeAndName(component)
 326         self.dlinfo = {}
 327
 328     def getDownloadInfo(self, branch):
 329         if not self.dlinfo.get(branch, False):
 330             self.dlinfo[branch] = PluginRepository().info(self.component, branch)
 331         return self.dlinfo.get(branch, False)
 332
 333     def getZip(self, branch, fileCache=None):
 334         dlinfo = self.getDownloadInfo(branch)
 335         if not dlinfo:
 336             return False
 337         return dlinfo.download(fileCache)
 338
 339
 340 class PluginDownloadInfo(dict):
 341
 342     def download(self, fileCache=None, cacheDir=C.get('dirs.mdk')):
 343         """Download a plugin"""
 344
 345         if fileCache == None:
 346             fileCache = C.get('plugins.fileCache')
 347
 348         dest = os.path.abspath(os.path.expanduser(os.path.join(cacheDir, 'plugins')))
 349         if not fileCache:
 350             dest = gettempdir()
 351
 352         if not 'downloadurl' in self.keys():
 353             raise ValueError('Expecting the key downloadurl')
 354         elif not 'component' in self.keys():
 355             raise ValueError('Expecting the key component')
 356         elif not 'branch' in self.keys():
 357             raise ValueError('Expecting the key branch')
 358
 359         dl = self.get('downloadurl')
 360         plugin = self.get('component')
 361         branch = self.get('branch')
 362         target = os.path.join(dest, '%s-%d.zip' % (plugin, branch))
 363         md5sum = self.get('downloadmd5')
 364         release = self.get('release', 'Unknown')
 365
 366         if fileCache:
 367             if not os.path.isdir(dest):
 368                 logging.debug('Creating directory %s' % (dest))
 369                 tools.mkdir(dest, 0777)
 370
 371             if os.path.isfile(target) and (md5sum == None or tools.md5file(target) == md5sum):
 372                 logging.info('Found cached plugin file: %s' % (os.path.basename(target)))
 373                 return target
 374
 375         logging.info('Downloading %s (%s)' % (plugin, release))
 376         if logging.getLogger().level <= logging.INFO:
 377             urlretrieve(dl, target, tools.downloadProcessHook)
 378             # Force a new line after the hook display
 379             logging.info('')
 380         else:
 381             urlretrieve(dl, target)
 382
 383         # Highly memory inefficient MD5 check
 384         if md5sum and tools.md5file(target) != md5sum:
 385             os.remove(target)
 386             logging.warning('Bad MD5 sum on downloaded file')
 387             return False
 388
 389         return target
 390
 391
 392 class PluginRepository(object):
 393
 394     apiversion = '1.2'
 395     uri = '/api'
 396     host = 'download.moodle.org'
 397     ssl = True
 398     localRepository = None
 399     _instance = None
 400
 401     def __new__(cls, *args, **kwargs):
 402         if not cls._instance:
 403             cls._instance = super(PluginRepository, cls).__new__(cls, *args, **kwargs)
 404             cls._instance.localRepository = C.get('plugins.localRepository')
 405             cls._instance.localRepository = {} if cls._instance.localRepository == None else cls._instance.localRepository
 406         return cls._instance
 407
 408     def info(self, plugin, branch):
 409         """Gets the download information of the plugin, branch is expected to be
 410         a whole integer, such as 25 for 2.5, etc...
 411         """
 412
 413         if type(branch) != int:
 414             raise ValueError('Branch must be an integer')
 415
 416         # Checking local repository
 417         lr = self.localRepository.get(plugin, False)
 418         if lr:
 419             info = lr.get(branch, None)
 420             if not info:
 421                 versions = [v for v in range(branch, 18, -1)]
 422                 for v in versions:
 423                     info = lr.get('>=%d' % v, None)
 424                     if info:
 425                         break
 426             if info and info.get('downloadurl'):
 427                 logging.info('Found a compatible version for the plugin in local repository')
 428                 info['component'] = plugin
 429                 info['branch'] = branch
 430                 return PluginDownloadInfo(info)
 431
 432         # Contacting the remote repository
 433         data = {
 434             "branch": round(float(branch) / 10., 1),
 435             "plugin": plugin
 436         }
 437
 438         logging.info('Retrieving information for plugin %s and branch %s' % (data['plugin'], data['branch']))
 439         try:
 440             resp = self.request('pluginfo.php', 'GET', data)
 441         except PluginRepositoryNotFoundException:
 442             logging.info('No result found')
 443             return False
 444         except PluginRepositoryException:
 445             logging.warning('Error while retrieving information from the plugin database')
 446             return False
 447
 448         pluginfo = resp.get('data', {}).get('pluginfo', {})
 449         pluginfo['branch'] = branch
 450
 451         return PluginDownloadInfo(pluginfo)
 452
 453     def request(self, uri, method, data, headers={}):
 454         """Sends a request to the server and returns the response status and data"""
 455
 456         uri = self.uri + '/' + str(self.apiversion) + '/' + uri.strip('/')
 457         method = method.upper()
 458         if method == 'GET':
 459             if type(data) == dict:
 460                 data = urlencode(data)
 461             uri += '?%s' % (data)
 462             data = ''
 463
 464         if self.ssl:
 465             r = httplib.HTTPSConnection(self.host)
 466         else:
 467             r = httplib.HTTPConnection(self.host)
 468         logging.debug('%s %s%s' % (method, self.host, uri))
 469         r.request(method, uri, data, headers)
 470
 471         resp = r.getresponse()
 472         if resp.status == 404:
 473             raise PluginRepositoryNotFoundException()
 474         elif resp.status != 200:
 475             raise PluginRepositoryException('Error during the request to the plugin database')
 476
 477         data = resp.read()
 478         if len(data) > 0:
 479             try:
 480                 data = json.loads(data)
 481             except ValueError:
 482                 raise PluginRepositoryException('Could not parse JSON data. Data received:\n%s' % data)
 483
 484         return {'status': resp.status, 'data': data}
 485
 486
 487 class PluginRepositoryException(Exception):
 488     pass
 489
 490
 491 class PluginRepositoryNotFoundException(Exception):
 492     pass