Source code for bkr.server.tasks

#
# Copyright (C) 2008 bpeck@redhat.com
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

from turbogears.database import session
from turbogears import controllers, expose, flash, widgets, validate, error_handler, validators, redirect, paginate, config, url
from turbogears import identity, redirect
from cherrypy import request, response
from kid import Element
from bkr.server import search_utility
from bkr.server.widgets import myPaginateDataGrid
from bkr.server.widgets import TasksWidget
from bkr.server.widgets import TaskSearchForm
from bkr.server.widgets import SearchBar
from bkr.server.widgets import TaskActionWidget
from bkr.server.xmlrpccontroller import RPCRoot
from bkr.server.helpers import make_link
from rhts import testinfo
from rhts.testinfo import ParserError, ParserWarning
from sqlalchemy.orm import joinedload, joinedload_all
from sqlalchemy.orm.exc import NoResultFound
from subprocess import *

import rpm
import os

import cherrypy

# from medusa import json
# import logging
# log = logging.getLogger("medusa.controllers")
#import model
from model import *
import string

__all__ = ['Tasks']

class Tasks(RPCRoot):
    # For XMLRPC methods in this class.
    exposed = True

    task_list_action_widget = TaskActionWidget()
    task_form = TaskSearchForm()
    task_widget = TasksWidget()
    task_dir = config.get("basepath.rpms", "/var/www/beaker/rpms")

    upload = widgets.FileField(name='task_rpm', label='Task rpm')
    form = widgets.TableForm(
        'task',
        fields = [upload],
        action = 'save_data',
        submit_text = _(u'Submit Data')
    )

    @expose(template='bkr.server.templates.form-post')
    @identity.require(identity.not_anonymous())
    def new(self, **kw):
        return_dict = dict(
            title = 'New Task',
            form = self.form,
            action = './save',
            options = {},
            value = kw,
        )
        return return_dict

    @cherrypy.expose
    def filter(self, filter):
        """
        Returns a list of tasks filtered by the given criteria.

        The *filter* argument must be an XML-RPC structure (dict), with any of the following keys:

            'distro_name'
                Distro name. Include only tasks which are compatible 
                with this distro.
            'osmajor'
                OSVersion OSMajor, like RedHatEnterpriseLinux6.  Include only
                tasks which are compatible with this OSMajor.
            'names'
                Task name. Include only tasks that are named. Useful when
                combined with 'osmajor' or 'distro_name'.
            'packages'
                List of package names. Include only tasks which have a Run-For 
                entry matching any of these packages.
            'types'
                List of task types. Include only tasks which have one or more 
                of these types.
            'valid'
                bool 0 or 1. Include only tasks which are valid or not.
            'destructive'
                bool 0 or 1. set to 0 for only non-destructive tasks
                             set to 1 for only destructive tasks

        The return value is an array of dicts, which are name and arches. 
          name is the name of the matching tasks.
          arches is an array of arches which this task does not apply for.
        Call :meth:`tasks.to_dict` to fetch metadata for a particular task.

        .. versionchanged:: 0.9
           Changed 'install_name' to 'distro_name' in the *filter* argument.
        """
        if filter.get('distro_name'):
            try:
                distro = Distro.by_name(filter['distro_name'])
            except NoResultFound:
                raise BX(_(u'Invalid Distro: %s') % filter['distro_name'])
            tasks = distro.tasks()
        elif 'osmajor' in filter and filter['osmajor']:
            try:
                osmajor = OSMajor.by_name(filter['osmajor'])
            except InvalidRequestError, err:
                raise BX(_('Invalid OSMajor: %s' % filter['osmajor']))
            tasks = osmajor.tasks()
        else:
            tasks = Task.query

        # Filter by valid task if requested
        if 'valid' in filter:
            tasks = tasks.filter(Task.valid==bool(filter['valid']))

        # Filter by destructive if requested
        if 'destructive' in filter:
            tasks = tasks.filter(Task.destructive==bool(filter['destructive']))

        # Filter by name if specified
        # /distribution/install, /distribution/reservesys
        if 'names' in filter and filter['names']:
            # if not a list, make it into a list.
            if isinstance(filter['names'], str):
                filter['names'] = [filter['names']]
            or_names = []
            for tname in filter['names']:
                or_names.append(Task.name==tname)
            tasks = tasks.filter(or_(*or_names))

        # Filter by packages if specified
        # apache, kernel, mysql, etc..
        if 'packages' in filter and filter['packages']:
            # if not a list, make it into a list.
            if isinstance(filter['packages'], str):
                filter['packages'] = [filter['packages']]
            tasks = tasks.filter(Task.runfor.any(or_(
                    *[TaskPackage.package == package for package in filter['packages']])))

        # Filter by type if specified
        # Tier1, Regression, KernelTier1, etc..
        if 'types' in filter and filter['types']:
            # if not a list, make it into a list.
            if isinstance(filter['types'], str):
                filter['types'] = [filter['types']]
            tasks = tasks.join('types')
            or_types = []
            for type in filter['types']:
                try:
                    tasktype = TaskType.by_name(type)
                except InvalidRequestError, err:
                    raise BX(_('Invalid Task Type: %s' % type))
                or_types.append(TaskType.id==tasktype.id)
            tasks = tasks.filter(or_(*or_types))

        # Return all task names
        return [dict(name = task.name, arches = [str(arch.arch) for arch in task.excluded_arch]) for task in tasks]

    @cherrypy.expose
    @identity.require(identity.not_anonymous())
    def upload(self, task_rpm_name, task_rpm_data):
        """
        Uploads a new task RPM.

        :param task_rpm_name: filename of the task RPM, for example 
            ``'beaker-distribution-install-1.10-11.noarch.rpm'``
        :type task_rpm_name: string
        :param task_rpm_data: contents of the task RPM
        :type task_rpm_data: XML-RPC binary
        """
        rpm_file = "%s/%s" % (self.task_dir, task_rpm_name)
        if os.path.exists("%s" % rpm_file):
            raise BX(_(u'Cannot import duplicate task %s') % task_rpm_name)
        else:
            FH = open(rpm_file, "w")
            FH.write(task_rpm_data.data)
            FH.close()
            try:
                task = self.process_taskinfo(self.read_taskinfo(rpm_file))
            except Exception, e:
                # Delete invalid rpm
                os.unlink(rpm_file)
                raise
            return "Success"

    @expose()
    @identity.require(identity.not_anonymous())
    def save(self, task_rpm, *args, **kw):
        """
        TurboGears method to upload task rpm package
        """
        rpm_file = "%s/%s" % (self.task_dir, task_rpm.filename)

        if os.path.exists("%s" % rpm_file):
            flash(_(u'Failed to import because we already have %s' % 
                                                     task_rpm.filename ))
            redirect(url("./new"))
        else:
            myrpm = task_rpm.file.read()
            FH = open(rpm_file, "w")
            FH.write(myrpm)
            FH.close()
            try:
                task = self.process_taskinfo(self.read_taskinfo(rpm_file))
            except Exception, err:
                # Delete invalid rpm
                os.unlink(rpm_file)
                session.rollback()
                log.exception('Failed to import %s', task_rpm.filename)
                flash(_(u'Failed to import task: %s' % err))
                redirect(url("./new"))

            flash(_(u"%s Added/Updated at id:%s" % (task.name,task.id)))
            redirect(".")

    @expose(template='bkr.server.templates.task_search')
    @validate(form=task_form)
    @paginate('tasks',default_order='-id', limit=30)
    def executed(self, hidden={}, **kw):
        tmp = self._do_search(hidden, **kw)
        tmp['form'] = self.task_form
        tmp['action'] = './do_search'
        tmp['value'] = None
        tmp['options'] = dict()
        return tmp
    
    @expose(template='bkr.server.templates.tasks')
    @validate(form=task_form)
    @paginate('tasks',default_order='-id', limit=30, max_limit=None)
    def do_search(self, hidden={}, **kw):
        return self._do_search(hidden=hidden, **kw)

    def _do_search(self, hidden={}, **kw):
        tasks = RecipeTask.query\
                .join(RecipeTask.recipe, Recipe.recipeset, RecipeSet.job)\
                .filter(and_(Job.to_delete == None, Job.deleted == None))\
                .options(joinedload(RecipeTask.task), joinedload(RecipeTask.logs),
                    joinedload_all(RecipeTask.results, RecipeTaskResult.logs))
        if 'recipe_id' in kw: #most likely we are coming here from a LinkRemoteFunction in recipe_widgets
            tasks = tasks.filter(Recipe.id == kw['recipe_id'])

            hidden = dict(distro_tree=1, system=1)
            return dict(tasks=tasks,hidden=hidden,task_widget=self.task_widget)

        if kw.get('distro_tree_id'):
            tasks = tasks.join(RecipeTask.recipe, Recipe.distro_tree)\
                    .filter(DistroTree.id == kw.get('distro_tree_id'))
            hidden = dict(distro_tree=1)
        elif kw.get('distro_id'):
            tasks = tasks.join(RecipeTask.recipe, Recipe.distro_tree, DistroTree.distro)\
                    .filter(Distro.id == kw.get('distro_id'))
        if kw.get('task_id'):
            try:
                tasks = tasks.join('task').filter(Task.id==kw.get('task_id'))
                hidden = dict(task = 1,
                             )
            except InvalidRequestError:
                return "<div>Invalid data:<br>%r</br></div>" % kw
        if kw.get('system_id'):
            try:
                tasks = tasks.join(['recipe','system']).filter(System.id==kw.get('system_id')).order_by(recipe_task_table.c.id.desc())
                hidden = dict(system = 1,
                             )
            except InvalidRequestError:
                return "<div>Invalid data:<br>%r</br></div>" % kw
        if kw.get('job_id'):
            job_id = kw.get('job_id')
            if not isinstance(job_id, list):
                job_id = [job_id]
            tasks = tasks.join(['recipe','recipeset','job']).filter(Job.id.in_(job_id))
        if kw.get('system'):
            tasks = tasks.join(['recipe','system']).filter(System.fqdn.like('%%%s%%' % kw.get('system')))
        if kw.get('task'):
            # Shouldn't have to do this.  This only happens on the LinkRemoteFunction calls
            kw['task'] = kw.get('task').replace('%2F','/')
            tasks = tasks.join('task').filter(Task.name.like('%s' % kw.get('task').replace('*','%%')))
        if kw.get('distro'):
            tasks = tasks.join(RecipeTask.recipe, Recipe.distro_tree, DistroTree.distro)\
                    .filter(Distro.name.like('%%%s%%' % kw.get('distro')))
        if kw.get('arch_id'):
            tasks = tasks.join(RecipeTask.recipe, Recipe.distro_tree, DistroTree.arch)\
                    .filter(Arch.id == kw.get('arch_id'))
        if kw.get('status'):
            tasks = tasks.filter(RecipeTask.status == kw['status'])
        if kw.get('result'):
            tasks = tasks.filter(RecipeTask.result == kw['result'])
        if kw.get('osmajor_id'):
            tasks = tasks.join(RecipeTask.recipe, Recipe.distro_tree,
                    DistoTree.distro, Distro.osversion, OSVersion.osmajor)\
                    .filter(OSMajor.id == kw.get('osmajor_id'))
        if kw.get('whiteboard'):
            tasks = tasks.join(['recipe']).filter(Recipe.whiteboard==kw.get('whiteboard'))
        return dict(tasks = tasks,
                    hidden = hidden,
                    task_widget = self.task_widget)

    @identity.require(identity.in_group('admin'))
    @expose()
    def disable_from_ui(self, t_id, *args, **kw):
        to_return = dict( t_id = t_id )
        try:
            self._disable(t_id)
            to_return['success'] = True
        except Exception, e:
            log.exception('Unable to disable task:%s' % t_id)
            to_return['success'] = False
            to_return['err_msg'] = unicode(e)
            session.rollback()
        return to_return

    def _disable(self, t_id, *args, **kw):
        """
        disable task
         task.valid=False
         remove task rpms from filesystem
        """
        task = Task.by_id(t_id)
        return task.disable()

    @expose(template='bkr.server.templates.grid')
    @paginate('list',default_order='name', limit=30)
    def index(self, *args, **kw):
        tasks = Task.query
        # FIXME What we really want is some default search options
        # For now we won't show deleted/invalid tasks in the grid
        # but for data integrity reasons we will allow you to view
        # the task directly.  Ideally we would have a default search
        # option of valid=True which the user could change to false
        # to see all "deleted" tasks
        tasks = tasks.filter(Task.valid==True)

        tasks_return = self._tasks(tasks,**kw)
        searchvalue = None
        search_options = {}
        if tasks_return:
            if 'tasks_found' in tasks_return:
                tasks = tasks_return['tasks_found']
            if 'searchvalue' in tasks_return:
                searchvalue = tasks_return['searchvalue']
            if 'simplesearch' in tasks_return:
                search_options['simplesearch'] = tasks_return['simplesearch']

        tasks_grid = myPaginateDataGrid(fields=[
		     widgets.PaginateDataGrid.Column(name='name', getter=lambda x: make_link("./%s" % x.id, x.name), title='Name', options=dict(sortable=True)),
		     widgets.PaginateDataGrid.Column(name='description', getter=lambda x:x.description, title='Description', options=dict(sortable=True)),
		     widgets.PaginateDataGrid.Column(name='version', getter=lambda x:x.version, title='Version', options=dict(sortable=True)),
                     widgets.PaginateDataGrid.Column(name='action', getter=lambda x: self.task_list_action_widget.display(task=x, type_='tasklist', title='Action', options=dict(sortable=False))),
                    ])

        search_bar = SearchBar(name='tasksearch',
                           label=_(u'Task Search'),
                           table = search_utility.Task.search.create_search_table(),
                           complete_data=search_utility.Task.search.create_complete_search_table(),
                           search_controller=url("/get_search_options_task"),
                           )
        return dict(title="Task Library",
                    object_count=tasks.count(),
                    grid=tasks_grid,
                    list=tasks,
                    search_bar=search_bar,
                    action='.',
                    action_widget = self.task_list_action_widget,  #Hack,inserts JS for us.
                    options=search_options,
                    searchvalue=searchvalue)

    @expose(template='bkr.server.templates.task')
    def default(self, *args, **kw):
        try:
            using_task_id = False
            if len(args) == 1:
                try:
                    task_id = int(args[0])
                    using_task_id = True
                except ValueError:
                    pass
            if using_task_id:
                task = Task.by_id(task_id)
            else:
                task = Task.by_name("/%s" % "/".join(args))
                #Would rather not redirect but do_search expects task_id in URL
                #This is the simplest way of dealing with it
                redirect("/tasks/%s" % task.id)
        except InvalidRequestError:
            if using_task_id:
                err_msg = u'Invalid task_id %s' % args[0]
            else:
                err_msg =  u'Invalid task /%s' % '/'.join(args)
            flash(_(err_msg))
            redirect("/tasks")
        return dict(task=task,
                    form = self.task_form,
                    value = dict(task_id = task.id),
                    options = dict(hidden=dict(task = 1)),
                    action = './do_search')

    def process_taskinfo(self, raw_taskinfo):
        tinfo = testinfo.parse_string(raw_taskinfo['desc'])

        try:
            task = Task.by_name(tinfo.test_name)
        except NoResultFound:
            task = Task(name=tinfo.test_name)
        # RPM is the same version we have. don't process		
        if task.version == raw_taskinfo['hdr']['ver']:
            raise BX(_("Failed to import,  %s is the same version we already have" % task.version))
        # Keep N-1 versions of task rpms.  This allows currently running tasks to finish.
        if task.oldrpm and os.path.exists("%s/%s" % (self.task_dir, task.oldrpm)):
            try:
                os.unlink("%s/%s" % (self.task_dir, task.oldrpm))
            except OSError, err:
                raise BX(_("%s" % err))   
        # Current becomes old
        task.oldrpm = task.rpm
        task.rpm = raw_taskinfo['hdr']['rpm']
        task.version = raw_taskinfo['hdr']['ver']
        task.description = tinfo.test_description
        task.types = []
        task.bugzillas = []
        task.required = []
        task.runfor = []
        task.needs = []
        task.excluded_osmajor = []
        task.excluded_arch = []
        includeFamily=[]
        for family in tinfo.releases:
            if family.startswith('-'):
                try:
                    if family.lstrip('-') not in task.excluded_osmajor:
                        task.excluded_osmajor.append(TaskExcludeOSMajor(osmajor=OSMajor.by_name_alias(family.lstrip('-'))))
                except InvalidRequestError:
                    pass
            else:
                try:
                    includeFamily.append(OSMajor.by_name_alias(family).osmajor)
                except InvalidRequestError:
                    pass
        families = set([ '%s' % family.osmajor for family in OSMajor.query])
        if includeFamily:
            for family in families.difference(set(includeFamily)):
                if family not in task.excluded_osmajor:
                    task.excluded_osmajor.append(TaskExcludeOSMajor(osmajor=OSMajor.by_name_alias(family)))
        if tinfo.test_archs:
            arches = set([ '%s' % arch.arch for arch in Arch.query])
            for arch in arches.difference(set(tinfo.test_archs)):
                if arch not in task.excluded_arch:
                    task.excluded_arch.append(TaskExcludeArch(arch=Arch.by_name(arch)))
        task.avg_time = tinfo.avg_test_time
        for type in tinfo.types:
            ttype = TaskType.lazy_create(type=type)
            task.types.append(ttype)
        for bug in tinfo.bugs:
            task.bugzillas.append(TaskBugzilla(bugzilla_id=bug))
        task.path = tinfo.test_path
        # Bug 772882. Remove duplicate required package here
        # Avoid ORM insert in task_packages_required_map twice.
        tinfo.runfor = list(set(tinfo.runfor))
        for runfor in tinfo.runfor:
            package = TaskPackage.lazy_create(package=runfor)
            task.runfor.append(package)
        task.priority = tinfo.priority
        task.destructive = tinfo.destructive
        # Bug 772882. Remove duplicate required package here
        # Avoid ORM insert in task_packages_required_map twice.
        tinfo.requires = list(set(tinfo.requires))
        for require in tinfo.requires:
            package = TaskPackage.lazy_create(package=require)
            task.required.append(package)
        for need in tinfo.needs:
            task.needs.append(TaskPropertyNeeded(property=need))
        task.license = tinfo.license
        task.owner = tinfo.owner
        task.uploader = identity.current.user
        task.valid = True

        return task

    @expose(format='json')
    def by_name(self, task):
        task = task.lower()
        return dict(tasks=[(task.name) for task in Task.query.filter(Task.name.like('%s%%' % task))])

    @cherrypy.expose
    def to_dict(self, name, valid=None):
        """
        Returns an XML-RPC structure (dict) with details about the given task.
        """
        return Task.by_name(name, valid).to_dict()

    def read_taskinfo(self, rpm_file):
        taskinfo = dict(desc = '',
                        hdr  = '',
                       )
        taskinfo['hdr'] = self.get_rpm_info(rpm_file)
        taskinfo_file = None
	for file in taskinfo['hdr']['files']:
            if file.endswith('testinfo.desc'):
                taskinfo_file = file
        if taskinfo_file:
            p1 = Popen(["rpm2cpio", rpm_file], stdout=PIPE)
            p2 = Popen(["cpio", "--quiet", "--extract" , "--to-stdout", ".%s" % taskinfo_file], stdin=p1.stdout, stdout=PIPE)
            taskinfo['desc'] = p2.communicate()[0]
        return taskinfo

    def get_rpm_info(self, rpm_file):
        """Returns rpm information by querying a rpm"""
        ts = rpm.ts()
        fdno = os.open(rpm_file, os.O_RDONLY)
        try:
            hdr = ts.hdrFromFdno(fdno)
        except rpm.error:
            fdno = os.open(rpm_file, os.O_RDONLY)
            ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
            hdr = ts.hdrFromFdno(fdno)
        os.close(fdno)
        return { 'name': hdr[rpm.RPMTAG_NAME], 
                 'ver' : "%s-%s" % (hdr[rpm.RPMTAG_VERSION],
                                    hdr[rpm.RPMTAG_RELEASE]), 
                 'epoch': hdr[rpm.RPMTAG_EPOCH],
                 'arch': hdr[rpm.RPMTAG_ARCH] , 
                 'rpm': "%s" % rpm_file.split('/')[-1:][0],
                 'files': hdr['filenames']}


    def _tasks(self,task,**kw):
        return_dict = {}                    
        if 'simplesearch' in kw:
            simplesearch = kw['simplesearch']
            kw['tasksearch'] = [{'table' : 'Name',   
                                 'operation' : 'contains', 
                                 'value' : kw['simplesearch']}]                    
        else:
            simplesearch = None

        return_dict.update({'simplesearch':simplesearch})

        if kw.get("tasksearch"):
            searchvalue = kw['tasksearch']  
            tasks_found = self._task_search(task,**kw) 
            return_dict.update({'tasks_found':tasks_found})               
            return_dict.update({'searchvalue':searchvalue})
        return return_dict

    def _task_search(self,task,**kw):
        task_search = search_utility.Task.search(task)
        for search in kw['tasksearch']:
            col = search['table'] 
            task_search.append_results(search['value'],col,search['operation'],**kw)
        return task_search.return_results()

    @expose(template='bkr.server.templates.tasks')
    def parrot(self,*args,**kw): 
        if 'recipe_id' in kw:
            recipe = Recipe.by_id(kw['recipe_id'])
            return recipe.all_tasks

# for sphinx
tasks = Tasks