source: OpenRLabs-Git/deploy/rlabs-docker/web2py-rlabs/gluon/contrib/redis_scheduler.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
| This file is part of the web2py Web Framework
| Created by niphlod@gmail.com
| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)

Scheduler with redis backend
---------------------------------
"""
from __future__ import print_function

import os
import time
import socket
import datetime
import logging
from json import loads, dumps
from gluon.utils import web2py_uuid
from gluon.storage import Storage
from gluon.scheduler import *
from gluon.scheduler import _decode_dict
from gluon.contrib.redis_utils import RWatchError

USAGE = """
## Example

For any existing app

Create File: app/models/scheduler.py ======
from gluon.contrib.redis_utils import RConn
from gluon.contrib.redis_scheduler import RScheduler

def demo1(*args,**vars):
    print('you passed args=%s and vars=%s' % (args, vars))
    return 'done!'

def demo2():
    1/0

rconn = RConn()
mysched = RScheduler(db, dict(demo1=demo1,demo2=demo2), ...., redis_conn=rconn)

## run worker nodes with:

   cd web2py
   python web2py.py -K app

"""


path = os.getcwd()

if 'WEB2PY_PATH' not in os.environ:
    os.environ['WEB2PY_PATH'] = path

IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())

logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)

POLLING = 'POLLING'


class RScheduler(Scheduler):

    def __init__(self, db, tasks=None, migrate=True,
                 worker_name=None, group_names=None, heartbeat=HEARTBEAT,
                 max_empty_runs=0, discard_results=False, utc_time=False,
                 redis_conn=None, mode=1):

        """
        Highly-experimental coordination with redis
        Takes all args from Scheduler except redis_conn which
        must be something closer to a StrictRedis instance.

        My only regret - and the reason why I kept this under the hood for a
        while - is that it's hard to hook up in web2py to something happening
        right after the commit to a table, which will enable this version of the
        scheduler to process "immediate" tasks right away instead of waiting a
        few seconds (see FIXME in queue_task())

        mode is reserved for future usage patterns.
        Right now it moves the coordination (which is the most intensive
        routine in the scheduler in matters of IPC) of workers to redis.
        I'd like to have incrementally redis-backed modes of operations,
        such as e.g.:
            - 1: IPC through redis (which is the current implementation)
            - 2: Store task results in redis (which will relieve further pressure
                 from the db leaving the scheduler_run table empty and possibly
                 keep things smooth as tasks results can be set to expire
                 after a bit of time)
            - 3: Move all the logic for storing and queueing tasks to redis
                 itself - which means no scheduler_task usage too - and use
                 the database only as an historical record-bookkeeping
                 (e.g. for reporting)

        As usual, I'm eager to see your comments.
        """

        Scheduler.__init__(self, db, tasks=tasks, migrate=migrate,
                           worker_name=worker_name, group_names=group_names,
                           heartbeat=heartbeat, max_empty_runs=max_empty_runs,
                           discard_results=discard_results, utc_time=utc_time)

        self.r_server = redis_conn
        from gluon import current
        self._application = current.request.application or 'appname'

    def _nkey(self, key):
        """Helper to restrict all keys to a namespace and track them."""
        prefix = 'w2p:rsched:%s' % self._application
        allkeys = '%s:allkeys' % prefix
        newkey = "%s:%s" % (prefix, key)
        self.r_server.sadd(allkeys, newkey)
        return newkey

    def prune_all(self):
        """Global housekeeping."""
        all_keys = self._nkey('allkeys')
        with self.r_server.pipeline() as pipe:
            while True:
                try:
                    pipe.watch('PRUNE_ALL')
                    while True:
                        k = pipe.spop(all_keys)
                        if k is None:
                            break
                        pipe.delete(k)
                    pipe.execute()
                    break
                except RWatchError:
                    time.sleep(0.1)
                    continue

    def dt2str(self, value):
        return value.strftime('%Y-%m-%d %H:%M:%S')

    def str2date(self, value):
        return datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')

    def send_heartbeat(self, counter):
        """
        Workers coordination in redis.
        It has evolved into something is not that easy.
        Here we try to do what we need in a single transaction,
        and retry that transaction if something goes wrong
        """
        with self.r_server.pipeline() as pipe:
            while True:
                try:
                    pipe.watch('SEND_HEARTBEAT')
                    self.inner_send_heartbeat(counter, pipe)
                    pipe.execute()
                    self.adj_hibernation()
                    self.sleep()
                    break
                except RWatchError:
                    time.sleep(0.1)
                    continue

    def inner_send_heartbeat(self, counter, pipe):
        """
        Do a few things in the "maintenance" thread.

        Specifically:
        - registers the workers
        - accepts commands sent to workers (KILL, TERMINATE, PICK, DISABLED, etc)
        - adjusts sleep
        - saves stats
        - elects master
        - does "housecleaning" for dead workers
        - triggers tasks assignment
        """
        r_server = pipe
        status_keyset = self._nkey('worker_statuses')
        status_key = self._nkey('worker_status:%s' % (self.worker_name))
        now = self.now()
        mybackedstatus = r_server.hgetall(status_key)
        if not mybackedstatus:
            r_server.hmset(
                status_key,
                dict(
                    status=ACTIVE, worker_name=self.worker_name,
                    first_heartbeat=self.dt2str(now),
                    last_heartbeat=self.dt2str(now),
                    group_names=dumps(self.group_names), is_ticker=False,
                    worker_stats=dumps(self.w_stats))
            )
            r_server.sadd(status_keyset, status_key)
            if not self.w_stats.status == POLLING:
                self.w_stats.status = ACTIVE
                self.w_stats.sleep = self.heartbeat
                mybackedstatus = ACTIVE
        else:
            mybackedstatus = mybackedstatus['status']
            if mybackedstatus == DISABLED:
                # keep sleeping
                self.w_stats.status = DISABLED
                r_server.hmset(
                    status_key,
                    dict(last_heartbeat=self.dt2str(now),
                         worker_stats=dumps(self.w_stats))
                )
            elif mybackedstatus == TERMINATE:
                self.w_stats.status = TERMINATE
                logger.debug("Waiting to terminate the current task")
                self.give_up()
            elif mybackedstatus == KILL:
                self.w_stats.status = KILL
                self.die()
            else:
                if mybackedstatus == STOP_TASK:
                    logger.info('Asked to kill the current task')
                    self.terminate_process()
                logger.info('........recording heartbeat (%s)',
                            self.w_stats.status)
                r_server.hmset(
                    status_key,
                    dict(
                        last_heartbeat=self.dt2str(now), status=ACTIVE,
                        worker_stats=dumps(self.w_stats)
                    )
                )
                # newroutine
                r_server.expire(status_key, self.heartbeat * 3 * 15)
                self.w_stats.sleep = self.heartbeat  # re-activating the process
                if self.w_stats.status not in (RUNNING, POLLING):
                    self.w_stats.status = ACTIVE

            self.do_assign_tasks = False
            if counter % 5 == 0 or mybackedstatus == PICK:
                try:
                    logger.info(
                        '    freeing workers that have not sent heartbeat')
                    registered_workers = r_server.smembers(status_keyset)
                    allkeys = self._nkey('allkeys')
                    for worker in registered_workers:
                        w = r_server.hgetall(worker)
                        w = Storage(w)
                        if not w:
                            r_server.srem(status_keyset, worker)
                            logger.info('removing %s from %s', worker, allkeys)
                            r_server.srem(allkeys, worker)
                            continue
                    try:
                        self.is_a_ticker = self.being_a_ticker(pipe)
                    except:
                        pass
                    if self.w_stats.status in (ACTIVE, POLLING):
                        self.do_assign_tasks = True
                    if self.is_a_ticker and self.do_assign_tasks:
                        # I'm a ticker, and 5 loops passed without reassigning tasks,
                        # let's do that and loop again
                        if not self.db_thread:
                            logger.debug('thread building own DAL object')
                            self.db_thread = DAL(
                                self.db._uri, folder=self.db._adapter.folder)
                            self.define_tables(self.db_thread, migrate=False)
                        db = self.db_thread
                        self.wrapped_assign_tasks(db)
                        return None
                except:
                    logger.error('Error assigning tasks')

    def being_a_ticker(self, pipe):
        """
        Elects a ticker.

        This is slightly more convoluted than the original
        but if far more efficient
        """
        r_server = pipe
        status_keyset = self._nkey('worker_statuses')
        registered_workers = r_server.smembers(status_keyset)
        ticker = None
        all_active = []
        all_workers = []
        for worker in registered_workers:
            w = r_server.hgetall(worker)
            if w['worker_name'] != self.worker_name and w['status'] == ACTIVE:
                all_active.append(w)
                if w['is_ticker'] == 'True' and ticker is None:
                    ticker = w
            all_workers.append(w)
        not_busy = self.w_stats.status in (ACTIVE, POLLING)
        if not ticker:
            if not_busy:
                # only if this worker isn't busy, otherwise wait for a free one
                for worker in all_workers:
                    key = self._nkey('worker_status:%s' % worker['worker_name'])
                    if worker['worker_name'] == self.worker_name:
                        r_server.hset(key, 'is_ticker', True)
                    else:
                        r_server.hset(key, 'is_ticker', False)
                logger.info("TICKER: I'm a ticker")
            else:
                # giving up, only if I'm not alone
                if len(all_active) > 1:
                    key = self._nkey('worker_status:%s' % (self.worker_name))
                    r_server.hset(key, 'is_ticker', False)
                else:
                    not_busy = True
            return not_busy
        else:
            logger.info(
                "%s is a ticker, I'm a poor worker" % ticker['worker_name'])
            return False

    def assign_tasks(self, db):
        """
        The real beauty.

        We don't need to ASSIGN tasks, we just put
        them into the relevant queue
        """
        st, sd = db.scheduler_task, db.scheduler_task_deps
        r_server = self.r_server
        now = self.now()
        status_keyset = self._nkey('worker_statuses')
        with r_server.pipeline() as pipe:
            while True:
                try:
                    # making sure we're the only one doing the job
                    pipe.watch('ASSIGN_TASKS')
                    registered_workers = pipe.smembers(status_keyset)
                    all_workers = []
                    for worker in registered_workers:
                        w = pipe.hgetall(worker)
                        if w['status'] == ACTIVE:
                            all_workers.append(Storage(w))
                    pipe.execute()
                    break
                except RWatchError:
                    time.sleep(0.1)
                    continue

        # build workers as dict of groups
        wkgroups = {}
        for w in all_workers:
            group_names = loads(w.group_names)
            for gname in group_names:
                if gname not in wkgroups:
                    wkgroups[gname] = dict(
                        workers=[{'name': w.worker_name, 'c': 0}])
                else:
                    wkgroups[gname]['workers'].append(
                        {'name': w.worker_name, 'c': 0})
        # set queued tasks that expired between "runs" (i.e., you turned off
        # the scheduler): then it wasn't expired, but now it is
        db(
            (st.status.belongs((QUEUED, ASSIGNED))) &
            (st.stop_time < now)
        ).update(status=EXPIRED)

        # calculate dependencies
        deps_with_no_deps = db(
            (sd.can_visit == False) &
            (~sd.task_child.belongs(
                db(sd.can_visit == False)._select(sd.task_parent)
            )
            )
        )._select(sd.task_child)
        no_deps = db(
            (st.status.belongs((QUEUED, ASSIGNED))) &
            (
                (sd.id == None) | (st.id.belongs(deps_with_no_deps))

            )
        )._select(st.id, distinct=True, left=sd.on(
                 (st.id == sd.task_parent) &
                 (sd.can_visit == False)
        )
        )

        all_available = db(
            (st.status.belongs((QUEUED, ASSIGNED))) &
            (st.next_run_time <= now) &
            (st.enabled == True) &
            (st.id.belongs(no_deps))
        )

        limit = len(all_workers) * (50 / (len(wkgroups) or 1))

        # let's freeze it up
        db.commit()
        x = 0
        r_server = self.r_server
        for group in wkgroups.keys():
            queued_list = self._nkey('queued:%s' % group)
            queued_set = self._nkey('queued_set:%s' % group)
            # if are running, let's don't assign them again
            running_list = self._nkey('running:%s' % group)
            while True:
                # the joys for rpoplpush!
                t = r_server.rpoplpush(running_list, queued_list)
                if not t:
                    # no more
                    break
                r_server.sadd(queued_set, t)

            tasks = all_available(st.group_name == group).select(
                limitby=(0, limit), orderby = st.next_run_time)

            # put tasks in the processing list

            for task in tasks:
                x += 1
                gname = task.group_name

                if r_server.sismember(queued_set, task.id):
                    # already queued, we don't put on the list
                    continue
                r_server.sadd(queued_set, task.id)
                r_server.lpush(queued_list, task.id)
                d = dict(status=QUEUED)
                if not task.task_name:
                    d['task_name'] = task.function_name
                db(
                    (st.id == task.id) &
                    (st.status.belongs((QUEUED, ASSIGNED)))
                ).update(**d)
            db.commit()
        # I didn't report tasks but I'm working nonetheless!!!!
        if x > 0:
            self.w_stats.empty_runs = 0
        self.w_stats.queue = x
        self.w_stats.distribution = wkgroups
        self.w_stats.workers = len(all_workers)
        # I'll be greedy only if tasks queued are equal to the limit
        # (meaning there could be others ready to be queued)
        self.greedy = x >= limit
        logger.info('TICKER: workers are %s', len(all_workers))
        logger.info('TICKER: tasks are %s', x)

    def pop_task(self, db):
        """Lift a task off a queue."""
        r_server = self.r_server
        st = self.db.scheduler_task
        task = None
        # ready to process something
        for group in self.group_names:
            queued_set = self._nkey('queued_set:%s' % group)
            queued_list = self._nkey('queued:%s' % group)
            running_list = self._nkey('running:%s' % group)
            running_dict = self._nkey('running_dict:%s' % group)
            self.w_stats.status = POLLING
            # polling for 1 minute in total. If more groups are in,
            # polling is 1 minute in total
            logger.debug('    polling on %s', group)
            task_id = r_server.brpoplpush(queued_list, running_list,
                                          timeout=60 / len(self.group_names))
            logger.debug('    finished polling')
            self.w_stats.status = ACTIVE
            if task_id:
                r_server.hset(running_dict, task_id, self.worker_name)
                r_server.srem(queued_set, task_id)
                task = db(
                    (st.id == task_id) &
                    (st.status == QUEUED)
                ).select().first()
                if not task:
                    r_server.lrem(running_list, 0, task_id)
                    r_server.hdel(running_dict, task_id)
                    r_server.lrem(queued_list, 0, task_id)
                    logger.error("we received a task that isn't there (%s)",
                                 task_id)
                    return None
                break
        now = self.now()
        if task:
            task.update_record(status=RUNNING, last_run_time=now)
            # noone will touch my task!
            db.commit()
            logger.debug('   work to do %s', task.id)
        else:
            logger.info('nothing to do')
            return None
        times_run = task.times_run + 1
        if task.cronline:
            cron_recur = CronParser(task.cronline, now.replace(second=0))
            next_run_time = cron_recur.get_next()
        elif not task.prevent_drift:
            next_run_time = task.last_run_time + datetime.timedelta(
                seconds=task.period
            )
        else:
            # calc next_run_time based on available slots
            # see #1191
            next_run_time = task.start_time
            secondspassed = (now - next_run_time).total_seconds()
            steps = secondspassed // task.period + 1
            next_run_time += datetime.timedelta(seconds=task.period * steps)

        if times_run < task.repeats or task.repeats == 0:
            # need to run (repeating task)
            run_again = True
        else:
            # no need to run again
            run_again = False
        run_id = 0
        while True and not self.discard_results:
            logger.debug('    new scheduler_run record')
            try:
                run_id = db.scheduler_run.insert(
                    task_id=task.id,
                    status=RUNNING,
                    start_time=now,
                    worker_name=self.worker_name)
                db.commit()
                break
            except:
                time.sleep(0.5)
                db.rollback()
        logger.info('new task %(id)s "%(task_name)s"'
                    ' %(application_name)s.%(function_name)s' % task)
        return Task(
            app=task.application_name,
            function=task.function_name,
            timeout=task.timeout,
            args=task.args,  # in json
            vars=task.vars,  # in json
            task_id=task.id,
            run_id=run_id,
            run_again=run_again,
            next_run_time=next_run_time,
            times_run=times_run,
            stop_time=task.stop_time,
            retry_failed=task.retry_failed,
            times_failed=task.times_failed,
            sync_output=task.sync_output,
            uuid=task.uuid,
            group_name=task.group_name)

    def report_task(self, task, task_report):
        """
        Override.

        Needs it only because we need to pop from the
        running tasks
        """
        r_server = self.r_server
        db = self.db
        now = self.now()
        st = db.scheduler_task
        sr = db.scheduler_run
        if not self.discard_results:
            if task_report.result != 'null' or task_report.tb:
                # result is 'null' as a string if task completed
                # if it's stopped it's None as NoneType, so we record
                # the STOPPED "run" anyway
                logger.debug(' recording task report in db (%s)',
                             task_report.status)
                db(sr.id == task.run_id).update(
                    status=task_report.status,
                    stop_time=now,
                    run_result=task_report.result,
                    run_output=task_report.output,
                    traceback=task_report.tb)
            else:
                logger.debug(' deleting task report in db because of no result')
                db(sr.id == task.run_id).delete()
        # if there is a stop_time and the following run would exceed it
        is_expired = (task.stop_time and
                      task.next_run_time > task.stop_time and
                      True or False)
        status = (task.run_again and is_expired and EXPIRED or
                  task.run_again and not is_expired and
                  QUEUED or COMPLETED)
        if task_report.status == COMPLETED:
            # assigned calculations
            d = dict(status=status,
                     next_run_time=task.next_run_time,
                     times_run=task.times_run,
                     times_failed=0,
                     assigned_worker_name=self.worker_name
                     )
            db(st.id == task.task_id).update(**d)
            if status == COMPLETED:
                self.update_dependencies(db, task.task_id)
        else:
            st_mapping = {'FAILED': 'FAILED',
                          'TIMEOUT': 'TIMEOUT',
                          'STOPPED': 'FAILED'}[task_report.status]
            status = (task.retry_failed and
                      task.times_failed < task.retry_failed and
                      QUEUED or task.retry_failed == -1 and
                      QUEUED or st_mapping)
            db(st.id == task.task_id).update(
                times_failed=st.times_failed + 1,
                next_run_time=task.next_run_time,
                status=status,
                assigned_worker_name=self.worker_name
            )
        logger.info('task completed (%s)', task_report.status)
        running_list = self._nkey('running:%s' % task.group_name)
        running_dict = self._nkey('running_dict:%s' % task.group_name)
        r_server.lrem(running_list, 0, task.task_id)
        r_server.hdel(running_dict, task.task_id)

    def wrapped_pop_task(self):
        """Commodity function to call `pop_task` and trap exceptions.
        If an exception is raised, assume it happened because of database
        contention and retries `pop_task` after 0.5 seconds
        """
        db = self.db
        db.commit()  # another nifty db.commit() only for Mysql
        x = 0
        while x < 10:
            try:
                rtn = self.pop_task(db)
                return rtn
                break
            # this is here to "interrupt" any blrpoplpush op easily
            except KeyboardInterrupt:
                self.give_up()
                break
            except:
                self.w_stats.errors += 1
                db.rollback()
                logger.error('    error popping tasks')
                x += 1
                time.sleep(0.5)

    def get_workers(self, only_ticker=False):
        """Return a dict holding worker_name : {**columns}
        representing all "registered" workers.
        only_ticker returns only the worker running as a TICKER,
        if there is any
        """
        r_server = self.r_server
        status_keyset = self._nkey('worker_statuses')
        registered_workers = r_server.smembers(status_keyset)
        all_workers = {}
        for worker in registered_workers:
            w = r_server.hgetall(worker)
            w = Storage(w)
            if not w:
                continue
            all_workers[w.worker_name] = Storage(
                status=w.status,
                first_heartbeat=self.str2date(w.first_heartbeat),
                last_heartbeat=self.str2date(w.last_heartbeat),
                group_names=loads(w.group_names, object_hook=_decode_dict),
                is_ticker=w.is_ticker == 'True' and True or False,
                worker_stats=loads(w.worker_stats, object_hook=_decode_dict)
            )
        if only_ticker:
            for k, v in all_workers.iteritems():
                if v['is_ticker']:
                    return {k: v}
            return {}
        return all_workers

    def set_worker_status(self, group_names=None, action=ACTIVE,
                          exclude=None, limit=None, worker_name=None):
        """Internal function to set worker's status"""
        r_server = self.r_server
        all_workers = self.get_workers()
        if not group_names:
            group_names = self.group_names
        elif isinstance(group_names, str):
            group_names = [group_names]
        exclusion = exclude and exclude.append(action) or [action]
        workers = []
        if worker_name is not None:
            if worker_name in all_workers.keys():
                workers = [worker_name]
        else:
            for k, v in all_workers.iteritems():
                if v.status not in exclusion and set(group_names) & set(v.group_names):
                    workers.append(k)
        if limit and worker_name is None:
            workers = workers[:limit]
        if workers:
            with r_server.pipeline() as pipe:
                while True:
                    try:
                        pipe.watch('SET_WORKER_STATUS')
                        for w in workers:
                            worker_key = self._nkey('worker_status:%s' % w)
                            pipe.hset(worker_key, 'status', action)
                        pipe.execute()
                        break
                    except RWatchError:
                        time.sleep(0.1)
                        continue

    def queue_task(self, function, pargs=[], pvars={}, **kwargs):
        """
        FIXME: immediate should put item in queue. The hard part is
        that currently there are no hooks happening at post-commit time
        Queue tasks. This takes care of handling the validation of all
        parameters

        Args:
            function: the function (anything callable with a __name__)
            pargs: "raw" args to be passed to the function. Automatically
                jsonified.
            pvars: "raw" kwargs to be passed to the function. Automatically
                jsonified
            kwargs: all the parameters available (basically, every
                `scheduler_task` column). If args and vars are here, they should
                be jsonified already, and they will override pargs and pvars

        Returns:
            a dict just as a normal validate_and_insert(), plus a uuid key
            holding the uuid of the queued task. If validation is not passed
            ( i.e. some parameters are invalid) both id and uuid will be None,
            and you'll get an "error" dict holding the errors found.
        """
        if hasattr(function, '__name__'):
            function = function.__name__
        targs = 'args' in kwargs and kwargs.pop('args') or dumps(pargs)
        tvars = 'vars' in kwargs and kwargs.pop('vars') or dumps(pvars)
        tuuid = 'uuid' in kwargs and kwargs.pop('uuid') or web2py_uuid()
        tname = 'task_name' in kwargs and kwargs.pop('task_name') or function
        immediate = 'immediate' in kwargs and kwargs.pop('immediate') or None
        cronline = kwargs.get('cronline')
        kwargs.update(function_name=function,
            task_name=tname,
            args=targs,
            vars=tvars,
            uuid=tuuid)
        if cronline:
            try:
                start_time = kwargs.get('start_time', self.now)
                next_run_time = CronParser(cronline, start_time).get_next()
                kwargs.update(start_time=start_time, next_run_time=next_run_time)
            except:
                pass
        rtn = self.db.scheduler_task.validate_and_insert(**kwargs)
        if not rtn.errors:
            rtn.uuid = tuuid
            if immediate:
                r_server = self.r_server
                ticker = self.get_workers(only_ticker=True)
                if ticker.keys():
                    ticker = ticker.keys()[0]
                    with r_server.pipeline() as pipe:
                        while True:
                            try:
                                pipe.watch('SET_WORKER_STATUS')
                                worker_key = self._nkey('worker_status:%s' % ticker)
                                pipe.hset(worker_key, 'status', 'PICK')
                                pipe.execute()
                                break
                            except RWatchError:
                                time.sleep(0.1)
                                continue
        else:
            rtn.uuid = None
        return rtn

    def stop_task(self, ref):
        """Shortcut for task termination.

        If the task is RUNNING it will terminate it, meaning that status
        will be set as FAILED.

        If the task is QUEUED, its stop_time will be set as to "now",
            the enabled flag will be set to False, and the status to STOPPED

        Args:
            ref: can be

              - an integer : lookup will be done by scheduler_task.id
              - a string : lookup will be done by scheduler_task.uuid

        Returns:
            - 1 if task was stopped (meaning an update has been done)
            - None if task was not found, or if task was not RUNNING or QUEUED

        Note:
            Experimental
        """
        r_server = self.r_server
        st = self.db.scheduler_task
        if isinstance(ref, int):
            q = st.id == ref
        elif isinstance(ref, str):
            q = st.uuid == ref
        else:
            raise SyntaxError(
                "You can retrieve results only by id or uuid")
        task = self.db(q).select(st.id, st.status, st.group_name)
        task = task.first()
        rtn = None
        if not task:
            return rtn
        running_dict = self._nkey('running_dict:%s' % task.group_name)
        if task.status == 'RUNNING':
            worker_key = r_server.hget(running_dict, task.id)
            worker_key = self._nkey('worker_status:%s' % (worker_key))
            r_server.hset(worker_key, 'status', STOP_TASK)
        elif task.status == 'QUEUED':
            rtn = self.db(q).update(
                stop_time=self.now(),
                enabled=False,
                status=STOPPED)
        return rtn