mvpa.misc.state

1 #emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*- 2 #ex: set sts=4 ts=4 sw=4 et: 3 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 4 # 5 # See COPYING file distributed along with the PyMVPA package for the 6 # copyright and license terms. 7 # 8 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 9 """Classes to control and store state information. 10 11 It was devised to provide conditional storage 12 """ 13 14 __docformat__ = 'restructuredtext' 15 16 import operator, copy 17 from sets import Set 18 from textwrap import TextWrapper 19 20 import numpy as N 21 22 from mvpa.misc.vproperty import VProperty 23 from mvpa.misc.exceptions import UnknownStateError 24 from mvpa.base.dochelpers import enhancedDocString 25 26 from mvpa.base import externals 27 28 if __debug__: 29 from mvpa.base import debug 30 31 32 _in_ipython = externals.exists('running ipython env') 33 # Separators around definitions, needed for ReST, but bogus for 34 # interactive sessions 35 _def_sep = ('`', '')[int(_in_ipython)] 36 37 _object_getattribute = object.__getattribute__ 38 _object_setattr = object.__setattr__ 39 40 ################################################################## 41 # Various attributes which will be collected into collections 42 #

43 -class CollectableAttribute(object):

44 """Base class for any custom behaving attribute intended to become 45 part of a collection. 46 47 Derived classes will have specific semantics: 48 49 * StateVariable: conditional storage 50 * AttributeWithUnique: easy access to a set of unique values 51 within a container 52 * Parameter: attribute with validity ranges. 53 54 - ClassifierParameter: specialization to become a part of 55 Classifier's params collection 56 - KernelParameter: --//-- to become a part of Kernel Classifier's 57 kernel_params collection 58 59 Those CollectableAttributes are to be groupped into corresponding 60 collections for each class by statecollector metaclass, ie it 61 would be done on a class creation (ie not per each object) 62 """ 63 64 _instance_index = 0 65

66 - def __init__(self, name=None, doc=None, index=None):

67 if index is None: 68 CollectableAttribute._instance_index += 1 69 index = CollectableAttribute._instance_index 70 self._instance_index = index 71 self.__doc__ = doc 72 self.__name = name 73 self._value = None 74 self._isset = False 75 self.reset() 76 if __debug__: 77 debug("COL", 78 "Initialized new collectable #%d:%s" % (index,name) + `self`)

79 80 81 # Instead of going for VProperty lets make use of virtual method

82 - def _getVirtual(self):

83 return self._get()

84 85

86 - def _setVirtual(self, value):

87 return self._set(value)

88 89

90 - def _get(self):

91 return self._value

92 93

94 - def _set(self, val):

95 if __debug__: 96 # Since this call is quite often, don't convert 97 # values to strings here, rely on passing them 98 # withing msgargs 99 debug("COL", 100 "Setting %(self)s to %(val)s ", 101 msgargs={'self':self, 'val':val}) 102 self._value = val 103 self._isset = True

104 105 106 @property

107 - def isSet(self):

108 return self._isset

109 110

111 - def reset(self):

112 """Simply reset the flag""" 113 if __debug__ and self._isset: 114 debug("COL", "Reset %s to being non-modified" % self.name) 115 self._isset = False

116 117 118 # TODO XXX unify all bloody __str__

119 - def __str__(self):

120 res = "%s" % (self.name) 121 if self.isSet: 122 res += '*' # so we have the value already 123 return res

124 125

126 - def _getName(self):

127 return self.__name

128 129

130 - def _setName(self, name):

131 if name is not None: 132 if isinstance(name, basestring): 133 if name[0] == '_': 134 raise ValueError, \ 135 "Collectable attribute name must not start " \ 136 "with _. Got %s" % name 137 else: 138 raise ValueError, \ 139 "Collectable attribute name must be a string. " \ 140 "Got %s" % `name` 141 self.__name = name

142 143 144 # XXX should become vproperty? 145 # YYY yoh: not sure... someone has to do performance testing 146 # to see which is more effective. My wild guess is that 147 # _[gs]etVirtual would be faster 148 value = property(_getVirtual, _setVirtual) 149 name = property(_getName, _setName)

150 151 152 153 # XXX think that may be discard hasunique and just devise top 154 # class DatasetAttribute

155 -class AttributeWithUnique(CollectableAttribute):

156 """Container which also takes care about recomputing unique values 157 158 XXX may be we could better link original attribute to additional 159 attribute which actually stores the values (and do reverse there 160 as well). 161 162 Pros: 163 * don't need to mess with getattr since it would become just another 164 attribute 165 166 Cons: 167 * might be worse design in terms of comprehension 168 * take care about _set, since we shouldn't allow 169 change it externally 170 171 For now lets do it within a single class and tune up getattr 172 """ 173

174 - def __init__(self, name=None, hasunique=True, doc="Attribute with unique"):

175 CollectableAttribute.__init__(self, name, doc) 176 self._hasunique = hasunique 177 self._resetUnique() 178 if __debug__: 179 debug("UATTR", 180 "Initialized new AttributeWithUnique %s " % name + `self`)

181 182

183 - def reset(self):

184 super(AttributeWithUnique, self).reset() 185 self._resetUnique()

186 187

188 - def _resetUnique(self):

189 self._uniqueValues = None

190 191

192 - def _set(self, *args, **kwargs):

193 self._resetUnique() 194 CollectableAttribute._set(self, *args, **kwargs)

195 196

197 - def _getUniqueValues(self):

198 if self.value is None: 199 return None 200 if self._uniqueValues is None: 201 # XXX we might better use Set, but yoh recalls that 202 # N.unique was more efficient. May be we should check 203 # on the the class and use Set only if we are not 204 # dealing with ndarray (or lists/tuples) 205 self._uniqueValues = N.unique(N.asanyarray(self.value)) 206 return self._uniqueValues

207 208 uniqueValues = property(fget=_getUniqueValues) 209 hasunique = property(fget=lambda self:self._hasunique)

210 211 # Hooks for comprehendable semantics and automatic collection generation

212 -class SampleAttribute(AttributeWithUnique):

213 pass

214

215 -class FeatureAttribute(AttributeWithUnique):

216 pass

217

218 -class DatasetAttribute(AttributeWithUnique):

219 pass

220 221

222 -class StateVariable(CollectableAttribute):

223 """Simple container intended to conditionally store the value 224 """ 225

226 - def __init__(self, name=None, enabled=True, doc="State variable"):

227 CollectableAttribute.__init__(self, name, doc) 228 self._isenabled = enabled 229 self._defaultenabled = enabled 230 if __debug__: 231 debug("STV", 232 "Initialized new state variable %s " % name + `self`)

233 234

235 - def _get(self):

236 if not self.isSet: 237 raise UnknownStateError("Unknown yet value of %s" % (self.name)) 238 return CollectableAttribute._get(self)

239 240

241 - def _set(self, val):

242 if self.isEnabled: 243 # XXX may be should have left simple assignment 244 # self._value = val 245 CollectableAttribute._set(self, val) 246 elif __debug__: 247 debug("COL", 248 "Not setting disabled %(self)s to %(val)s ", 249 msgargs={'self':self, 'val':val})

250 251

252 - def reset(self):

253 """Simply detach the value, and reset the flag""" 254 CollectableAttribute.reset(self) 255 self._value = None

256 257 258 @property

259 - def isEnabled(self):

260 return self._isenabled

261 262

263 - def enable(self, value=False):

264 if self._isenabled == value: 265 # Do nothing since it is already in proper state 266 return 267 if __debug__: 268 debug("STV", "%s %s" % 269 ({True: 'Enabling', False: 'Disabling'}[value], str(self))) 270 self._isenabled = value

271 272

273 - def __str__(self):

274 res = CollectableAttribute.__str__(self) 275 if self.isEnabled: 276 res += '+' # it is enabled but no value is assigned yet 277 return res

278 279 280 ################################################################### 281 # Collections 282 # 283 # TODO: refactor into attributes.py and collections.py. state.py now has 284 # little in common with the main part of this file 285 #

286 -class Collection(object):

287 """Container of some CollectableAttributes. 288 289 :Groups: 290 - `Public Access Functions`: `isKnown` 291 - `Access Implementors`: `_getListing`, `_getNames` 292 - `Mutators`: `__init__` 293 - `R/O Properties`: `listing`, `names`, `items` 294 295 XXX Seems to be not used and duplicating functionality: `_getListing` 296 (thus `listing` property) 297 """ 298

299 - def __init__(self, items=None, owner=None, name=None):

300 """Initialize the Collection 301 302 :Parameters: 303 items : dict of CollectableAttribute's 304 items to initialize with 305 owner : object 306 an object to which collection belongs 307 name : basestring 308 name of the collection (as seen in the owner, e.g. 'states') 309 """ 310 311 self.__owner = owner 312 313 if items == None: 314 items = {} 315 self._items = items 316 """Dictionary to contain registered states as keys and 317 values signal either they are enabled 318 """ 319 self.__name = name

320

321 - def _setName(self, name):

322 self.__name = name

323

324 - def __str__(self):

325 num = len(self._items) 326 if __debug__ and "ST" in debug.active: 327 maxnumber = 1000 # I guess all 328 else: 329 maxnumber = 4 330 if self.__name is not None: 331 res = self.__name 332 else: 333 res = "" 334 res += "{" 335 for i in xrange(min(num, maxnumber)): 336 if i > 0: 337 res += " " 338 res += "%s" % str(self._items.values()[i]) 339 if len(self._items) > maxnumber: 340 res += "..." 341 res += "}" 342 if __debug__: 343 if "ST" in debug.active: 344 res += " owner:%s#%s" % (self.owner.__class__.__name__, 345 id(self.owner)) 346 return res

347 348

349 - def _cls_repr(self):

350 """Collection specific part of __repr__ for a class containing 351 it, ie a part of __repr__ for the owner object 352 353 :Return: 354 list of items to be appended within __repr__ after a .join() 355 """ 356 # XXX For now we do not expect any pure non-specialized 357 # collection , thus just override in derived classes 358 raise NotImplementedError, "Class %s should override _cls_repr" \ 359 % self.__class__.__name__

360

361 - def _is_initializable(self, index):

362 """Checks if index could be assigned within collection via 363 _initialize 364 365 :Return: bool value for a given `index` 366 367 It is to facilitate dynamic assignment of collections' items 368 within derived classes' __init__ depending on the present 369 collections in the class. 370 """ 371 # XXX Each collection has to provide what indexes it allows 372 # to be set within constructor. Custom handling of some 373 # arguments (like (dis|en)able_states) is to be performed 374 # in _initialize 375 # raise NotImplementedError, \ 376 # "Class %s should override _is_initializable" \ 377 # % self.__class__.__name__ 378 379 # YYY lets just check if it is in the keys 380 return index in self._items.keys()

381 382

383 - def _initialize(self, index, value):

384 """Initialize `index` (no check performed) with `value` 385 """ 386 # by default we just set corresponding value 387 self.setvalue(index, value)

388 389

390 - def __repr__(self):

391 s = "%s(" % self.__class__.__name__ 392 items_s = "" 393 sep = "" 394 for item in self._items: 395 try: 396 itemvalue = "%s" % `self._items[item].value` 397 if len(itemvalue)>50: 398 itemvalue = itemvalue[:10] + '...' + itemvalue[-10:] 399 items_s += "%s'%s':%s" % (sep, item, itemvalue) 400 sep = ', ' 401 except: 402 pass 403 if items_s != "": 404 s += "items={%s}" % items_s 405 if self.owner is not None: 406 s += "%sowner=%s" % (sep, `self.owner`) 407 s += ")" 408 return s

409 410 411 # 412 # XXX TODO: figure out if there is a way to define proper 413 # __copy__'s for a hierarchy of classes. Probably we had 414 # to define __getinitargs__, etc... read more... 415 # 416 #def __copy__(self): 417 # TODO Remove or refactor? 418 # def _copy_states_(self, fromstate, deep=False): 419 # """Copy known here states from `fromstate` object into current object 420 # 421 # Crafted to overcome a problem mentioned above in the comment 422 # and is to be called from __copy__ of derived classes 423 # 424 # Probably sooner than later will get proper __getstate__, 425 # __setstate__ 426 # """ 427 # # Bad check... doesn't generalize well... 428 # # if not issubclass(fromstate.__class__, self.__class__): 429 # # raise ValueError, \ 430 # # "Class %s is not subclass of %s, " % \ 431 # # (fromstate.__class__, self.__class__) + \ 432 # # "thus not eligible for _copy_states_" 433 # # TODO: FOR NOW NO TEST! But this beast needs to be fixed... 434 # operation = { True: copy.deepcopy, 435 # False: copy.copy }[deep] 436 # 437 # if isinstance(fromstate, Stateful): 438 # fromstate = fromstate.states 439 # 440 # self.enabled = fromstate.enabled 441 # for name in self.names: 442 # if fromstate.isKnown(name): 443 # self._items[name] = operation(fromstate._items[name]) 444 445

446 - def isKnown(self, index):

447 """Returns `True` if state `index` is known at all""" 448 return self._items.has_key(index)

449 450

451 - def __isSet1(self, index):

452 """Returns `True` if state `index` has value set""" 453 self._checkIndex(index) 454 return self._items[index].isSet

455 456

457 - def isSet(self, index=None):

458 """If item (or any in the present or listed) was set 459 460 :Parameters: 461 index : None or basestring or list of basestring 462 What items to check if they were set in the collection 463 """ 464 _items = self._items 465 if not (index is None): 466 if isinstance(index, basestring): 467 self._checkIndex(index) # process just that single index 468 return _items[index].isSet 469 else: 470 items = index # assume that we got some list 471 else: 472 items = self._items # go through all the items 473 474 for index in items: 475 self._checkIndex(index) 476 if _items[index].isSet: 477 return True 478 return False

479 480

481 - def whichSet(self):

482 """Return list of indexes which were set""" 483 result = [] 484 # go through all members and if any isSet -- return True 485 for index,v in self._items.iteritems(): 486 if v.isSet: 487 result.append(index) 488 return result

489 490

491 - def _checkIndex(self, index):

492 """Verify that given `index` is a known/registered state. 493 494 :Raise `KeyError`: if given `index` is not known 495 """ 496 # OPT: lets not reuse isKnown, to don't incure 1 more function 497 # call 498 if not self._items.has_key(index): 499 raise KeyError, \ 500 "%s of %s has no key '%s' registered" \ 501 % (self.__class__.__name__, 502 self.__owner.__class__.__name__, 503 index)

504 505

506 - def add(self, item):

507 """Add a new CollectableAttribute to the collection 508 509 :Parameters: 510 item : CollectableAttribute 511 or of derived class. Must have 'name' assigned 512 513 TODO: we should make it stricter to don't add smth of 514 wrong type into Collection since it might lead to problems 515 516 Also we might convert to __setitem__ 517 """ 518 if not isinstance(item, CollectableAttribute): 519 raise ValueError, \ 520 "Collection can add only instances of " + \ 521 "CollectableAttribute-derived classes. Got %s" % `item` 522 if item.name is None: 523 raise ValueError, \ 524 "CollectableAttribute to be added %s must have 'name' set" % \ 525 item 526 self._items[item.name] = item 527 528 if not self.owner is None: 529 self._updateOwner(item.name)

530 531

532 - def remove(self, index):

533 """Remove item from the collection 534 """ 535 self._checkIndex(index) 536 self._updateOwner(index, register=False) 537 discard = self._items.pop(index)

538 539

540 - def __getattribute__(self, index):

541 """ 542 """ 543 #return all private and protected ones first since we will not have 544 # collectable's with _ (we should not have!) 545 if index[0] == '_': 546 return _object_getattribute(self, index) 547 _items = _object_getattribute(self, '_items') 548 if index in _items: 549 return _items[index].value 550 return _object_getattribute(self, index)

551 552

553 - def __setattr__(self, index, value):

554 if index[0] == '_': 555 return _object_setattr(self, index, value) 556 _items = _object_getattribute(self, '_items') 557 if index in _items: 558 _items[index].value = value 559 else: 560 _object_setattr(self, index, value)

561 562

563 - def __getitem__(self, index):

564 _items = _object_getattribute(self, '_items') 565 if index in _items: 566 self._checkIndex(index) 567 return _items[index] 568 else: 569 raise AttributeError("State collection %s has no %s attribute" 570 % (self, index))

571 572 573 # Probably not needed -- enable if need arises 574 # 575 #def __setattr__(self, index, value): 576 # if self._items.has_key(index): 577 # self._updateOwner(index, register=False) 578 # self._items[index] = value 579 # self._updateOwner(index, register=True) 580 # 581 # _object_setattr(self, index, value) 582 583

584 - def getvalue(self, index):

585 """Returns the value by index""" 586 self._checkIndex(index) 587 return self._items[index].value

588 589

590 - def get(self, index, default):

591 """Access the value by a given index. 592 593 Mimiquing regular dictionary behavior, if value cannot be obtained 594 (i.e. if any exception is caught) return default value. 595 """ 596 try: 597 return self[index].value 598 except Exception, e: 599 #if default is not None: 600 return default

601 #else: 602 # raise e 603 604

605 - def setvalue(self, index, value):

606 """Sets the value by index""" 607 self._checkIndex(index) 608 self._items[index].value = value

609 610

611 - def _action(self, index, func, missingok=False, **kwargs):

612 """Run specific func either on a single item or on all of them 613 614 :Parameters: 615 index : basestr 616 Name of the state variable 617 func 618 Function (not bound) to call given an item, and **kwargs 619 missingok : bool 620 If True - do not complain about wrong index 621 """ 622 if isinstance(index, basestring): 623 if index.upper() == 'ALL': 624 for index_ in self._items: 625 self._action(index_, func, missingok=missingok, **kwargs) 626 else: 627 try: 628 self._checkIndex(index) 629 func(self._items[index], **kwargs) 630 except: 631 if missingok: 632 return 633 raise 634 elif operator.isSequenceType(index): 635 for item in index: 636 self._action(item, func, missingok=missingok, **kwargs) 637 else: 638 raise ValueError, \ 639 "Don't know how to handle variable given by %s" % index

640 641

642 - def reset(self, index=None):

643 """Reset the state variable defined by `index`""" 644 645 if not index is None: 646 indexes = [ index ] 647 else: 648 indexes = self.names 649 650 if len(self.items): 651 for index in indexes: 652 # XXX Check if that works as desired 653 self._action(index, self._items.values()[0].__class__.reset, 654 missingok=False)

655 656

657 - def _getListing(self):

658 """Return a list of registered states along with the documentation""" 659 660 # lets assure consistent litsting order 661 items = self._items.items() 662 items.sort() 663 return [ "%s%s%s: %s" % (_def_sep, str(x[1]), _def_sep, x[1].__doc__) 664 for x in items ]

665 666

667 - def _getNames(self):

668 """Return ids for all registered state variables""" 669 return self._items.keys()

670 671

672 - def _getOwner(self):

673 return self.__owner

674 675

676 - def _setOwner(self, owner):

677 if not isinstance(owner, Stateful): 678 raise ValueError, \ 679 "Owner of the StateCollection must be Stateful object" 680 if __debug__: 681 try: strowner = str(owner) 682 except: strowner = "UNDEF: <%s#%s>" % (owner.__class__, id(owner)) 683 debug("ST", "Setting owner for %s to be %s" % (self, strowner)) 684 if not self.__owner is None: 685 # Remove attributes which were registered to that owner previousely 686 self._updateOwner(register=False) 687 self.__owner = owner 688 if not self.__owner is None: 689 self._updateOwner(register=True)

690 691

692 - def _updateOwner(self, index=None, register=True):

693 """Define an entry within owner's __dict__ 694 so ipython could easily complete it 695 696 :Parameters: 697 index : basestring or list of basestring 698 Name of the attribute. If None -- all known get registered 699 register : bool 700 Register if True or unregister if False 701 702 XXX Needs refactoring since we duplicate the logic of expansion of 703 index value 704 """ 705 if not index is None: 706 if not index in self._items: 707 raise ValueError, \ 708 "Attribute %s is not known to %s" % (index, self) 709 indexes = [ index ] 710 else: 711 indexes = self.names 712 713 ownerdict = self.owner.__dict__ 714 selfdict = self.__dict__ 715 owner_known = ownerdict['_known_attribs'] 716 for index_ in indexes: 717 if register: 718 if index_ in ownerdict: 719 raise RuntimeError, \ 720 "Cannot register attribute %s within %s " % \ 721 (index_, self.owner) + "since it has one already" 722 ownerdict[index_] = self._items[index_] 723 if index_ in selfdict: 724 raise RuntimeError, \ 725 "Cannot register attribute %s within %s " % \ 726 (index_, self) + "since it has one already" 727 selfdict[index_] = self._items[index_] 728 owner_known[index_] = self.__name 729 else: 730 if index_ in ownerdict: 731 # yoh doesn't think that we need to complain if False 732 ownerdict.pop(index_) 733 owner_known.pop(index_) 734 if index_ in selfdict: 735 selfdict.pop(index_)

736 737 738 # Properties 739 names = property(fget=_getNames) 740 items = property(fget=lambda x:x._items) 741 owner = property(fget=_getOwner, fset=_setOwner) 742 name = property(fget=lambda x:x.__name, fset=_setName) 743 744 # Virtual properties 745 listing = VProperty(fget=_getListing)

746 747 748

749 -class ParameterCollection(Collection):

750 """Container of Parameters for a stateful object. 751 """ 752 753 # def __init__(self, items=None, owner=None, name=None): 754 # """Initialize the state variables of a derived class 755 # 756 # :Parameters: 757 # items : dict 758 # dictionary of states 759 # """ 760 # Collection.__init__(self, items, owner, name) 761 # 762

763 - def _cls_repr(self):

764 """Part of __repr__ for the owner object 765 """ 766 prefixes = [] 767 for k in self.names: 768 # list only params with not default values 769 if self[k].isDefault: 770 continue 771 prefixes.append("%s=%s" % (k, self[k].value)) 772 return prefixes

773 774

775 - def resetvalue(self, index, missingok=False):

776 """Reset all parameters to default values""" 777 from param import Parameter 778 self._action(index, Parameter.resetvalue, missingok=False)

779 780

781 -class SampleAttributesCollection(Collection):

782 """Container for data and attributes of samples (ie data/labels/chunks/...) 783 """ 784 785 # def __init__(self, items=None, owner=None, name=None): 786 # """Initialize the state variables of a derived class 787 # 788 # :Parameters: 789 # items : dict 790 # dictionary of states 791 # """ 792 # Collection.__init__(self, items, owner, name) 793 # 794

795 - def _cls_repr(self):

796 """Part of __repr__ for the owner object 797 """ 798 return [] # TODO: return I guess samples/labels/chunks

799 800 801

802 -class StateCollection(Collection):

803 """Container of StateVariables for a stateful object. 804 805 :Groups: 806 - `Public Access Functions`: `isKnown`, `isEnabled`, `isActive` 807 - `Access Implementors`: `_getListing`, `_getNames`, `_getEnabled` 808 - `Mutators`: `__init__`, `enable`, `disable`, `_setEnabled` 809 - `R/O Properties`: `listing`, `names`, `items` 810 - `R/W Properties`: `enabled` 811 """ 812

813 - def __init__(self, items=None, owner=None):

814 """Initialize the state variables of a derived class 815 816 :Parameters: 817 items : dict 818 dictionary of states 819 owner : Stateful 820 object which owns the collection 821 name : basestring 822 literal description. Usually just attribute name for the 823 collection, e.g. 'states' 824 """ 825 Collection.__init__(self, items=items, owner=owner) 826 827 self.__storedTemporarily = [] 828 """List to contain sets of enabled states which were enabled 829 temporarily. 830 """

831 832 # 833 # XXX TODO: figure out if there is a way to define proper 834 # __copy__'s for a hierarchy of classes. Probably we had 835 # to define __getinitargs__, etc... read more... 836 # 837 #def __copy__(self): 838

839 - def _cls_repr(self):

840 """Part of __repr__ for the owner object 841 """ 842 prefixes = [] 843 for name, invert in ( ('enable', False), ('disable', True) ): 844 states = self._getEnabled(nondefault=False, 845 invert=invert) 846 if len(states): 847 prefixes.append("%s_states=%s" % (name, str(states))) 848 return prefixes

849 850

851 - def _is_initializable(self, index):

852 """Checks if index could be assigned within collection via 853 setvalue 854 """ 855 return index in ['enable_states', 'disable_states']

856 857

858 - def _initialize(self, index, value):

859 if value is None: 860 value = [] 861 if index == 'enable_states': 862 self.enable(value, missingok=True) 863 elif index == 'disable_states': 864 self.disable(value) 865 else: 866 raise ValueError, "StateCollection can accept only enable_states " \ 867 "and disable_states arguments for the initialization. " \ 868 "Got %s" % index

869 870

871 - def _copy_states_(self, fromstate, index=None, deep=False):

872 """Copy known here states from `fromstate` object into current object 873 874 :Parameters: 875 fromstate : Collection or Stateful 876 Source states to copy from 877 index : None or list of basestring 878 If not to copy all set state variables, index provides 879 selection of what to copy 880 deep : bool 881 Optional control over the way to copy 882 883 Crafted to overcome a problem mentioned above in the comment 884 and is to be called from __copy__ of derived classes 885 886 Probably sooner than later will get proper __getstate__, 887 __setstate__ 888 """ 889 # Bad check... doesn't generalize well... 890 # if not issubclass(fromstate.__class__, self.__class__): 891 # raise ValueError, \ 892 # "Class %s is not subclass of %s, " % \ 893 # (fromstate.__class__, self.__class__) + \ 894 # "thus not eligible for _copy_states_" 895 # TODO: FOR NOW NO TEST! But this beast needs to be fixed... 896 operation = { True: copy.deepcopy, 897 False: copy.copy }[deep] 898 899 if isinstance(fromstate, Stateful): 900 fromstate = fromstate.states 901 902 #self.enabled = fromstate.enabled 903 _items, from_items = self._items, fromstate._items 904 if index is None: 905 # copy all set ones 906 for name in fromstate.whichSet():#self.names: 907 #if fromstate.isKnown(name): 908 _items[name] = operation(from_items[name]) 909 else: 910 isKnown = fromstate.isKnown 911 for name in index: 912 if isKnown(name): 913 _items[name] = operation(from_items[name])

914 915

916 - def isEnabled(self, index):

917 """Returns `True` if state `index` is enabled""" 918 self._checkIndex(index) 919 return self._items[index].isEnabled

920 921

922 - def isActive(self, index):

923 """Returns `True` if state `index` is known and is enabled""" 924 return self.isKnown(index) and self.isEnabled(index)

925 926

927 - def enable(self, index, value=True, missingok=False):

928 """Enable state variable given in `index`""" 929 self._action(index, StateVariable.enable, missingok=missingok, 930 value=value)

931 932

933 - def disable(self, index):

934 """Disable state variable defined by `index` id""" 935 self._action(index, StateVariable.enable, missingok=False, value=False)

936 937 938 # TODO XXX think about some more generic way to grab temporary 939 # snapshot of CollectableAttributes to be restored later on...

940 - def _changeTemporarily(self, enable_states=None, 941 disable_states=None, other=None):

942 """Temporarily enable/disable needed states for computation 943 944 Enable or disable states which are enabled in `other` and listed in 945 `enable _states`. Use `resetEnabledTemporarily` to reset 946 to previous state of enabled. 947 948 `other` can be a Stateful object or StateCollection 949 """ 950 if enable_states == None: 951 enable_states = [] 952 if disable_states == None: 953 disable_states = [] 954 self.__storedTemporarily.append(self.enabled) 955 other_ = other 956 if isinstance(other, Stateful): 957 other = other.states 958 959 if not other is None: 960 # lets take states which are enabled in other but not in 961 # self 962 add_enable_states = list(Set(other.enabled).difference( 963 Set(enable_states)).intersection(self.names)) 964 if len(add_enable_states)>0: 965 if __debug__: 966 debug("ST", 967 "Adding states %s from %s to be enabled temporarily" % 968 (add_enable_states, other_) + 969 " since they are not enabled in %s" % 970 (self)) 971 enable_states += add_enable_states 972 973 # Lets go one by one enabling only disabled once... but could be as 974 # simple as 975 self.enable(enable_states) 976 self.disable(disable_states)

977 978

979 - def _resetEnabledTemporarily(self):

980 """Reset to previousely stored set of enabled states""" 981 if __debug__: 982 debug("ST", "Resetting to previous set of enabled states") 983 if len(self.enabled)>0: 984 self.enabled = self.__storedTemporarily.pop() 985 else: 986 raise ValueError("Trying to restore not-stored list of enabled " \ 987 "states")

988 989

990 - def _getEnabled(self, nondefault=True, invert=False):

991 """Return list of enabled states 992 993 :Parameters: 994 nondefault : bool 995 Either to return also states which are enabled simply by default 996 invert : bool 997 Would invert the meaning, ie would return disabled states 998 """ 999 if invert: 1000 fmatch = lambda y: not self.isEnabled(y) 1001 else: 1002 fmatch = lambda y: self.isEnabled(y) 1003 1004 if nondefault: 1005 ffunc = fmatch 1006 else: 1007 ffunc = lambda y: fmatch(y) and \ 1008 self._items[y]._defaultenabled != self.isEnabled(y) 1009 return filter(ffunc, self.names)

1010 1011

1012 - def _setEnabled(self, indexlist):

1013 """Given `indexlist` make only those in the list enabled 1014 1015 It might be handy to store set of enabled states and then to restore 1016 it later on. It can be easily accomplished now:: 1017 1018 >>> from mvpa.misc.state import Stateful, StateVariable 1019 >>> class Blah(Stateful): 1020 ... bleh = StateVariable(enabled=False, doc='Example') 1021 ... 1022 >>> blah = Blah() 1023 >>> states_enabled = blah.states.enabled 1024 >>> blah.states.enabled = ['bleh'] 1025 >>> blah.states.enabled = states_enabled 1026 """ 1027 for index in self._items.keys(): 1028 self.enable(index, index in indexlist)

1029 1030 1031 # Properties 1032 enabled = property(fget=_getEnabled, fset=_setEnabled)

1033 1034 1035 ################################################################## 1036 # Base classes (and metaclass) which use collections 1037 # 1038 1039 1040 # 1041 # Helper dictionaries for AttributesCollector 1042 # 1043 _known_collections = { 1044 # Quite a generic one but mostly in classifiers 1045 'StateVariable': ("states", StateCollection), 1046 # For classifiers only 1047 'Parameter': ("params", ParameterCollection), 1048 'KernelParameter': ("kernel_params", ParameterCollection), 1049 # For datasets 1050 # XXX custom collections needed? 1051 'SampleAttribute': ("s_attr", SampleAttributesCollection), 1052 'FeatureAttribute': ("f_attr", SampleAttributesCollection), 1053 'DatasetAttribute': ("ds_attr", SampleAttributesCollection), 1054 } 1055 1056 1057 _col2class = dict(_known_collections.values()) 1058 """Mapping from collection name into Collection class""" 1059 1060 1061 _COLLECTIONS_ORDER = ['s_attr', 'f_attr', 'ds_attr', 1062 'params', 'kernel_params', 'states'] 1063 1064

1065 -class AttributesCollector(type):

1066 """Intended to collect and compose StateCollection for any child 1067 class of this metaclass 1068 """ 1069 1070

1071 - def __init__(cls, name, bases, dict):

1072 1073 if __debug__: 1074 debug( 1075 "COLR", 1076 "AttributesCollector call for %s.%s, where bases=%s, dict=%s " \ 1077 % (cls, name, bases, dict)) 1078 1079 super(AttributesCollector, cls).__init__(name, bases, dict) 1080 1081 collections = {} 1082 for name, value in dict.iteritems(): 1083 if isinstance(value, CollectableAttribute): 1084 baseclassname = value.__class__.__name__ 1085 col = _known_collections[baseclassname][0] 1086 # XXX should we allow to throw exceptions here? 1087 if not collections.has_key(col): 1088 collections[col] = {} 1089 collections[col][name] = value 1090 # and assign name if not yet was set 1091 if value.name is None: 1092 value.name = name 1093 1094 # XXX can we first collect parent's states and then populate with ours? 1095 # TODO 1096 1097 for base in bases: 1098 if hasattr(base, "__metaclass__") and \ 1099 base.__metaclass__ == AttributesCollector: 1100 # TODO take care about overriding one from super class 1101 # for state in base.states: 1102 # if state[0] = 1103 newcollections = base._collections_template 1104 if len(newcollections) == 0: 1105 continue 1106 if __debug__: 1107 debug("COLR", 1108 "Collect collections %s for %s from %s" % 1109 (newcollections, cls, base)) 1110 for col, collection in newcollections.iteritems(): 1111 newitems = collection.items 1112 if collections.has_key(col): 1113 collections[col].update(newitems) 1114 else: 1115 collections[col] = newitems 1116 1117 1118 if __debug__: 1119 debug("COLR", 1120 "Creating StateCollection template %s with collections %s" 1121 % (cls, collections.keys())) 1122 1123 # if there is an explicit 1124 if hasattr(cls, "_ATTRIBUTE_COLLECTIONS"): 1125 for col in cls._ATTRIBUTE_COLLECTIONS: 1126 if not col in _col2class: 1127 raise ValueError, \ 1128 "Requested collection %s is unknown to collector" % \ 1129 col 1130 if not col in collections: 1131 collections[col] = None 1132 1133 # TODO: check on conflict in names of Collections' items! since 1134 # otherwise even order is not definite since we use dict for 1135 # collections. 1136 # XXX should we switch to tuple? 1137 1138 for col, colitems in collections.iteritems(): 1139 collections[col] = _col2class[col](colitems) 1140 1141 setattr(cls, "_collections_template", collections) 1142 1143 # 1144 # Expand documentation for the class based on the listed 1145 # parameters an if it is stateful 1146 # 1147 # TODO -- figure nice way on how to alter __init__ doc directly... 1148 textwrapper = TextWrapper(subsequent_indent=" ", 1149 initial_indent=" ", 1150 width=70) 1151 1152 # Parameters 1153 paramsdoc = "" 1154 paramscols = [] 1155 for col in ('params', 'kernel_params'): 1156 if collections.has_key(col): 1157 paramscols.append(col) 1158 # lets at least sort the parameters for consistent output 1159 col_items = collections[col].items 1160 params = [(v._instance_index, k) for k,v in col_items.iteritems()] 1161 params.sort() 1162 paramsdoc += '\n'.join( 1163 [col_items[param].doc(indent=' ') 1164 for index,param in params]) + '\n' 1165 1166 # Parameters collection could be taked hash of to decide if 1167 # any were changed? XXX may be not needed at all? 1168 setattr(cls, "_paramscols", paramscols) 1169 1170 # States doc 1171 statesdoc = "" 1172 if collections.has_key('states'): 1173 paramsdoc += """ enable_states : None or list of basestring 1174 Names of the state variables which should be enabled additionally 1175 to default ones 1176 disable_states : None or list of basestring 1177 Names of the state variables which should be disabled 1178 """ 1179 statesdoc = " * " 1180 statesdoc += '\n * '.join(collections['states'].listing) 1181 statesdoc += "\n\n(States enabled by default are listed with `+`)" 1182 if __debug__: 1183 debug("COLR", "Assigning __statesdoc to be %s" % statesdoc) 1184 setattr(cls, "_statesdoc", statesdoc) 1185 1186 if paramsdoc != "": 1187 if __debug__ and 'COLR' in debug.active: 1188 debug("COLR", "Assigning __paramsdoc to be %s" % paramsdoc) 1189 setattr(cls, "_paramsdoc", paramsdoc) 1190 1191 if paramsdoc + statesdoc != "": 1192 cls.__doc__ = enhancedDocString(cls, *bases)

1193 1194 1195

1196 -class ClassWithCollections(object):

1197 """Base class for objects which contain any known collection 1198 1199 Classes inherited from this class gain ability to access 1200 collections and their items as simple attributes. Access to 1201 collection items "internals" is done via <collection_name> attribute 1202 and interface of a corresponding `Collection`. 1203 """ 1204 1205 _DEV__doc__ = """ 1206 TODO: rename 'descr'? -- it should simply 1207 be 'doc' -- no need to drag classes docstring imho. 1208 """ 1209 1210 __metaclass__ = AttributesCollector 1211

1212 - def __new__(cls, *args, **kwargs):

1213 """Initialize ClassWithCollections object 1214 1215 :Parameters: 1216 descr : basestring 1217 Description of the instance 1218 """ 1219 self = super(ClassWithCollections, cls).__new__(cls) 1220 1221 s__dict__ = self.__dict__ 1222 1223 # init variable 1224 # XXX: Added as pylint complained (rightfully) -- not sure if false 1225 # is the proper default 1226 self.__params_set = False 1227 1228 # need to check to avoid override of enabled states in the case 1229 # of multiple inheritance, like both Statefull and Harvestable 1230 if not s__dict__.has_key('_collections'): 1231 s__class__ = self.__class__ 1232 1233 collections = copy.deepcopy(s__class__._collections_template) 1234 s__dict__['_collections'] = collections 1235 s__dict__['_known_attribs'] = {} 1236 """Dictionary to contain 'links' to the collections from each 1237 known attribute. Is used to gain some speed up in lookup within 1238 __getattribute__ and __setattr__ 1239 """ 1240 1241 # Assign owner to all collections 1242 for col, collection in collections.iteritems(): 1243 if col in s__dict__: 1244 raise ValueError, \ 1245 "Object %s has already attribute %s" % \ 1246 (self, col) 1247 s__dict__[col] = collection 1248 collection.name = col 1249 collection.owner = self 1250 1251 self.__params_set = False 1252 1253 if __debug__: 1254 descr = kwargs.get('descr', None) 1255 debug("COL", "ClassWithCollections.__new__ was done " 1256 "for %s#%s with descr=%s" \ 1257 % (s__class__.__name__, id(self), descr)) 1258 1259 return self

1260 1261

1262 - def __init__(self, descr=None, **kwargs):

1263 1264 if not self.__params_set: 1265 self.__descr = descr 1266 """Set humane description for the object""" 1267 1268 # To avoid double initialization in case of multiple inheritance 1269 self.__params_set = True 1270 1271 collections = self._collections 1272 # Assign attributes values if they are given among 1273 # **kwargs 1274 for arg, argument in kwargs.items(): 1275 set = False 1276 for collection in collections.itervalues(): 1277 if collection._is_initializable(arg): 1278 collection._initialize(arg, argument) 1279 set = True 1280 break 1281 if set: 1282 trash = kwargs.pop(arg) 1283 else: 1284 known_params = reduce( 1285 lambda x,y:x+y, 1286 [x.items.keys() for x in collections.itervalues()], []) 1287 raise TypeError, \ 1288 "Unexpected keyword argument %s=%s for %s." \ 1289 % (arg, argument, self) \ 1290 + " Valid parameters are %s" % known_params 1291 1292 ## Initialize other base classes 1293 ## commented out since it seems to be of no use for now 1294 #if init_classes is not None: 1295 # # return back stateful arguments since they might be 1296 # # processed by underlying classes 1297 # kwargs.update(kwargs_stateful) 1298 # for cls in init_classes: 1299 # cls.__init__(self, **kwargs) 1300 #else: 1301 # if len(kwargs)>0: 1302 # known_params = reduce(lambda x, y: x + y, \ 1303 # [x.items.keys() for x in collections], 1304 # []) 1305 # raise TypeError, \ 1306 # "Unknown parameters %s for %s." % (kwargs.keys(), 1307 # self) \ 1308 # + " Valid parameters are %s" % known_params 1309 if __debug__: 1310 debug("COL", "ClassWithCollections.__init__ was done " 1311 "for %s#%s with descr=%s" \ 1312 % (self.__class__.__name__, id(self), descr))

1313 1314 1315 #__doc__ = enhancedDocString('Stateful', locals()) 1316 1317

1318 - def __getattribute__(self, index):

1319 # return all private ones first since smth like __dict__ might be 1320 # queried by copy before instance is __init__ed 1321 if index[0] == '_': 1322 return _object_getattribute(self, index) 1323 1324 s_dict = _object_getattribute(self, '__dict__') 1325 # check if it is a known collection 1326 collections = s_dict['_collections'] 1327 if index in collections: 1328 return collections[index] 1329 1330 # check if it is a part of any collection 1331 known_attribs = s_dict['_known_attribs'] 1332 if index in known_attribs: 1333 return collections[known_attribs[index]].getvalue(index) 1334 1335 # just a generic return 1336 return _object_getattribute(self, index)

1337 1338

1339 - def __setattr__(self, index, value):

1340 if index[0] == '_': 1341 return _object_setattr(self, index, value) 1342 1343 # Check if a part of a collection, and set appropriately 1344 s_dict = _object_getattribute(self, '__dict__') 1345 known_attribs = s_dict['_known_attribs'] 1346 if index in known_attribs: 1347 collections = s_dict['_collections'] 1348 return collections[known_attribs[index]].setvalue(index, value) 1349 1350 # Generic setattr 1351 return _object_setattr(self, index, value)

1352 1353 1354 # XXX not sure if we shouldn't implement anything else...

1355 - def reset(self):

1356 for collection in self._collections.values(): 1357 collection.reset()

1358 1359

1360 - def __str__(self):

1361 s = "%s:" % (self.__class__.__name__) 1362 if self.__descr is not None: 1363 s += "/%s " % self.__descr 1364 if hasattr(self, "_collections"): 1365 for col, collection in self._collections.iteritems(): 1366 s += " %d %s:%s" % (len(collection.items), col, str(collection)) 1367 return s

1368 1369

1370 - def __repr__(self, prefixes=None, fullname=False):

1371 """String definition of the object of ClassWithCollections object 1372 1373 :Parameters: 1374 fullname : bool 1375 Either to include full name of the module 1376 prefixes : list of strings 1377 What other prefixes to prepend to list of arguments 1378 """ 1379 if prefixes is None: 1380 prefixes = [] 1381 prefixes = prefixes[:] # copy list 1382 id_str = "" 1383 module_str = "" 1384 if __debug__: 1385 if 'MODULE_IN_REPR' in debug.active: 1386 fullname = True 1387 if 'ID_IN_REPR' in debug.active: 1388 id_str = '#%s' % id(self) 1389 1390 if fullname: 1391 modulename = '%s' % self.__class__.__module__ 1392 if modulename != "__main__": 1393 module_str = "%s." % modulename 1394 1395 # Collections' attributes 1396 collections = self._collections 1397 # we want them in this particular order 1398 for col in _COLLECTIONS_ORDER: 1399 collection = collections.get(col, None) 1400 if collection is None: 1401 continue 1402 prefixes += collection._cls_repr() 1403 1404 # Description if present 1405 descr = self.__descr 1406 if descr is not None: 1407 prefixes.append("descr='%s'" % (descr)) 1408 1409 return "%s%s(%s)%s" % (module_str, self.__class__.__name__, 1410 ', '.join(prefixes), id_str)

1411 1412 1413 descr = property(lambda self: self.__descr, 1414 doc="Description of the object if any")

1415 1416 1417 1418 # No actual separation is needed now between ClassWithCollections 1419 # and a specific usecase. 1420 Stateful = ClassWithCollections 1421 Parametrized = ClassWithCollections 1422 1423 1424

1425 -class Harvestable(Stateful):

1426 """Classes inherited from this class intend to collect attributes 1427 within internal processing. 1428 1429 Subclassing Harvestable we gain ability to collect any internal 1430 data from the processing which is especially important if an 1431 object performs something in loop and discards some intermidiate 1432 possibly interesting results (like in case of 1433 CrossValidatedTransferError and states of the trained classifier 1434 or TransferError). 1435 1436 """ 1437 1438 harvested = StateVariable(enabled=False, doc= 1439 """Store specified attributes of classifiers at each split""") 1440 1441 _KNOWN_COPY_METHODS = [ None, 'copy', 'deepcopy' ] 1442 1443

1444 - def __init__(self, harvest_attribs=None, copy_attribs='copy', **kwargs):

1445 """Initialize state of harvestable 1446 1447 :Parameters: 1448 harvest_attribs : list of basestr or dicts 1449 What attributes of call to store and return within 1450 harvested state variable. If an item is a dictionary, 1451 following keys are used ['name', 'copy'] 1452 copy_attribs : None or basestr 1453 Default copying. If None -- no copying, 'copy' 1454 - shallow copying, 'deepcopy' -- deepcopying 1455 1456 """ 1457 Stateful.__init__(self, **kwargs) 1458 1459 self.__atribs = harvest_attribs 1460 self.__copy_attribs = copy_attribs 1461 1462 self._setAttribs(harvest_attribs)

1463 1464

1465 - def _setAttribs(self, attribs):

1466 """Set attributes to harvest 1467 1468 Each attribute in self.__attribs must have following fields 1469 - name : functional (or arbitrary if 'obj' or 'attr' is set) 1470 description of the thing to harvest, 1471 e.g. 'transerror.clf.training_time' 1472 - obj : name of the object to harvest from (if empty, 1473 'self' is assumed), 1474 e.g 'transerror' 1475 - attr : attribute of 'obj' to harvest, 1476 e.g. 'clf.training_time' 1477 - copy : None, 'copy' or 'deepcopy' - way to copy attribute 1478 """ 1479 if attribs: 1480 # force the state 1481 self.states.enable('harvested') 1482 self.__attribs = [] 1483 for i, attrib in enumerate(attribs): 1484 if isinstance(attrib, dict): 1485 if not 'name' in attrib: 1486 raise ValueError, \ 1487 "Harvestable: attribute must be a string or " + \ 1488 "a dictionary with 'name'" 1489 else: 1490 attrib = {'name': attrib} 1491 1492 # assign default method to copy 1493 if not 'copy' in attrib: 1494 attrib['copy'] = self.__copy_attribs 1495 1496 # check copy method 1497 if not attrib['copy'] in self._KNOWN_COPY_METHODS: 1498 raise ValueError, "Unknown method %s. Known are %s" % \ 1499 (attrib['copy'], self._KNOWN_COPY_METHODS) 1500 1501 if not ('obj' in attrib or 'attr' in attrib): 1502 # Process the item to harvest 1503 # split into obj, attr. If obj is empty, then assume self 1504 split = attrib['name'].split('.', 1) 1505 if len(split)==1: 1506 obj, attr = split[0], None 1507 else: 1508 obj, attr = split 1509 attrib.update({'obj':obj, 'attr':attr}) 1510 1511 if attrib['obj'] == '': 1512 attrib['obj'] = 'self' 1513 1514 # TODO: may be enabling of the states?? 1515 1516 self.__attribs.append(attrib) # place value back 1517 else: 1518 # just to make sure it is not None or 0 1519 self.__attribs = []

1520 1521

1522 - def _harvest(self, vars):

1523 """The harvesting function: must obtain dictionary of variables 1524 from the caller. 1525 1526 :Parameters: 1527 vars : dict 1528 Dictionary of available data. Most often locals() could be 1529 passed as `vars`. Mention that desired to be harvested 1530 private attributes better be bound locally to some variable 1531 1532 :Returns: 1533 nothing 1534 """ 1535 1536 if not self.states.isEnabled('harvested') or len(self.__attribs)==0: 1537 return 1538 1539 if not self.states.isSet('harvested'): 1540 self.harvested = dict([(a['name'], []) for a in self.__attribs]) 1541 1542 for attrib in self.__attribs: 1543 attrv = vars[attrib['obj']] 1544 1545 # access particular attribute if needed 1546 if not attrib['attr'] is None: 1547 attrv = eval('attrv.%s' % attrib['attr']) 1548 1549 # copy the value if needed 1550 attrv = {'copy':copy.copy, 1551 'deepcopy':copy.deepcopy, 1552 None:lambda x:x}[attrib['copy']](attrv) 1553 1554 self.harvested[attrib['name']].append(attrv)

1555 1556 1557 harvest_attribs = property(fget=lambda self:self.__attribs, 1558 fset=_setAttribs)

1559

Source Code for Module mvpa.misc.state