Source Code for Module mvpa.clfs.libsvmc.svm

  1  #emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  #ex: set sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Wrap the libsvm package into a very simple class interface.""" 
 10   
 11  __docformat__ = 'restructuredtext' 
 12   
 13  import numpy as N 
 14   
 15  import operator 
 16   
 17  from mvpa.misc.param import Parameter 
 18  from mvpa.base import warning, externals 
 19  from mvpa.misc.state import StateVariable 
 20   
 21  from mvpa.clfs.base import Classifier 
 22  from mvpa.clfs._svmbase import _SVM 
 23  from mvpa.measures.base import Sensitivity 
 24   
 25  from mvpa.clfs.libsvmc import _svm as svm 
 26  from sens import * 
 27   
 28  if __debug__: 
 29      from mvpa.base import debug 
 30   
 31  # we better expose those since they are mentioned in docstrings 
 32  from mvpa.clfs.libsvmc._svmc import \ 
 33       C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, \ 
 34       NU_SVR, LINEAR, POLY, RBF, SIGMOID, \ 
 35       PRECOMPUTED 
 36   
 37   
 38 -class SVM(_SVM): 
 39      """Support Vector Machine Classifier. 
 40   
 41      This is a simple interface to the libSVM package. 
 42      """ 
 43   
 44      # Since this is internal feature of LibSVM, this state variable is present 
 45      # here 
 46      probabilities = StateVariable(enabled=False, 
 47          doc="Estimates of samples probabilities as provided by LibSVM") 
 48   
 49      _KERNELS = { "linear":  (svm.svmc.LINEAR, None, LinearSVMWeights), 
 50                   "rbf" :    (svm.svmc.RBF, ('gamma',), None), 
 51                   "poly":    (svm.svmc.POLY, ('gamma', 'degree', 'coef0'), None), 
 52                   "sigmoid": (svm.svmc.SIGMOID, ('gamma', 'coef0'), None), 
 53                   } 
 54      # TODO: Complete the list ;-) 
 55   
 56      # TODO p is specific for SVR 
 57      _KNOWN_PARAMS = [ 'epsilon', 'probability', 'shrinking', 
 58                        'weight_label', 'weight'] 
 59   
 60      _KNOWN_KERNEL_PARAMS = [ 'cache_size' ] 
 61   
 62      _KNOWN_IMPLEMENTATIONS = { 
 63          'C_SVC' : (svm.svmc.C_SVC, ('C',), 
 64                     ('binary', 'multiclass'), 'C-SVM classification'), 
 65          'NU_SVC' : (svm.svmc.NU_SVC, ('nu',), 
 66                      ('binary', 'multiclass'), 'nu-SVM classification'), 
 67          'ONE_CLASS' : (svm.svmc.ONE_CLASS, (), 
 68                         ('oneclass',), 'one-class-SVM'), 
 69          'EPSILON_SVR' : (svm.svmc.EPSILON_SVR, ('C', 'tube_epsilon'), 
 70                           ('regression',), 'epsilon-SVM regression'), 
 71          'NU_SVR' : (svm.svmc.NU_SVR, ('nu', 'tube_epsilon'), 
 72                      ('regression',), 'nu-SVM regression') 
 73          } 
 74   
 75      _clf_internals = _SVM._clf_internals + [ 'libsvm' ] 
 76   
 77 -    def __init__(self, 
 78                   kernel_type='linear', 
 79                   **kwargs): 
 80          # XXX Determine which parameters depend on each other and implement 
 81          # safety/simplifying logic around them 
 82          # already done for: nr_weight 
 83          # thought: weight and weight_label should be a dict 
 84          """Interface class to LIBSVM classifiers and regressions. 
 85   
 86          Default implementation (C/nu/epsilon SVM) is chosen depending 
 87          on the given parameters (C/nu/tube_epsilon). 
 88          """ 
 89   
 90          svm_impl = kwargs.get('svm_impl', None) 
 91          # Depending on given arguments, figure out desired SVM 
 92          # implementation 
 93          if svm_impl is None: 
 94              for arg, impl in [ ('tube_epsilon', 'EPSILON_SVR'), 
 95                                 ('C', 'C_SVC'), 
 96                                 ('nu', 'NU_SVC') ]: 
 97                  if kwargs.has_key(arg): 
 98                      svm_impl = impl 
 99                      if __debug__: 
100                          debug('SVM', 'No implementation was specified. Since ' 
101                                '%s is given among arguments, assume %s' % 
102                                (arg, impl)) 
103                      break 
104              if svm_impl is None: 
105                  svm_impl = 'C_SVC' 
106                  if __debug__: 
107                        debug('SVM', 'Assign C_SVC "by default"') 
108          kwargs['svm_impl'] = svm_impl 
109   
110          # init base class 
111          _SVM.__init__(self, kernel_type, **kwargs) 
112   
113          self._svm_type = self._KNOWN_IMPLEMENTATIONS[svm_impl][0] 
114   
115          if 'nu' in self._KNOWN_PARAMS and 'epsilon' in self._KNOWN_PARAMS: 
116              # overwrite eps param with new default value (information taken from libSVM 
117              # docs 
118              self.params['epsilon'].setDefault(0.001) 
119   
120          self.__model = None 
121          """Holds the trained SVM.""" 
122   
123   
124   
125 -    def _train(self, dataset): 
126          """Train SVM 
127          """ 
128          # libsvm needs doubles 
129          if dataset.samples.dtype == 'float64': 
130              src = dataset.samples 
131          else: 
132              src = dataset.samples.astype('double') 
133   
134          svmprob = svm.SVMProblem( dataset.labels.tolist(), src ) 
135   
136          # Translate few params 
137          TRANSLATEDICT={'epsilon': 'eps', 
138                         'tube_epsilon': 'p'} 
139          args = [] 
140          for paramname, param in self.params.items.items() \ 
141                  + self.kernel_params.items.items(): 
142              if paramname in TRANSLATEDICT: 
143                  argname = TRANSLATEDICT[paramname] 
144              elif paramname in svm.SVMParameter.default_parameters: 
145                  argname = paramname 
146              else: 
147                  if __debug__: 
148                      debug("SVM_", "Skipping parameter %s since it is not known" 
149                            "to libsvm" % paramname) 
150                  continue 
151              args.append( (argname, param.value) ) 
152   
153          # XXX All those parameters should be fetched if present from 
154          # **kwargs and create appropriate parameters within .params or .kernel_params 
155          libsvm_param = svm.SVMParameter( 
156              kernel_type=self._kernel_type, 
157              svm_type=self._svm_type, 
158              **dict(args)) 
159          """Store SVM parameters in libSVM compatible format.""" 
160   
161          if self.params.isKnown('C'):#svm_type in [svm.svmc.C_SVC]: 
162              C = self.params.C 
163              if not operator.isSequenceType(C): 
164                  # we were not given a tuple for balancing between classes 
165                  C = [C] 
166   
167              Cs = list(C[:])               # copy 
168              for i in xrange(len(Cs)): 
169                  if Cs[i]<0: 
170                      Cs[i] = self._getDefaultC(dataset.samples)*abs(Cs[i]) 
171                      if __debug__: 
172                          debug("SVM", "Default C for %s was computed to be %s" % 
173                                (C[i], Cs[i])) 
174   
175              libsvm_param._setParameter('C', Cs[0]) 
176   
177              if len(Cs)>1: 
178                  C0 = abs(C[0]) 
179                  scale = 1.0/(C0)#*N.sqrt(C0)) 
180                  # so we got 1 C per label 
181                  if len(Cs) != len(dataset.uniquelabels): 
182                      raise ValueError, "SVM was parametrized with %d Cs but " \ 
183                            "there are %d labels in the dataset" % \ 
184                            (len(Cs), len(dataset.uniquelabels)) 
185                  weight = [ c*scale for c in Cs ] 
186                  libsvm_param._setParameter('weight', weight) 
187   
188          self.__model = svm.SVMModel(svmprob, libsvm_param) 
189   
190   
191 -    def _predict(self, data): 
192          """Predict values for the data 
193          """ 
194          # libsvm needs doubles 
195          if data.dtype == 'float64': 
196              src = data 
197          else: 
198              src = data.astype('double') 
199          states = self.states 
200   
201          predictions = [ self.model.predict(p) for p in src ] 
202   
203          if states.isEnabled("values"): 
204              if self.regression: 
205                  values = [ self.model.predictValuesRaw(p)[0] for p in src ] 
206              else: 
207                  trained_labels = self.trained_labels 
208                  nlabels = len(trained_labels) 
209                  # XXX We do duplicate work. model.predict calls predictValuesRaw 
210                  # internally and then does voting or thresholding. So if speed becomes 
211                  # a factor we might want to move out logic from libsvm over here to base 
212                  # predictions on obtined values, or adjust libsvm to spit out values from 
213                  # predict() as well 
214                  if nlabels == 2: 
215                      # Apperently libsvm reorders labels so we need to track (1,0) 
216                      # values instead of (0,1) thus just lets take negative reverse 
217                      values = [ self.model.predictValues(p)[(trained_labels[1], trained_labels[0])] for p in src ] 
218                      if len(values)>0: 
219                          if __debug__: 
220                              debug("SVM","Forcing values to be ndarray and reshaping " + 
221                                    "them to be 1D vector") 
222                          values = N.asarray(values).reshape(len(values)) 
223                  else: 
224                      # In multiclass we return dictionary for all pairs of labels, 
225                      # since libsvm does 1-vs-1 pairs 
226                      values = [ self.model.predictValues(p) for p in src ] 
227              states.values = values 
228   
229          if states.isEnabled("probabilities"): 
230              self.probabilities = [ self.model.predictProbability(p) for p in src ] 
231              try: 
232                  states.probabilities = [ self.model.predictProbability(p) for p in src ] 
233              except TypeError: 
234                  warning("Current SVM %s doesn't support probability estimation," % 
235                          self + " thus no 'values' state") 
236          return predictions 
237   
238   
239 -    def summary(self): 
240          """Provide quick summary over the SVM classifier""" 
241          s = super(SVM, self).summary() 
242          if self.trained: 
243              s += '\n # of SVs: %d' % self.__model.getTotalNSV() 
244              try: 
245                  prm = svm.svmc.svm_model_param_get(self.__model.model) 
246                  C = svm.svmc.svm_parameter_C_get(prm) 
247                  # extract information of how many SVs sit inside the margin, 
248                  # i.e. so called 'bounded SVs' 
249                  inside_margin = N.sum( 
250                      # take 0.99 to avoid rounding issues 
251                      N.abs(self.__model.getSVCoef()) >= 0.99*svm.svmc.svm_parameter_C_get(prm)) 
252                  s += ' #bounded SVs:%d' % inside_margin 
253                  s += ' used C:%5g' % C 
254              except: 
255                  pass 
256          return s 
257   
258   
259 -    def untrain(self): 
260          if __debug__: 
261              debug("SVM", "Untraining %s and destroying libsvm model" % self) 
262          super(SVM, self).untrain() 
263          del self.__model 
264          self.__model = None 
265   
266      model = property(fget=lambda self: self.__model) 
267      """Access to the SVM model.""" 
268   
269   
270   
271  #class LinearSVM(SVM): 
272  #    """Base class of all linear SVM classifiers that make use of the libSVM 
273  #    package. Still not meant to be used directly. 
274  #    """ 
275  # 
276  #    def __init__(self, svm_impl, **kwargs): 
277  #        """The constructor arguments are virtually identical to the ones of 
278  #        the SVM class, except that 'kernel_type' is set to LINEAR. 
279  #        """ 
280  #        # init base class 
281  #        SVM.__init__(self, kernel_type='linear', 
282  #                         svm_impl=svm_impl, **kwargs) 
283  # 
284  # 
285  #    def getSensitivityAnalyzer(self, **kwargs): 
286  #        """Returns an appropriate SensitivityAnalyzer.""" 
287  #        return LibSVMLinearSVMWeights(self, **kwargs) 
288  # 
289  # 
290   
291  #class LinearNuSVMC(LinearSVM): 
292  #    """Classifier for linear Nu-SVM classification. 
293  #    """ 
294  # 
295  #    def __init__(self, **kwargs): 
296  #        """ 
297  #        """ 
298  #        # init base class 
299  #        LinearSVM.__init__(self, svm_impl='NU_SVC', **kwargs) 
300  # 
301  # 
302  #class LinearCSVMC(LinearSVM): 
303  #    """Classifier for linear C-SVM classification. 
304  #    """ 
305  # 
306  #    def __init__(self, **kwargs): 
307  #        """ 
308  #        """ 
309  #        # init base class 
310  #        LinearSVM.__init__(self, svm_impl='C_SVC', **kwargs) 
311  # 
312  # 
313  # 
314  #class RbfNuSVMC(SVM): 
315  #    """Nu-SVM classifier using a radial basis function kernel. 
316  #    """ 
317  # 
318  #    def __init__(self, **kwargs): 
319  #        """ 
320  #        """ 
321  #        # init base class 
322  #        SVM.__init__(self, kernel_type='rbf', 
323  #                     svm_impl='NU_SVC', **kwargs) 
324  # 
325  # 
326  #class RbfCSVMC(SVM): 
327  #    """C-SVM classifier using a radial basis function kernel. 
328  #    """ 
329  # 
330  #    def __init__(self, **kwargs): 
331  #        """ 
332  #        """ 
333  #        # init base class 
334  #        SVM.__init__(self, kernel_type='rbf', 
335  #                     svm_impl='C_SVC', **kwargs) 
336  # 
337   
338  # try to configure libsvm 'noise reduction'. Due to circular imports, 
339  # we can't check externals here since it would not work. 
340  try: 
341      # if externals.exists('libsvm verbosity control'): 
342      if __debug__ and "LIBSVM" in debug.active: 
343          debug("LIBSVM", "Setting verbosity for libsvm to 255") 
344          svm.svmc.svm_set_verbosity(255) 
345      else: 
346          svm.svmc.svm_set_verbosity(0) 
347  except AttributeError: 
348      warning("Available LIBSVM has no way to control verbosity of the output") 
349   
350   
351 -class LinearSVMWeights(Sensitivity): 
352      """`SensitivityAnalyzer` for the LIBSVM implementation of a linear SVM. 
353      """ 
354   
355      biases = StateVariable(enabled=True, 
356                             doc="Offsets of separating hyperplanes") 
357   
358   
359      _LEGAL_CLFS = [ SVM ] 
360   
361   
362 -    def __init__(self, clf, **kwargs): 
363          """Initialize the analyzer with the classifier it shall use. 
364   
365          :Parameters: 
366            clf: LinearSVM 
367              classifier to use. Only classifiers sub-classed from 
368              `LinearSVM` may be used. 
369          """ 
370          # init base classes first 
371          Sensitivity.__init__(self, clf, **kwargs) 
372   
373   
374 -    def _call(self, dataset, callables=[]): 
375          if self.clf.model.nr_class != 2: 
376              warning("You are estimating sensitivity for SVM %s trained on %d" % 
377                      (str(self.clf), self.clf.model.nr_class) + 
378                      " classes. Make sure that it is what you intended to do" ) 
379   
380          svcoef = N.matrix(self.clf.model.getSVCoef()) 
381          svs = N.matrix(self.clf.model.getSV()) 
382          rhos = N.asarray(self.clf.model.getRho()) 
383   
384          self.biases = rhos 
385          # XXX yoh: .mean() is effectively 
386          # averages across "sensitivities" of all paired classifiers (I 
387          # think). See more info on this topic in svm.py on how sv_coefs 
388          # are stored 
389          # 
390          # First multiply SV coefficients with the actuall SVs to get 
391          # weighted impact of SVs on decision, then for each feature 
392          # take mean across SVs to get a single weight value 
393          # per feature 
394          weights = svcoef * svs 
395   
396          if __debug__: 
397              debug('SVM', 
398                    "Extracting weights for %d-class SVM: #SVs=%s, " % \ 
399                    (self.clf.model.nr_class, str(self.clf.model.getNSV())) + \ 
400                    " SVcoefshape=%s SVs.shape=%s Rhos=%s." % \ 
401                    (svcoef.shape, svs.shape, rhos) + \ 
402                    " Result: min=%f max=%f" % (N.min(weights), N.max(weights))) 
403   
404          return N.asarray(weights.T) 
405   
406   
407      _customizeDocInherit = True 
408