misc.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944
  1. """
  2. Miscellaneous function (re)definitions from the Py3.4+ standard library
  3. for Python 2.6/2.7.
  4. - math.ceil (for Python 2.7)
  5. - collections.OrderedDict (for Python 2.6)
  6. - collections.Counter (for Python 2.6)
  7. - collections.ChainMap (for all versions prior to Python 3.3)
  8. - itertools.count (for Python 2.6, with step parameter)
  9. - subprocess.check_output (for Python 2.6)
  10. - reprlib.recursive_repr (for Python 2.6+)
  11. - functools.cmp_to_key (for Python 2.6)
  12. """
  13. from __future__ import absolute_import
  14. import subprocess
  15. from math import ceil as oldceil
  16. from operator import itemgetter as _itemgetter, eq as _eq
  17. import sys
  18. import heapq as _heapq
  19. from _weakref import proxy as _proxy
  20. from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
  21. from socket import getaddrinfo, SOCK_STREAM, error, socket
  22. from future.utils import iteritems, itervalues, PY2, PY26, PY3
  23. if PY2:
  24. from collections import Mapping, MutableMapping
  25. else:
  26. from collections.abc import Mapping, MutableMapping
  27. def ceil(x):
  28. """
  29. Return the ceiling of x as an int.
  30. This is the smallest integral value >= x.
  31. """
  32. return int(oldceil(x))
  33. ########################################################################
  34. ### reprlib.recursive_repr decorator from Py3.4
  35. ########################################################################
  36. from itertools import islice
  37. if PY3:
  38. try:
  39. from _thread import get_ident
  40. except ImportError:
  41. from _dummy_thread import get_ident
  42. else:
  43. try:
  44. from thread import get_ident
  45. except ImportError:
  46. from dummy_thread import get_ident
  47. def recursive_repr(fillvalue='...'):
  48. 'Decorator to make a repr function return fillvalue for a recursive call'
  49. def decorating_function(user_function):
  50. repr_running = set()
  51. def wrapper(self):
  52. key = id(self), get_ident()
  53. if key in repr_running:
  54. return fillvalue
  55. repr_running.add(key)
  56. try:
  57. result = user_function(self)
  58. finally:
  59. repr_running.discard(key)
  60. return result
  61. # Can't use functools.wraps() here because of bootstrap issues
  62. wrapper.__module__ = getattr(user_function, '__module__')
  63. wrapper.__doc__ = getattr(user_function, '__doc__')
  64. wrapper.__name__ = getattr(user_function, '__name__')
  65. wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
  66. return wrapper
  67. return decorating_function
  68. ################################################################################
  69. ### OrderedDict
  70. ################################################################################
  71. class _Link(object):
  72. __slots__ = 'prev', 'next', 'key', '__weakref__'
  73. class OrderedDict(dict):
  74. 'Dictionary that remembers insertion order'
  75. # An inherited dict maps keys to values.
  76. # The inherited dict provides __getitem__, __len__, __contains__, and get.
  77. # The remaining methods are order-aware.
  78. # Big-O running times for all methods are the same as regular dictionaries.
  79. # The internal self.__map dict maps keys to links in a doubly linked list.
  80. # The circular doubly linked list starts and ends with a sentinel element.
  81. # The sentinel element never gets deleted (this simplifies the algorithm).
  82. # The sentinel is in self.__hardroot with a weakref proxy in self.__root.
  83. # The prev links are weakref proxies (to prevent circular references).
  84. # Individual links are kept alive by the hard reference in self.__map.
  85. # Those hard references disappear when a key is deleted from an OrderedDict.
  86. def __init__(*args, **kwds):
  87. '''Initialize an ordered dictionary. The signature is the same as
  88. regular dictionaries, but keyword arguments are not recommended because
  89. their insertion order is arbitrary.
  90. '''
  91. if not args:
  92. raise TypeError("descriptor '__init__' of 'OrderedDict' object "
  93. "needs an argument")
  94. self = args[0]
  95. args = args[1:]
  96. if len(args) > 1:
  97. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  98. try:
  99. self.__root
  100. except AttributeError:
  101. self.__hardroot = _Link()
  102. self.__root = root = _proxy(self.__hardroot)
  103. root.prev = root.next = root
  104. self.__map = {}
  105. self.__update(*args, **kwds)
  106. def __setitem__(self, key, value,
  107. dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link):
  108. 'od.__setitem__(i, y) <==> od[i]=y'
  109. # Setting a new item creates a new link at the end of the linked list,
  110. # and the inherited dictionary is updated with the new key/value pair.
  111. if key not in self:
  112. self.__map[key] = link = Link()
  113. root = self.__root
  114. last = root.prev
  115. link.prev, link.next, link.key = last, root, key
  116. last.next = link
  117. root.prev = proxy(link)
  118. dict_setitem(self, key, value)
  119. def __delitem__(self, key, dict_delitem=dict.__delitem__):
  120. 'od.__delitem__(y) <==> del od[y]'
  121. # Deleting an existing item uses self.__map to find the link which gets
  122. # removed by updating the links in the predecessor and successor nodes.
  123. dict_delitem(self, key)
  124. link = self.__map.pop(key)
  125. link_prev = link.prev
  126. link_next = link.next
  127. link_prev.next = link_next
  128. link_next.prev = link_prev
  129. def __iter__(self):
  130. 'od.__iter__() <==> iter(od)'
  131. # Traverse the linked list in order.
  132. root = self.__root
  133. curr = root.next
  134. while curr is not root:
  135. yield curr.key
  136. curr = curr.next
  137. def __reversed__(self):
  138. 'od.__reversed__() <==> reversed(od)'
  139. # Traverse the linked list in reverse order.
  140. root = self.__root
  141. curr = root.prev
  142. while curr is not root:
  143. yield curr.key
  144. curr = curr.prev
  145. def clear(self):
  146. 'od.clear() -> None. Remove all items from od.'
  147. root = self.__root
  148. root.prev = root.next = root
  149. self.__map.clear()
  150. dict.clear(self)
  151. def popitem(self, last=True):
  152. '''od.popitem() -> (k, v), return and remove a (key, value) pair.
  153. Pairs are returned in LIFO order if last is true or FIFO order if false.
  154. '''
  155. if not self:
  156. raise KeyError('dictionary is empty')
  157. root = self.__root
  158. if last:
  159. link = root.prev
  160. link_prev = link.prev
  161. link_prev.next = root
  162. root.prev = link_prev
  163. else:
  164. link = root.next
  165. link_next = link.next
  166. root.next = link_next
  167. link_next.prev = root
  168. key = link.key
  169. del self.__map[key]
  170. value = dict.pop(self, key)
  171. return key, value
  172. def move_to_end(self, key, last=True):
  173. '''Move an existing element to the end (or beginning if last==False).
  174. Raises KeyError if the element does not exist.
  175. When last=True, acts like a fast version of self[key]=self.pop(key).
  176. '''
  177. link = self.__map[key]
  178. link_prev = link.prev
  179. link_next = link.next
  180. link_prev.next = link_next
  181. link_next.prev = link_prev
  182. root = self.__root
  183. if last:
  184. last = root.prev
  185. link.prev = last
  186. link.next = root
  187. last.next = root.prev = link
  188. else:
  189. first = root.next
  190. link.prev = root
  191. link.next = first
  192. root.next = first.prev = link
  193. def __sizeof__(self):
  194. sizeof = sys.getsizeof
  195. n = len(self) + 1 # number of links including root
  196. size = sizeof(self.__dict__) # instance dictionary
  197. size += sizeof(self.__map) * 2 # internal dict and inherited dict
  198. size += sizeof(self.__hardroot) * n # link objects
  199. size += sizeof(self.__root) * n # proxy objects
  200. return size
  201. update = __update = MutableMapping.update
  202. keys = MutableMapping.keys
  203. values = MutableMapping.values
  204. items = MutableMapping.items
  205. __ne__ = MutableMapping.__ne__
  206. __marker = object()
  207. def pop(self, key, default=__marker):
  208. '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
  209. value. If key is not found, d is returned if given, otherwise KeyError
  210. is raised.
  211. '''
  212. if key in self:
  213. result = self[key]
  214. del self[key]
  215. return result
  216. if default is self.__marker:
  217. raise KeyError(key)
  218. return default
  219. def setdefault(self, key, default=None):
  220. 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
  221. if key in self:
  222. return self[key]
  223. self[key] = default
  224. return default
  225. @recursive_repr()
  226. def __repr__(self):
  227. 'od.__repr__() <==> repr(od)'
  228. if not self:
  229. return '%s()' % (self.__class__.__name__,)
  230. return '%s(%r)' % (self.__class__.__name__, list(self.items()))
  231. def __reduce__(self):
  232. 'Return state information for pickling'
  233. inst_dict = vars(self).copy()
  234. for k in vars(OrderedDict()):
  235. inst_dict.pop(k, None)
  236. return self.__class__, (), inst_dict or None, None, iter(self.items())
  237. def copy(self):
  238. 'od.copy() -> a shallow copy of od'
  239. return self.__class__(self)
  240. @classmethod
  241. def fromkeys(cls, iterable, value=None):
  242. '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
  243. If not specified, the value defaults to None.
  244. '''
  245. self = cls()
  246. for key in iterable:
  247. self[key] = value
  248. return self
  249. def __eq__(self, other):
  250. '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
  251. while comparison to a regular mapping is order-insensitive.
  252. '''
  253. if isinstance(other, OrderedDict):
  254. return dict.__eq__(self, other) and all(map(_eq, self, other))
  255. return dict.__eq__(self, other)
  256. # {{{ http://code.activestate.com/recipes/576611/ (r11)
  257. try:
  258. from operator import itemgetter
  259. from heapq import nlargest
  260. except ImportError:
  261. pass
  262. ########################################################################
  263. ### Counter
  264. ########################################################################
  265. def _count_elements(mapping, iterable):
  266. 'Tally elements from the iterable.'
  267. mapping_get = mapping.get
  268. for elem in iterable:
  269. mapping[elem] = mapping_get(elem, 0) + 1
  270. class Counter(dict):
  271. '''Dict subclass for counting hashable items. Sometimes called a bag
  272. or multiset. Elements are stored as dictionary keys and their counts
  273. are stored as dictionary values.
  274. >>> c = Counter('abcdeabcdabcaba') # count elements from a string
  275. >>> c.most_common(3) # three most common elements
  276. [('a', 5), ('b', 4), ('c', 3)]
  277. >>> sorted(c) # list all unique elements
  278. ['a', 'b', 'c', 'd', 'e']
  279. >>> ''.join(sorted(c.elements())) # list elements with repetitions
  280. 'aaaaabbbbcccdde'
  281. >>> sum(c.values()) # total of all counts
  282. 15
  283. >>> c['a'] # count of letter 'a'
  284. 5
  285. >>> for elem in 'shazam': # update counts from an iterable
  286. ... c[elem] += 1 # by adding 1 to each element's count
  287. >>> c['a'] # now there are seven 'a'
  288. 7
  289. >>> del c['b'] # remove all 'b'
  290. >>> c['b'] # now there are zero 'b'
  291. 0
  292. >>> d = Counter('simsalabim') # make another counter
  293. >>> c.update(d) # add in the second counter
  294. >>> c['a'] # now there are nine 'a'
  295. 9
  296. >>> c.clear() # empty the counter
  297. >>> c
  298. Counter()
  299. Note: If a count is set to zero or reduced to zero, it will remain
  300. in the counter until the entry is deleted or the counter is cleared:
  301. >>> c = Counter('aaabbc')
  302. >>> c['b'] -= 2 # reduce the count of 'b' by two
  303. >>> c.most_common() # 'b' is still in, but its count is zero
  304. [('a', 3), ('c', 1), ('b', 0)]
  305. '''
  306. # References:
  307. # http://en.wikipedia.org/wiki/Multiset
  308. # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html
  309. # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm
  310. # http://code.activestate.com/recipes/259174/
  311. # Knuth, TAOCP Vol. II section 4.6.3
  312. def __init__(*args, **kwds):
  313. '''Create a new, empty Counter object. And if given, count elements
  314. from an input iterable. Or, initialize the count from another mapping
  315. of elements to their counts.
  316. >>> c = Counter() # a new, empty counter
  317. >>> c = Counter('gallahad') # a new counter from an iterable
  318. >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
  319. >>> c = Counter(a=4, b=2) # a new counter from keyword args
  320. '''
  321. if not args:
  322. raise TypeError("descriptor '__init__' of 'Counter' object "
  323. "needs an argument")
  324. self = args[0]
  325. args = args[1:]
  326. if len(args) > 1:
  327. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  328. super(Counter, self).__init__()
  329. self.update(*args, **kwds)
  330. def __missing__(self, key):
  331. 'The count of elements not in the Counter is zero.'
  332. # Needed so that self[missing_item] does not raise KeyError
  333. return 0
  334. def most_common(self, n=None):
  335. '''List the n most common elements and their counts from the most
  336. common to the least. If n is None, then list all element counts.
  337. >>> Counter('abcdeabcdabcaba').most_common(3)
  338. [('a', 5), ('b', 4), ('c', 3)]
  339. '''
  340. # Emulate Bag.sortedByCount from Smalltalk
  341. if n is None:
  342. return sorted(self.items(), key=_itemgetter(1), reverse=True)
  343. return _heapq.nlargest(n, self.items(), key=_itemgetter(1))
  344. def elements(self):
  345. '''Iterator over elements repeating each as many times as its count.
  346. >>> c = Counter('ABCABC')
  347. >>> sorted(c.elements())
  348. ['A', 'A', 'B', 'B', 'C', 'C']
  349. # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1
  350. >>> prime_factors = Counter({2: 2, 3: 3, 17: 1})
  351. >>> product = 1
  352. >>> for factor in prime_factors.elements(): # loop over factors
  353. ... product *= factor # and multiply them
  354. >>> product
  355. 1836
  356. Note, if an element's count has been set to zero or is a negative
  357. number, elements() will ignore it.
  358. '''
  359. # Emulate Bag.do from Smalltalk and Multiset.begin from C++.
  360. return _chain.from_iterable(_starmap(_repeat, self.items()))
  361. # Override dict methods where necessary
  362. @classmethod
  363. def fromkeys(cls, iterable, v=None):
  364. # There is no equivalent method for counters because setting v=1
  365. # means that no element can have a count greater than one.
  366. raise NotImplementedError(
  367. 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.')
  368. def update(*args, **kwds):
  369. '''Like dict.update() but add counts instead of replacing them.
  370. Source can be an iterable, a dictionary, or another Counter instance.
  371. >>> c = Counter('which')
  372. >>> c.update('witch') # add elements from another iterable
  373. >>> d = Counter('watch')
  374. >>> c.update(d) # add elements from another counter
  375. >>> c['h'] # four 'h' in which, witch, and watch
  376. 4
  377. '''
  378. # The regular dict.update() operation makes no sense here because the
  379. # replace behavior results in the some of original untouched counts
  380. # being mixed-in with all of the other counts for a mismash that
  381. # doesn't have a straight-forward interpretation in most counting
  382. # contexts. Instead, we implement straight-addition. Both the inputs
  383. # and outputs are allowed to contain zero and negative counts.
  384. if not args:
  385. raise TypeError("descriptor 'update' of 'Counter' object "
  386. "needs an argument")
  387. self = args[0]
  388. args = args[1:]
  389. if len(args) > 1:
  390. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  391. iterable = args[0] if args else None
  392. if iterable is not None:
  393. if isinstance(iterable, Mapping):
  394. if self:
  395. self_get = self.get
  396. for elem, count in iterable.items():
  397. self[elem] = count + self_get(elem, 0)
  398. else:
  399. super(Counter, self).update(iterable) # fast path when counter is empty
  400. else:
  401. _count_elements(self, iterable)
  402. if kwds:
  403. self.update(kwds)
  404. def subtract(*args, **kwds):
  405. '''Like dict.update() but subtracts counts instead of replacing them.
  406. Counts can be reduced below zero. Both the inputs and outputs are
  407. allowed to contain zero and negative counts.
  408. Source can be an iterable, a dictionary, or another Counter instance.
  409. >>> c = Counter('which')
  410. >>> c.subtract('witch') # subtract elements from another iterable
  411. >>> c.subtract(Counter('watch')) # subtract elements from another counter
  412. >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch
  413. 0
  414. >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch
  415. -1
  416. '''
  417. if not args:
  418. raise TypeError("descriptor 'subtract' of 'Counter' object "
  419. "needs an argument")
  420. self = args[0]
  421. args = args[1:]
  422. if len(args) > 1:
  423. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  424. iterable = args[0] if args else None
  425. if iterable is not None:
  426. self_get = self.get
  427. if isinstance(iterable, Mapping):
  428. for elem, count in iterable.items():
  429. self[elem] = self_get(elem, 0) - count
  430. else:
  431. for elem in iterable:
  432. self[elem] = self_get(elem, 0) - 1
  433. if kwds:
  434. self.subtract(kwds)
  435. def copy(self):
  436. 'Return a shallow copy.'
  437. return self.__class__(self)
  438. def __reduce__(self):
  439. return self.__class__, (dict(self),)
  440. def __delitem__(self, elem):
  441. 'Like dict.__delitem__() but does not raise KeyError for missing values.'
  442. if elem in self:
  443. super(Counter, self).__delitem__(elem)
  444. def __repr__(self):
  445. if not self:
  446. return '%s()' % self.__class__.__name__
  447. try:
  448. items = ', '.join(map('%r: %r'.__mod__, self.most_common()))
  449. return '%s({%s})' % (self.__class__.__name__, items)
  450. except TypeError:
  451. # handle case where values are not orderable
  452. return '{0}({1!r})'.format(self.__class__.__name__, dict(self))
  453. # Multiset-style mathematical operations discussed in:
  454. # Knuth TAOCP Volume II section 4.6.3 exercise 19
  455. # and at http://en.wikipedia.org/wiki/Multiset
  456. #
  457. # Outputs guaranteed to only include positive counts.
  458. #
  459. # To strip negative and zero counts, add-in an empty counter:
  460. # c += Counter()
  461. def __add__(self, other):
  462. '''Add counts from two counters.
  463. >>> Counter('abbb') + Counter('bcc')
  464. Counter({'b': 4, 'c': 2, 'a': 1})
  465. '''
  466. if not isinstance(other, Counter):
  467. return NotImplemented
  468. result = Counter()
  469. for elem, count in self.items():
  470. newcount = count + other[elem]
  471. if newcount > 0:
  472. result[elem] = newcount
  473. for elem, count in other.items():
  474. if elem not in self and count > 0:
  475. result[elem] = count
  476. return result
  477. def __sub__(self, other):
  478. ''' Subtract count, but keep only results with positive counts.
  479. >>> Counter('abbbc') - Counter('bccd')
  480. Counter({'b': 2, 'a': 1})
  481. '''
  482. if not isinstance(other, Counter):
  483. return NotImplemented
  484. result = Counter()
  485. for elem, count in self.items():
  486. newcount = count - other[elem]
  487. if newcount > 0:
  488. result[elem] = newcount
  489. for elem, count in other.items():
  490. if elem not in self and count < 0:
  491. result[elem] = 0 - count
  492. return result
  493. def __or__(self, other):
  494. '''Union is the maximum of value in either of the input counters.
  495. >>> Counter('abbb') | Counter('bcc')
  496. Counter({'b': 3, 'c': 2, 'a': 1})
  497. '''
  498. if not isinstance(other, Counter):
  499. return NotImplemented
  500. result = Counter()
  501. for elem, count in self.items():
  502. other_count = other[elem]
  503. newcount = other_count if count < other_count else count
  504. if newcount > 0:
  505. result[elem] = newcount
  506. for elem, count in other.items():
  507. if elem not in self and count > 0:
  508. result[elem] = count
  509. return result
  510. def __and__(self, other):
  511. ''' Intersection is the minimum of corresponding counts.
  512. >>> Counter('abbb') & Counter('bcc')
  513. Counter({'b': 1})
  514. '''
  515. if not isinstance(other, Counter):
  516. return NotImplemented
  517. result = Counter()
  518. for elem, count in self.items():
  519. other_count = other[elem]
  520. newcount = count if count < other_count else other_count
  521. if newcount > 0:
  522. result[elem] = newcount
  523. return result
  524. def __pos__(self):
  525. 'Adds an empty counter, effectively stripping negative and zero counts'
  526. return self + Counter()
  527. def __neg__(self):
  528. '''Subtracts from an empty counter. Strips positive and zero counts,
  529. and flips the sign on negative counts.
  530. '''
  531. return Counter() - self
  532. def _keep_positive(self):
  533. '''Internal method to strip elements with a negative or zero count'''
  534. nonpositive = [elem for elem, count in self.items() if not count > 0]
  535. for elem in nonpositive:
  536. del self[elem]
  537. return self
  538. def __iadd__(self, other):
  539. '''Inplace add from another counter, keeping only positive counts.
  540. >>> c = Counter('abbb')
  541. >>> c += Counter('bcc')
  542. >>> c
  543. Counter({'b': 4, 'c': 2, 'a': 1})
  544. '''
  545. for elem, count in other.items():
  546. self[elem] += count
  547. return self._keep_positive()
  548. def __isub__(self, other):
  549. '''Inplace subtract counter, but keep only results with positive counts.
  550. >>> c = Counter('abbbc')
  551. >>> c -= Counter('bccd')
  552. >>> c
  553. Counter({'b': 2, 'a': 1})
  554. '''
  555. for elem, count in other.items():
  556. self[elem] -= count
  557. return self._keep_positive()
  558. def __ior__(self, other):
  559. '''Inplace union is the maximum of value from either counter.
  560. >>> c = Counter('abbb')
  561. >>> c |= Counter('bcc')
  562. >>> c
  563. Counter({'b': 3, 'c': 2, 'a': 1})
  564. '''
  565. for elem, other_count in other.items():
  566. count = self[elem]
  567. if other_count > count:
  568. self[elem] = other_count
  569. return self._keep_positive()
  570. def __iand__(self, other):
  571. '''Inplace intersection is the minimum of corresponding counts.
  572. >>> c = Counter('abbb')
  573. >>> c &= Counter('bcc')
  574. >>> c
  575. Counter({'b': 1})
  576. '''
  577. for elem, count in self.items():
  578. other_count = other[elem]
  579. if other_count < count:
  580. self[elem] = other_count
  581. return self._keep_positive()
  582. def check_output(*popenargs, **kwargs):
  583. """
  584. For Python 2.6 compatibility: see
  585. http://stackoverflow.com/questions/4814970/
  586. """
  587. if 'stdout' in kwargs:
  588. raise ValueError('stdout argument not allowed, it will be overridden.')
  589. process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
  590. output, unused_err = process.communicate()
  591. retcode = process.poll()
  592. if retcode:
  593. cmd = kwargs.get("args")
  594. if cmd is None:
  595. cmd = popenargs[0]
  596. raise subprocess.CalledProcessError(retcode, cmd)
  597. return output
  598. def count(start=0, step=1):
  599. """
  600. ``itertools.count`` in Py 2.6 doesn't accept a step
  601. parameter. This is an enhanced version of ``itertools.count``
  602. for Py2.6 equivalent to ``itertools.count`` in Python 2.7+.
  603. """
  604. while True:
  605. yield start
  606. start += step
  607. ########################################################################
  608. ### ChainMap (helper for configparser and string.Template)
  609. ### From the Py3.4 source code. See also:
  610. ### https://github.com/kkxue/Py2ChainMap/blob/master/py2chainmap.py
  611. ########################################################################
  612. class ChainMap(MutableMapping):
  613. ''' A ChainMap groups multiple dicts (or other mappings) together
  614. to create a single, updateable view.
  615. The underlying mappings are stored in a list. That list is public and can
  616. accessed or updated using the *maps* attribute. There is no other state.
  617. Lookups search the underlying mappings successively until a key is found.
  618. In contrast, writes, updates, and deletions only operate on the first
  619. mapping.
  620. '''
  621. def __init__(self, *maps):
  622. '''Initialize a ChainMap by setting *maps* to the given mappings.
  623. If no mappings are provided, a single empty dictionary is used.
  624. '''
  625. self.maps = list(maps) or [{}] # always at least one map
  626. def __missing__(self, key):
  627. raise KeyError(key)
  628. def __getitem__(self, key):
  629. for mapping in self.maps:
  630. try:
  631. return mapping[key] # can't use 'key in mapping' with defaultdict
  632. except KeyError:
  633. pass
  634. return self.__missing__(key) # support subclasses that define __missing__
  635. def get(self, key, default=None):
  636. return self[key] if key in self else default
  637. def __len__(self):
  638. return len(set().union(*self.maps)) # reuses stored hash values if possible
  639. def __iter__(self):
  640. return iter(set().union(*self.maps))
  641. def __contains__(self, key):
  642. return any(key in m for m in self.maps)
  643. def __bool__(self):
  644. return any(self.maps)
  645. # Py2 compatibility:
  646. __nonzero__ = __bool__
  647. @recursive_repr()
  648. def __repr__(self):
  649. return '{0.__class__.__name__}({1})'.format(
  650. self, ', '.join(map(repr, self.maps)))
  651. @classmethod
  652. def fromkeys(cls, iterable, *args):
  653. 'Create a ChainMap with a single dict created from the iterable.'
  654. return cls(dict.fromkeys(iterable, *args))
  655. def copy(self):
  656. 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
  657. return self.__class__(self.maps[0].copy(), *self.maps[1:])
  658. __copy__ = copy
  659. def new_child(self, m=None): # like Django's Context.push()
  660. '''
  661. New ChainMap with a new map followed by all previous maps. If no
  662. map is provided, an empty dict is used.
  663. '''
  664. if m is None:
  665. m = {}
  666. return self.__class__(m, *self.maps)
  667. @property
  668. def parents(self): # like Django's Context.pop()
  669. 'New ChainMap from maps[1:].'
  670. return self.__class__(*self.maps[1:])
  671. def __setitem__(self, key, value):
  672. self.maps[0][key] = value
  673. def __delitem__(self, key):
  674. try:
  675. del self.maps[0][key]
  676. except KeyError:
  677. raise KeyError('Key not found in the first mapping: {0!r}'.format(key))
  678. def popitem(self):
  679. 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'
  680. try:
  681. return self.maps[0].popitem()
  682. except KeyError:
  683. raise KeyError('No keys found in the first mapping.')
  684. def pop(self, key, *args):
  685. 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'
  686. try:
  687. return self.maps[0].pop(key, *args)
  688. except KeyError:
  689. raise KeyError('Key not found in the first mapping: {0!r}'.format(key))
  690. def clear(self):
  691. 'Clear maps[0], leaving maps[1:] intact.'
  692. self.maps[0].clear()
  693. # Re-use the same sentinel as in the Python stdlib socket module:
  694. from socket import _GLOBAL_DEFAULT_TIMEOUT
  695. # Was: _GLOBAL_DEFAULT_TIMEOUT = object()
  696. def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
  697. source_address=None):
  698. """Backport of 3-argument create_connection() for Py2.6.
  699. Connect to *address* and return the socket object.
  700. Convenience function. Connect to *address* (a 2-tuple ``(host,
  701. port)``) and return the socket object. Passing the optional
  702. *timeout* parameter will set the timeout on the socket instance
  703. before attempting to connect. If no *timeout* is supplied, the
  704. global default timeout setting returned by :func:`getdefaulttimeout`
  705. is used. If *source_address* is set it must be a tuple of (host, port)
  706. for the socket to bind as a source address before making the connection.
  707. An host of '' or port 0 tells the OS to use the default.
  708. """
  709. host, port = address
  710. err = None
  711. for res in getaddrinfo(host, port, 0, SOCK_STREAM):
  712. af, socktype, proto, canonname, sa = res
  713. sock = None
  714. try:
  715. sock = socket(af, socktype, proto)
  716. if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
  717. sock.settimeout(timeout)
  718. if source_address:
  719. sock.bind(source_address)
  720. sock.connect(sa)
  721. return sock
  722. except error as _:
  723. err = _
  724. if sock is not None:
  725. sock.close()
  726. if err is not None:
  727. raise err
  728. else:
  729. raise error("getaddrinfo returns an empty list")
  730. # Backport from Py2.7 for Py2.6:
  731. def cmp_to_key(mycmp):
  732. """Convert a cmp= function into a key= function"""
  733. class K(object):
  734. __slots__ = ['obj']
  735. def __init__(self, obj, *args):
  736. self.obj = obj
  737. def __lt__(self, other):
  738. return mycmp(self.obj, other.obj) < 0
  739. def __gt__(self, other):
  740. return mycmp(self.obj, other.obj) > 0
  741. def __eq__(self, other):
  742. return mycmp(self.obj, other.obj) == 0
  743. def __le__(self, other):
  744. return mycmp(self.obj, other.obj) <= 0
  745. def __ge__(self, other):
  746. return mycmp(self.obj, other.obj) >= 0
  747. def __ne__(self, other):
  748. return mycmp(self.obj, other.obj) != 0
  749. def __hash__(self):
  750. raise TypeError('hash not implemented')
  751. return K
  752. # Back up our definitions above in case they're useful
  753. _OrderedDict = OrderedDict
  754. _Counter = Counter
  755. _check_output = check_output
  756. _count = count
  757. _ceil = ceil
  758. __count_elements = _count_elements
  759. _recursive_repr = recursive_repr
  760. _ChainMap = ChainMap
  761. _create_connection = create_connection
  762. _cmp_to_key = cmp_to_key
  763. # Overwrite the definitions above with the usual ones
  764. # from the standard library:
  765. if sys.version_info >= (2, 7):
  766. from collections import OrderedDict, Counter
  767. from itertools import count
  768. from functools import cmp_to_key
  769. try:
  770. from subprocess import check_output
  771. except ImportError:
  772. # Not available. This happens with Google App Engine: see issue #231
  773. pass
  774. from socket import create_connection
  775. if sys.version_info >= (3, 0):
  776. from math import ceil
  777. from collections import _count_elements
  778. if sys.version_info >= (3, 3):
  779. from reprlib import recursive_repr
  780. from collections import ChainMap