__init__.py 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272
  1. '''This module implements specialized container datatypes providing
  2. alternatives to Python's general purpose built-in containers, dict,
  3. list, set, and tuple.
  4. * namedtuple factory function for creating tuple subclasses with named fields
  5. * deque list-like container with fast appends and pops on either end
  6. * ChainMap dict-like class for creating a single view of multiple mappings
  7. * Counter dict subclass for counting hashable objects
  8. * OrderedDict dict subclass that remembers the order entries were added
  9. * defaultdict dict subclass that calls a factory function to supply missing values
  10. * UserDict wrapper around dictionary objects for easier dict subclassing
  11. * UserList wrapper around list objects for easier list subclassing
  12. * UserString wrapper around string objects for easier string subclassing
  13. '''
  14. __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList',
  15. 'UserString', 'Counter', 'OrderedDict', 'ChainMap']
  16. import _collections_abc
  17. from operator import itemgetter as _itemgetter, eq as _eq
  18. from keyword import iskeyword as _iskeyword
  19. import sys as _sys
  20. import heapq as _heapq
  21. from _weakref import proxy as _proxy
  22. from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
  23. from reprlib import recursive_repr as _recursive_repr
  24. try:
  25. from _collections import deque
  26. except ImportError:
  27. pass
  28. else:
  29. _collections_abc.MutableSequence.register(deque)
  30. try:
  31. from _collections import defaultdict
  32. except ImportError:
  33. pass
  34. def __getattr__(name):
  35. # For backwards compatibility, continue to make the collections ABCs
  36. # through Python 3.6 available through the collections module.
  37. # Note, no new collections ABCs were added in Python 3.7
  38. if name in _collections_abc.__all__:
  39. obj = getattr(_collections_abc, name)
  40. import warnings
  41. warnings.warn("Using or importing the ABCs from 'collections' instead "
  42. "of from 'collections.abc' is deprecated since Python 3.3, "
  43. "and in 3.9 it will stop working",
  44. DeprecationWarning, stacklevel=2)
  45. globals()[name] = obj
  46. return obj
  47. raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
  48. ################################################################################
  49. ### OrderedDict
  50. ################################################################################
  51. class _OrderedDictKeysView(_collections_abc.KeysView):
  52. def __reversed__(self):
  53. yield from reversed(self._mapping)
  54. class _OrderedDictItemsView(_collections_abc.ItemsView):
  55. def __reversed__(self):
  56. for key in reversed(self._mapping):
  57. yield (key, self._mapping[key])
  58. class _OrderedDictValuesView(_collections_abc.ValuesView):
  59. def __reversed__(self):
  60. for key in reversed(self._mapping):
  61. yield self._mapping[key]
  62. class _Link(object):
  63. __slots__ = 'prev', 'next', 'key', '__weakref__'
  64. class OrderedDict(dict):
  65. 'Dictionary that remembers insertion order'
  66. # An inherited dict maps keys to values.
  67. # The inherited dict provides __getitem__, __len__, __contains__, and get.
  68. # The remaining methods are order-aware.
  69. # Big-O running times for all methods are the same as regular dictionaries.
  70. # The internal self.__map dict maps keys to links in a doubly linked list.
  71. # The circular doubly linked list starts and ends with a sentinel element.
  72. # The sentinel element never gets deleted (this simplifies the algorithm).
  73. # The sentinel is in self.__hardroot with a weakref proxy in self.__root.
  74. # The prev links are weakref proxies (to prevent circular references).
  75. # Individual links are kept alive by the hard reference in self.__map.
  76. # Those hard references disappear when a key is deleted from an OrderedDict.
  77. def __init__(self, other=(), /, **kwds):
  78. '''Initialize an ordered dictionary. The signature is the same as
  79. regular dictionaries. Keyword argument order is preserved.
  80. '''
  81. try:
  82. self.__root
  83. except AttributeError:
  84. self.__hardroot = _Link()
  85. self.__root = root = _proxy(self.__hardroot)
  86. root.prev = root.next = root
  87. self.__map = {}
  88. self.__update(other, **kwds)
  89. def __setitem__(self, key, value,
  90. dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link):
  91. 'od.__setitem__(i, y) <==> od[i]=y'
  92. # Setting a new item creates a new link at the end of the linked list,
  93. # and the inherited dictionary is updated with the new key/value pair.
  94. if key not in self:
  95. self.__map[key] = link = Link()
  96. root = self.__root
  97. last = root.prev
  98. link.prev, link.next, link.key = last, root, key
  99. last.next = link
  100. root.prev = proxy(link)
  101. dict_setitem(self, key, value)
  102. def __delitem__(self, key, dict_delitem=dict.__delitem__):
  103. 'od.__delitem__(y) <==> del od[y]'
  104. # Deleting an existing item uses self.__map to find the link which gets
  105. # removed by updating the links in the predecessor and successor nodes.
  106. dict_delitem(self, key)
  107. link = self.__map.pop(key)
  108. link_prev = link.prev
  109. link_next = link.next
  110. link_prev.next = link_next
  111. link_next.prev = link_prev
  112. link.prev = None
  113. link.next = None
  114. def __iter__(self):
  115. 'od.__iter__() <==> iter(od)'
  116. # Traverse the linked list in order.
  117. root = self.__root
  118. curr = root.next
  119. while curr is not root:
  120. yield curr.key
  121. curr = curr.next
  122. def __reversed__(self):
  123. 'od.__reversed__() <==> reversed(od)'
  124. # Traverse the linked list in reverse order.
  125. root = self.__root
  126. curr = root.prev
  127. while curr is not root:
  128. yield curr.key
  129. curr = curr.prev
  130. def clear(self):
  131. 'od.clear() -> None. Remove all items from od.'
  132. root = self.__root
  133. root.prev = root.next = root
  134. self.__map.clear()
  135. dict.clear(self)
  136. def popitem(self, last=True):
  137. '''Remove and return a (key, value) pair from the dictionary.
  138. Pairs are returned in LIFO order if last is true or FIFO order if false.
  139. '''
  140. if not self:
  141. raise KeyError('dictionary is empty')
  142. root = self.__root
  143. if last:
  144. link = root.prev
  145. link_prev = link.prev
  146. link_prev.next = root
  147. root.prev = link_prev
  148. else:
  149. link = root.next
  150. link_next = link.next
  151. root.next = link_next
  152. link_next.prev = root
  153. key = link.key
  154. del self.__map[key]
  155. value = dict.pop(self, key)
  156. return key, value
  157. def move_to_end(self, key, last=True):
  158. '''Move an existing element to the end (or beginning if last is false).
  159. Raise KeyError if the element does not exist.
  160. '''
  161. link = self.__map[key]
  162. link_prev = link.prev
  163. link_next = link.next
  164. soft_link = link_next.prev
  165. link_prev.next = link_next
  166. link_next.prev = link_prev
  167. root = self.__root
  168. if last:
  169. last = root.prev
  170. link.prev = last
  171. link.next = root
  172. root.prev = soft_link
  173. last.next = link
  174. else:
  175. first = root.next
  176. link.prev = root
  177. link.next = first
  178. first.prev = soft_link
  179. root.next = link
  180. def __sizeof__(self):
  181. sizeof = _sys.getsizeof
  182. n = len(self) + 1 # number of links including root
  183. size = sizeof(self.__dict__) # instance dictionary
  184. size += sizeof(self.__map) * 2 # internal dict and inherited dict
  185. size += sizeof(self.__hardroot) * n # link objects
  186. size += sizeof(self.__root) * n # proxy objects
  187. return size
  188. update = __update = _collections_abc.MutableMapping.update
  189. def keys(self):
  190. "D.keys() -> a set-like object providing a view on D's keys"
  191. return _OrderedDictKeysView(self)
  192. def items(self):
  193. "D.items() -> a set-like object providing a view on D's items"
  194. return _OrderedDictItemsView(self)
  195. def values(self):
  196. "D.values() -> an object providing a view on D's values"
  197. return _OrderedDictValuesView(self)
  198. __ne__ = _collections_abc.MutableMapping.__ne__
  199. __marker = object()
  200. def pop(self, key, default=__marker):
  201. '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
  202. value. If key is not found, d is returned if given, otherwise KeyError
  203. is raised.
  204. '''
  205. if key in self:
  206. result = self[key]
  207. del self[key]
  208. return result
  209. if default is self.__marker:
  210. raise KeyError(key)
  211. return default
  212. def setdefault(self, key, default=None):
  213. '''Insert key with a value of default if key is not in the dictionary.
  214. Return the value for key if key is in the dictionary, else default.
  215. '''
  216. if key in self:
  217. return self[key]
  218. self[key] = default
  219. return default
  220. @_recursive_repr()
  221. def __repr__(self):
  222. 'od.__repr__() <==> repr(od)'
  223. if not self:
  224. return '%s()' % (self.__class__.__name__,)
  225. return '%s(%r)' % (self.__class__.__name__, list(self.items()))
  226. def __reduce__(self):
  227. 'Return state information for pickling'
  228. inst_dict = vars(self).copy()
  229. for k in vars(OrderedDict()):
  230. inst_dict.pop(k, None)
  231. return self.__class__, (), inst_dict or None, None, iter(self.items())
  232. def copy(self):
  233. 'od.copy() -> a shallow copy of od'
  234. return self.__class__(self)
  235. @classmethod
  236. def fromkeys(cls, iterable, value=None):
  237. '''Create a new ordered dictionary with keys from iterable and values set to value.
  238. '''
  239. self = cls()
  240. for key in iterable:
  241. self[key] = value
  242. return self
  243. def __eq__(self, other):
  244. '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
  245. while comparison to a regular mapping is order-insensitive.
  246. '''
  247. if isinstance(other, OrderedDict):
  248. return dict.__eq__(self, other) and all(map(_eq, self, other))
  249. return dict.__eq__(self, other)
  250. try:
  251. from _collections import OrderedDict
  252. except ImportError:
  253. # Leave the pure Python version in place.
  254. pass
  255. ################################################################################
  256. ### namedtuple
  257. ################################################################################
  258. try:
  259. from _collections import _tuplegetter
  260. except ImportError:
  261. _tuplegetter = lambda index, doc: property(_itemgetter(index), doc=doc)
  262. def namedtuple(typename, field_names, *, rename=False, defaults=None, module=None):
  263. """Returns a new subclass of tuple with named fields.
  264. >>> Point = namedtuple('Point', ['x', 'y'])
  265. >>> Point.__doc__ # docstring for the new class
  266. 'Point(x, y)'
  267. >>> p = Point(11, y=22) # instantiate with positional args or keywords
  268. >>> p[0] + p[1] # indexable like a plain tuple
  269. 33
  270. >>> x, y = p # unpack like a regular tuple
  271. >>> x, y
  272. (11, 22)
  273. >>> p.x + p.y # fields also accessible by name
  274. 33
  275. >>> d = p._asdict() # convert to a dictionary
  276. >>> d['x']
  277. 11
  278. >>> Point(**d) # convert from a dictionary
  279. Point(x=11, y=22)
  280. >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields
  281. Point(x=100, y=22)
  282. """
  283. # Validate the field names. At the user's option, either generate an error
  284. # message or automatically replace the field name with a valid name.
  285. if isinstance(field_names, str):
  286. field_names = field_names.replace(',', ' ').split()
  287. field_names = list(map(str, field_names))
  288. typename = _sys.intern(str(typename))
  289. if rename:
  290. seen = set()
  291. for index, name in enumerate(field_names):
  292. if (not name.isidentifier()
  293. or _iskeyword(name)
  294. or name.startswith('_')
  295. or name in seen):
  296. field_names[index] = f'_{index}'
  297. seen.add(name)
  298. for name in [typename] + field_names:
  299. if type(name) is not str:
  300. raise TypeError('Type names and field names must be strings')
  301. if not name.isidentifier():
  302. raise ValueError('Type names and field names must be valid '
  303. f'identifiers: {name!r}')
  304. if _iskeyword(name):
  305. raise ValueError('Type names and field names cannot be a '
  306. f'keyword: {name!r}')
  307. seen = set()
  308. for name in field_names:
  309. if name.startswith('_') and not rename:
  310. raise ValueError('Field names cannot start with an underscore: '
  311. f'{name!r}')
  312. if name in seen:
  313. raise ValueError(f'Encountered duplicate field name: {name!r}')
  314. seen.add(name)
  315. field_defaults = {}
  316. if defaults is not None:
  317. defaults = tuple(defaults)
  318. if len(defaults) > len(field_names):
  319. raise TypeError('Got more default values than field names')
  320. field_defaults = dict(reversed(list(zip(reversed(field_names),
  321. reversed(defaults)))))
  322. # Variables used in the methods and docstrings
  323. field_names = tuple(map(_sys.intern, field_names))
  324. num_fields = len(field_names)
  325. arg_list = repr(field_names).replace("'", "")[1:-1]
  326. repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')'
  327. tuple_new = tuple.__new__
  328. _dict, _tuple, _len, _map, _zip = dict, tuple, len, map, zip
  329. # Create all the named tuple methods to be added to the class namespace
  330. s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))'
  331. namespace = {'_tuple_new': tuple_new, '__name__': f'namedtuple_{typename}'}
  332. # Note: exec() has the side-effect of interning the field names
  333. exec(s, namespace)
  334. __new__ = namespace['__new__']
  335. __new__.__doc__ = f'Create new instance of {typename}({arg_list})'
  336. if defaults is not None:
  337. __new__.__defaults__ = defaults
  338. @classmethod
  339. def _make(cls, iterable):
  340. result = tuple_new(cls, iterable)
  341. if _len(result) != num_fields:
  342. raise TypeError(f'Expected {num_fields} arguments, got {len(result)}')
  343. return result
  344. _make.__func__.__doc__ = (f'Make a new {typename} object from a sequence '
  345. 'or iterable')
  346. def _replace(self, /, **kwds):
  347. result = self._make(_map(kwds.pop, field_names, self))
  348. if kwds:
  349. raise ValueError(f'Got unexpected field names: {list(kwds)!r}')
  350. return result
  351. _replace.__doc__ = (f'Return a new {typename} object replacing specified '
  352. 'fields with new values')
  353. def __repr__(self):
  354. 'Return a nicely formatted representation string'
  355. return self.__class__.__name__ + repr_fmt % self
  356. def _asdict(self):
  357. 'Return a new dict which maps field names to their values.'
  358. return _dict(_zip(self._fields, self))
  359. def __getnewargs__(self):
  360. 'Return self as a plain tuple. Used by copy and pickle.'
  361. return _tuple(self)
  362. # Modify function metadata to help with introspection and debugging
  363. for method in (__new__, _make.__func__, _replace,
  364. __repr__, _asdict, __getnewargs__):
  365. method.__qualname__ = f'{typename}.{method.__name__}'
  366. # Build-up the class namespace dictionary
  367. # and use type() to build the result class
  368. class_namespace = {
  369. '__doc__': f'{typename}({arg_list})',
  370. '__slots__': (),
  371. '_fields': field_names,
  372. '_field_defaults': field_defaults,
  373. # alternate spelling for backward compatibility
  374. '_fields_defaults': field_defaults,
  375. '__new__': __new__,
  376. '_make': _make,
  377. '_replace': _replace,
  378. '__repr__': __repr__,
  379. '_asdict': _asdict,
  380. '__getnewargs__': __getnewargs__,
  381. }
  382. for index, name in enumerate(field_names):
  383. doc = _sys.intern(f'Alias for field number {index}')
  384. class_namespace[name] = _tuplegetter(index, doc)
  385. result = type(typename, (tuple,), class_namespace)
  386. # For pickling to work, the __module__ variable needs to be set to the frame
  387. # where the named tuple is created. Bypass this step in environments where
  388. # sys._getframe is not defined (Jython for example) or sys._getframe is not
  389. # defined for arguments greater than 0 (IronPython), or where the user has
  390. # specified a particular module.
  391. if module is None:
  392. try:
  393. module = _sys._getframe(1).f_globals.get('__name__', '__main__')
  394. except (AttributeError, ValueError):
  395. pass
  396. if module is not None:
  397. result.__module__ = module
  398. return result
  399. ########################################################################
  400. ### Counter
  401. ########################################################################
  402. def _count_elements(mapping, iterable):
  403. 'Tally elements from the iterable.'
  404. mapping_get = mapping.get
  405. for elem in iterable:
  406. mapping[elem] = mapping_get(elem, 0) + 1
  407. try: # Load C helper function if available
  408. from _collections import _count_elements
  409. except ImportError:
  410. pass
  411. class Counter(dict):
  412. '''Dict subclass for counting hashable items. Sometimes called a bag
  413. or multiset. Elements are stored as dictionary keys and their counts
  414. are stored as dictionary values.
  415. >>> c = Counter('abcdeabcdabcaba') # count elements from a string
  416. >>> c.most_common(3) # three most common elements
  417. [('a', 5), ('b', 4), ('c', 3)]
  418. >>> sorted(c) # list all unique elements
  419. ['a', 'b', 'c', 'd', 'e']
  420. >>> ''.join(sorted(c.elements())) # list elements with repetitions
  421. 'aaaaabbbbcccdde'
  422. >>> sum(c.values()) # total of all counts
  423. 15
  424. >>> c['a'] # count of letter 'a'
  425. 5
  426. >>> for elem in 'shazam': # update counts from an iterable
  427. ... c[elem] += 1 # by adding 1 to each element's count
  428. >>> c['a'] # now there are seven 'a'
  429. 7
  430. >>> del c['b'] # remove all 'b'
  431. >>> c['b'] # now there are zero 'b'
  432. 0
  433. >>> d = Counter('simsalabim') # make another counter
  434. >>> c.update(d) # add in the second counter
  435. >>> c['a'] # now there are nine 'a'
  436. 9
  437. >>> c.clear() # empty the counter
  438. >>> c
  439. Counter()
  440. Note: If a count is set to zero or reduced to zero, it will remain
  441. in the counter until the entry is deleted or the counter is cleared:
  442. >>> c = Counter('aaabbc')
  443. >>> c['b'] -= 2 # reduce the count of 'b' by two
  444. >>> c.most_common() # 'b' is still in, but its count is zero
  445. [('a', 3), ('c', 1), ('b', 0)]
  446. '''
  447. # References:
  448. # http://en.wikipedia.org/wiki/Multiset
  449. # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html
  450. # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm
  451. # http://code.activestate.com/recipes/259174/
  452. # Knuth, TAOCP Vol. II section 4.6.3
  453. def __init__(self, iterable=None, /, **kwds):
  454. '''Create a new, empty Counter object. And if given, count elements
  455. from an input iterable. Or, initialize the count from another mapping
  456. of elements to their counts.
  457. >>> c = Counter() # a new, empty counter
  458. >>> c = Counter('gallahad') # a new counter from an iterable
  459. >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
  460. >>> c = Counter(a=4, b=2) # a new counter from keyword args
  461. '''
  462. super(Counter, self).__init__()
  463. self.update(iterable, **kwds)
  464. def __missing__(self, key):
  465. 'The count of elements not in the Counter is zero.'
  466. # Needed so that self[missing_item] does not raise KeyError
  467. return 0
  468. def most_common(self, n=None):
  469. '''List the n most common elements and their counts from the most
  470. common to the least. If n is None, then list all element counts.
  471. >>> Counter('abracadabra').most_common(3)
  472. [('a', 5), ('b', 2), ('r', 2)]
  473. '''
  474. # Emulate Bag.sortedByCount from Smalltalk
  475. if n is None:
  476. return sorted(self.items(), key=_itemgetter(1), reverse=True)
  477. return _heapq.nlargest(n, self.items(), key=_itemgetter(1))
  478. def elements(self):
  479. '''Iterator over elements repeating each as many times as its count.
  480. >>> c = Counter('ABCABC')
  481. >>> sorted(c.elements())
  482. ['A', 'A', 'B', 'B', 'C', 'C']
  483. # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1
  484. >>> prime_factors = Counter({2: 2, 3: 3, 17: 1})
  485. >>> product = 1
  486. >>> for factor in prime_factors.elements(): # loop over factors
  487. ... product *= factor # and multiply them
  488. >>> product
  489. 1836
  490. Note, if an element's count has been set to zero or is a negative
  491. number, elements() will ignore it.
  492. '''
  493. # Emulate Bag.do from Smalltalk and Multiset.begin from C++.
  494. return _chain.from_iterable(_starmap(_repeat, self.items()))
  495. # Override dict methods where necessary
  496. @classmethod
  497. def fromkeys(cls, iterable, v=None):
  498. # There is no equivalent method for counters because the semantics
  499. # would be ambiguous in cases such as Counter.fromkeys('aaabbc', v=2).
  500. # Initializing counters to zero values isn't necessary because zero
  501. # is already the default value for counter lookups. Initializing
  502. # to one is easily accomplished with Counter(set(iterable)). For
  503. # more exotic cases, create a dictionary first using a dictionary
  504. # comprehension or dict.fromkeys().
  505. raise NotImplementedError(
  506. 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.')
  507. def update(self, iterable=None, /, **kwds):
  508. '''Like dict.update() but add counts instead of replacing them.
  509. Source can be an iterable, a dictionary, or another Counter instance.
  510. >>> c = Counter('which')
  511. >>> c.update('witch') # add elements from another iterable
  512. >>> d = Counter('watch')
  513. >>> c.update(d) # add elements from another counter
  514. >>> c['h'] # four 'h' in which, witch, and watch
  515. 4
  516. '''
  517. # The regular dict.update() operation makes no sense here because the
  518. # replace behavior results in the some of original untouched counts
  519. # being mixed-in with all of the other counts for a mismash that
  520. # doesn't have a straight-forward interpretation in most counting
  521. # contexts. Instead, we implement straight-addition. Both the inputs
  522. # and outputs are allowed to contain zero and negative counts.
  523. if iterable is not None:
  524. if isinstance(iterable, _collections_abc.Mapping):
  525. if self:
  526. self_get = self.get
  527. for elem, count in iterable.items():
  528. self[elem] = count + self_get(elem, 0)
  529. else:
  530. super(Counter, self).update(iterable) # fast path when counter is empty
  531. else:
  532. _count_elements(self, iterable)
  533. if kwds:
  534. self.update(kwds)
  535. def subtract(self, iterable=None, /, **kwds):
  536. '''Like dict.update() but subtracts counts instead of replacing them.
  537. Counts can be reduced below zero. Both the inputs and outputs are
  538. allowed to contain zero and negative counts.
  539. Source can be an iterable, a dictionary, or another Counter instance.
  540. >>> c = Counter('which')
  541. >>> c.subtract('witch') # subtract elements from another iterable
  542. >>> c.subtract(Counter('watch')) # subtract elements from another counter
  543. >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch
  544. 0
  545. >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch
  546. -1
  547. '''
  548. if iterable is not None:
  549. self_get = self.get
  550. if isinstance(iterable, _collections_abc.Mapping):
  551. for elem, count in iterable.items():
  552. self[elem] = self_get(elem, 0) - count
  553. else:
  554. for elem in iterable:
  555. self[elem] = self_get(elem, 0) - 1
  556. if kwds:
  557. self.subtract(kwds)
  558. def copy(self):
  559. 'Return a shallow copy.'
  560. return self.__class__(self)
  561. def __reduce__(self):
  562. return self.__class__, (dict(self),)
  563. def __delitem__(self, elem):
  564. 'Like dict.__delitem__() but does not raise KeyError for missing values.'
  565. if elem in self:
  566. super().__delitem__(elem)
  567. def __repr__(self):
  568. if not self:
  569. return '%s()' % self.__class__.__name__
  570. try:
  571. items = ', '.join(map('%r: %r'.__mod__, self.most_common()))
  572. return '%s({%s})' % (self.__class__.__name__, items)
  573. except TypeError:
  574. # handle case where values are not orderable
  575. return '{0}({1!r})'.format(self.__class__.__name__, dict(self))
  576. # Multiset-style mathematical operations discussed in:
  577. # Knuth TAOCP Volume II section 4.6.3 exercise 19
  578. # and at http://en.wikipedia.org/wiki/Multiset
  579. #
  580. # Outputs guaranteed to only include positive counts.
  581. #
  582. # To strip negative and zero counts, add-in an empty counter:
  583. # c += Counter()
  584. def __add__(self, other):
  585. '''Add counts from two counters.
  586. >>> Counter('abbb') + Counter('bcc')
  587. Counter({'b': 4, 'c': 2, 'a': 1})
  588. '''
  589. if not isinstance(other, Counter):
  590. return NotImplemented
  591. result = Counter()
  592. for elem, count in self.items():
  593. newcount = count + other[elem]
  594. if newcount > 0:
  595. result[elem] = newcount
  596. for elem, count in other.items():
  597. if elem not in self and count > 0:
  598. result[elem] = count
  599. return result
  600. def __sub__(self, other):
  601. ''' Subtract count, but keep only results with positive counts.
  602. >>> Counter('abbbc') - Counter('bccd')
  603. Counter({'b': 2, 'a': 1})
  604. '''
  605. if not isinstance(other, Counter):
  606. return NotImplemented
  607. result = Counter()
  608. for elem, count in self.items():
  609. newcount = count - other[elem]
  610. if newcount > 0:
  611. result[elem] = newcount
  612. for elem, count in other.items():
  613. if elem not in self and count < 0:
  614. result[elem] = 0 - count
  615. return result
  616. def __or__(self, other):
  617. '''Union is the maximum of value in either of the input counters.
  618. >>> Counter('abbb') | Counter('bcc')
  619. Counter({'b': 3, 'c': 2, 'a': 1})
  620. '''
  621. if not isinstance(other, Counter):
  622. return NotImplemented
  623. result = Counter()
  624. for elem, count in self.items():
  625. other_count = other[elem]
  626. newcount = other_count if count < other_count else count
  627. if newcount > 0:
  628. result[elem] = newcount
  629. for elem, count in other.items():
  630. if elem not in self and count > 0:
  631. result[elem] = count
  632. return result
  633. def __and__(self, other):
  634. ''' Intersection is the minimum of corresponding counts.
  635. >>> Counter('abbb') & Counter('bcc')
  636. Counter({'b': 1})
  637. '''
  638. if not isinstance(other, Counter):
  639. return NotImplemented
  640. result = Counter()
  641. for elem, count in self.items():
  642. other_count = other[elem]
  643. newcount = count if count < other_count else other_count
  644. if newcount > 0:
  645. result[elem] = newcount
  646. return result
  647. def __pos__(self):
  648. 'Adds an empty counter, effectively stripping negative and zero counts'
  649. result = Counter()
  650. for elem, count in self.items():
  651. if count > 0:
  652. result[elem] = count
  653. return result
  654. def __neg__(self):
  655. '''Subtracts from an empty counter. Strips positive and zero counts,
  656. and flips the sign on negative counts.
  657. '''
  658. result = Counter()
  659. for elem, count in self.items():
  660. if count < 0:
  661. result[elem] = 0 - count
  662. return result
  663. def _keep_positive(self):
  664. '''Internal method to strip elements with a negative or zero count'''
  665. nonpositive = [elem for elem, count in self.items() if not count > 0]
  666. for elem in nonpositive:
  667. del self[elem]
  668. return self
  669. def __iadd__(self, other):
  670. '''Inplace add from another counter, keeping only positive counts.
  671. >>> c = Counter('abbb')
  672. >>> c += Counter('bcc')
  673. >>> c
  674. Counter({'b': 4, 'c': 2, 'a': 1})
  675. '''
  676. for elem, count in other.items():
  677. self[elem] += count
  678. return self._keep_positive()
  679. def __isub__(self, other):
  680. '''Inplace subtract counter, but keep only results with positive counts.
  681. >>> c = Counter('abbbc')
  682. >>> c -= Counter('bccd')
  683. >>> c
  684. Counter({'b': 2, 'a': 1})
  685. '''
  686. for elem, count in other.items():
  687. self[elem] -= count
  688. return self._keep_positive()
  689. def __ior__(self, other):
  690. '''Inplace union is the maximum of value from either counter.
  691. >>> c = Counter('abbb')
  692. >>> c |= Counter('bcc')
  693. >>> c
  694. Counter({'b': 3, 'c': 2, 'a': 1})
  695. '''
  696. for elem, other_count in other.items():
  697. count = self[elem]
  698. if other_count > count:
  699. self[elem] = other_count
  700. return self._keep_positive()
  701. def __iand__(self, other):
  702. '''Inplace intersection is the minimum of corresponding counts.
  703. >>> c = Counter('abbb')
  704. >>> c &= Counter('bcc')
  705. >>> c
  706. Counter({'b': 1})
  707. '''
  708. for elem, count in self.items():
  709. other_count = other[elem]
  710. if other_count < count:
  711. self[elem] = other_count
  712. return self._keep_positive()
  713. ########################################################################
  714. ### ChainMap
  715. ########################################################################
  716. class ChainMap(_collections_abc.MutableMapping):
  717. ''' A ChainMap groups multiple dicts (or other mappings) together
  718. to create a single, updateable view.
  719. The underlying mappings are stored in a list. That list is public and can
  720. be accessed or updated using the *maps* attribute. There is no other
  721. state.
  722. Lookups search the underlying mappings successively until a key is found.
  723. In contrast, writes, updates, and deletions only operate on the first
  724. mapping.
  725. '''
  726. def __init__(self, *maps):
  727. '''Initialize a ChainMap by setting *maps* to the given mappings.
  728. If no mappings are provided, a single empty dictionary is used.
  729. '''
  730. self.maps = list(maps) or [{}] # always at least one map
  731. def __missing__(self, key):
  732. raise KeyError(key)
  733. def __getitem__(self, key):
  734. for mapping in self.maps:
  735. try:
  736. return mapping[key] # can't use 'key in mapping' with defaultdict
  737. except KeyError:
  738. pass
  739. return self.__missing__(key) # support subclasses that define __missing__
  740. def get(self, key, default=None):
  741. return self[key] if key in self else default
  742. def __len__(self):
  743. return len(set().union(*self.maps)) # reuses stored hash values if possible
  744. def __iter__(self):
  745. d = {}
  746. for mapping in reversed(self.maps):
  747. d.update(mapping) # reuses stored hash values if possible
  748. return iter(d)
  749. def __contains__(self, key):
  750. return any(key in m for m in self.maps)
  751. def __bool__(self):
  752. return any(self.maps)
  753. @_recursive_repr()
  754. def __repr__(self):
  755. return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})'
  756. @classmethod
  757. def fromkeys(cls, iterable, *args):
  758. 'Create a ChainMap with a single dict created from the iterable.'
  759. return cls(dict.fromkeys(iterable, *args))
  760. def copy(self):
  761. 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
  762. return self.__class__(self.maps[0].copy(), *self.maps[1:])
  763. __copy__ = copy
  764. def new_child(self, m=None): # like Django's Context.push()
  765. '''New ChainMap with a new map followed by all previous maps.
  766. If no map is provided, an empty dict is used.
  767. '''
  768. if m is None:
  769. m = {}
  770. return self.__class__(m, *self.maps)
  771. @property
  772. def parents(self): # like Django's Context.pop()
  773. 'New ChainMap from maps[1:].'
  774. return self.__class__(*self.maps[1:])
  775. def __setitem__(self, key, value):
  776. self.maps[0][key] = value
  777. def __delitem__(self, key):
  778. try:
  779. del self.maps[0][key]
  780. except KeyError:
  781. raise KeyError('Key not found in the first mapping: {!r}'.format(key))
  782. def popitem(self):
  783. 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'
  784. try:
  785. return self.maps[0].popitem()
  786. except KeyError:
  787. raise KeyError('No keys found in the first mapping.')
  788. def pop(self, key, *args):
  789. 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'
  790. try:
  791. return self.maps[0].pop(key, *args)
  792. except KeyError:
  793. raise KeyError('Key not found in the first mapping: {!r}'.format(key))
  794. def clear(self):
  795. 'Clear maps[0], leaving maps[1:] intact.'
  796. self.maps[0].clear()
  797. ################################################################################
  798. ### UserDict
  799. ################################################################################
  800. class UserDict(_collections_abc.MutableMapping):
  801. # Start by filling-out the abstract methods
  802. def __init__(*args, **kwargs):
  803. if not args:
  804. raise TypeError("descriptor '__init__' of 'UserDict' object "
  805. "needs an argument")
  806. self, *args = args
  807. if len(args) > 1:
  808. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  809. if args:
  810. dict = args[0]
  811. elif 'dict' in kwargs:
  812. dict = kwargs.pop('dict')
  813. import warnings
  814. warnings.warn("Passing 'dict' as keyword argument is deprecated",
  815. DeprecationWarning, stacklevel=2)
  816. else:
  817. dict = None
  818. self.data = {}
  819. if dict is not None:
  820. self.update(dict)
  821. if kwargs:
  822. self.update(kwargs)
  823. __init__.__text_signature__ = '($self, dict=None, /, **kwargs)'
  824. def __len__(self): return len(self.data)
  825. def __getitem__(self, key):
  826. if key in self.data:
  827. return self.data[key]
  828. if hasattr(self.__class__, "__missing__"):
  829. return self.__class__.__missing__(self, key)
  830. raise KeyError(key)
  831. def __setitem__(self, key, item): self.data[key] = item
  832. def __delitem__(self, key): del self.data[key]
  833. def __iter__(self):
  834. return iter(self.data)
  835. # Modify __contains__ to work correctly when __missing__ is present
  836. def __contains__(self, key):
  837. return key in self.data
  838. # Now, add the methods in dicts but not in MutableMapping
  839. def __repr__(self): return repr(self.data)
  840. def __copy__(self):
  841. inst = self.__class__.__new__(self.__class__)
  842. inst.__dict__.update(self.__dict__)
  843. # Create a copy and avoid triggering descriptors
  844. inst.__dict__["data"] = self.__dict__["data"].copy()
  845. return inst
  846. def copy(self):
  847. if self.__class__ is UserDict:
  848. return UserDict(self.data.copy())
  849. import copy
  850. data = self.data
  851. try:
  852. self.data = {}
  853. c = copy.copy(self)
  854. finally:
  855. self.data = data
  856. c.update(self)
  857. return c
  858. @classmethod
  859. def fromkeys(cls, iterable, value=None):
  860. d = cls()
  861. for key in iterable:
  862. d[key] = value
  863. return d
  864. ################################################################################
  865. ### UserList
  866. ################################################################################
  867. class UserList(_collections_abc.MutableSequence):
  868. """A more or less complete user-defined wrapper around list objects."""
  869. def __init__(self, initlist=None):
  870. self.data = []
  871. if initlist is not None:
  872. # XXX should this accept an arbitrary sequence?
  873. if type(initlist) == type(self.data):
  874. self.data[:] = initlist
  875. elif isinstance(initlist, UserList):
  876. self.data[:] = initlist.data[:]
  877. else:
  878. self.data = list(initlist)
  879. def __repr__(self): return repr(self.data)
  880. def __lt__(self, other): return self.data < self.__cast(other)
  881. def __le__(self, other): return self.data <= self.__cast(other)
  882. def __eq__(self, other): return self.data == self.__cast(other)
  883. def __gt__(self, other): return self.data > self.__cast(other)
  884. def __ge__(self, other): return self.data >= self.__cast(other)
  885. def __cast(self, other):
  886. return other.data if isinstance(other, UserList) else other
  887. def __contains__(self, item): return item in self.data
  888. def __len__(self): return len(self.data)
  889. def __getitem__(self, i):
  890. if isinstance(i, slice):
  891. return self.__class__(self.data[i])
  892. else:
  893. return self.data[i]
  894. def __setitem__(self, i, item): self.data[i] = item
  895. def __delitem__(self, i): del self.data[i]
  896. def __add__(self, other):
  897. if isinstance(other, UserList):
  898. return self.__class__(self.data + other.data)
  899. elif isinstance(other, type(self.data)):
  900. return self.__class__(self.data + other)
  901. return self.__class__(self.data + list(other))
  902. def __radd__(self, other):
  903. if isinstance(other, UserList):
  904. return self.__class__(other.data + self.data)
  905. elif isinstance(other, type(self.data)):
  906. return self.__class__(other + self.data)
  907. return self.__class__(list(other) + self.data)
  908. def __iadd__(self, other):
  909. if isinstance(other, UserList):
  910. self.data += other.data
  911. elif isinstance(other, type(self.data)):
  912. self.data += other
  913. else:
  914. self.data += list(other)
  915. return self
  916. def __mul__(self, n):
  917. return self.__class__(self.data*n)
  918. __rmul__ = __mul__
  919. def __imul__(self, n):
  920. self.data *= n
  921. return self
  922. def __copy__(self):
  923. inst = self.__class__.__new__(self.__class__)
  924. inst.__dict__.update(self.__dict__)
  925. # Create a copy and avoid triggering descriptors
  926. inst.__dict__["data"] = self.__dict__["data"][:]
  927. return inst
  928. def append(self, item): self.data.append(item)
  929. def insert(self, i, item): self.data.insert(i, item)
  930. def pop(self, i=-1): return self.data.pop(i)
  931. def remove(self, item): self.data.remove(item)
  932. def clear(self): self.data.clear()
  933. def copy(self): return self.__class__(self)
  934. def count(self, item): return self.data.count(item)
  935. def index(self, item, *args): return self.data.index(item, *args)
  936. def reverse(self): self.data.reverse()
  937. def sort(self, /, *args, **kwds): self.data.sort(*args, **kwds)
  938. def extend(self, other):
  939. if isinstance(other, UserList):
  940. self.data.extend(other.data)
  941. else:
  942. self.data.extend(other)
  943. ################################################################################
  944. ### UserString
  945. ################################################################################
  946. class UserString(_collections_abc.Sequence):
  947. def __init__(self, seq):
  948. if isinstance(seq, str):
  949. self.data = seq
  950. elif isinstance(seq, UserString):
  951. self.data = seq.data[:]
  952. else:
  953. self.data = str(seq)
  954. def __str__(self): return str(self.data)
  955. def __repr__(self): return repr(self.data)
  956. def __int__(self): return int(self.data)
  957. def __float__(self): return float(self.data)
  958. def __complex__(self): return complex(self.data)
  959. def __hash__(self): return hash(self.data)
  960. def __getnewargs__(self):
  961. return (self.data[:],)
  962. def __eq__(self, string):
  963. if isinstance(string, UserString):
  964. return self.data == string.data
  965. return self.data == string
  966. def __lt__(self, string):
  967. if isinstance(string, UserString):
  968. return self.data < string.data
  969. return self.data < string
  970. def __le__(self, string):
  971. if isinstance(string, UserString):
  972. return self.data <= string.data
  973. return self.data <= string
  974. def __gt__(self, string):
  975. if isinstance(string, UserString):
  976. return self.data > string.data
  977. return self.data > string
  978. def __ge__(self, string):
  979. if isinstance(string, UserString):
  980. return self.data >= string.data
  981. return self.data >= string
  982. def __contains__(self, char):
  983. if isinstance(char, UserString):
  984. char = char.data
  985. return char in self.data
  986. def __len__(self): return len(self.data)
  987. def __getitem__(self, index): return self.__class__(self.data[index])
  988. def __add__(self, other):
  989. if isinstance(other, UserString):
  990. return self.__class__(self.data + other.data)
  991. elif isinstance(other, str):
  992. return self.__class__(self.data + other)
  993. return self.__class__(self.data + str(other))
  994. def __radd__(self, other):
  995. if isinstance(other, str):
  996. return self.__class__(other + self.data)
  997. return self.__class__(str(other) + self.data)
  998. def __mul__(self, n):
  999. return self.__class__(self.data*n)
  1000. __rmul__ = __mul__
  1001. def __mod__(self, args):
  1002. return self.__class__(self.data % args)
  1003. def __rmod__(self, template):
  1004. return self.__class__(str(template) % self)
  1005. # the following methods are defined in alphabetical order:
  1006. def capitalize(self): return self.__class__(self.data.capitalize())
  1007. def casefold(self):
  1008. return self.__class__(self.data.casefold())
  1009. def center(self, width, *args):
  1010. return self.__class__(self.data.center(width, *args))
  1011. def count(self, sub, start=0, end=_sys.maxsize):
  1012. if isinstance(sub, UserString):
  1013. sub = sub.data
  1014. return self.data.count(sub, start, end)
  1015. def encode(self, encoding='utf-8', errors='strict'):
  1016. encoding = 'utf-8' if encoding is None else encoding
  1017. errors = 'strict' if errors is None else errors
  1018. return self.data.encode(encoding, errors)
  1019. def endswith(self, suffix, start=0, end=_sys.maxsize):
  1020. return self.data.endswith(suffix, start, end)
  1021. def expandtabs(self, tabsize=8):
  1022. return self.__class__(self.data.expandtabs(tabsize))
  1023. def find(self, sub, start=0, end=_sys.maxsize):
  1024. if isinstance(sub, UserString):
  1025. sub = sub.data
  1026. return self.data.find(sub, start, end)
  1027. def format(self, /, *args, **kwds):
  1028. return self.data.format(*args, **kwds)
  1029. def format_map(self, mapping):
  1030. return self.data.format_map(mapping)
  1031. def index(self, sub, start=0, end=_sys.maxsize):
  1032. return self.data.index(sub, start, end)
  1033. def isalpha(self): return self.data.isalpha()
  1034. def isalnum(self): return self.data.isalnum()
  1035. def isascii(self): return self.data.isascii()
  1036. def isdecimal(self): return self.data.isdecimal()
  1037. def isdigit(self): return self.data.isdigit()
  1038. def isidentifier(self): return self.data.isidentifier()
  1039. def islower(self): return self.data.islower()
  1040. def isnumeric(self): return self.data.isnumeric()
  1041. def isprintable(self): return self.data.isprintable()
  1042. def isspace(self): return self.data.isspace()
  1043. def istitle(self): return self.data.istitle()
  1044. def isupper(self): return self.data.isupper()
  1045. def join(self, seq): return self.data.join(seq)
  1046. def ljust(self, width, *args):
  1047. return self.__class__(self.data.ljust(width, *args))
  1048. def lower(self): return self.__class__(self.data.lower())
  1049. def lstrip(self, chars=None): return self.__class__(self.data.lstrip(chars))
  1050. maketrans = str.maketrans
  1051. def partition(self, sep):
  1052. return self.data.partition(sep)
  1053. def replace(self, old, new, maxsplit=-1):
  1054. if isinstance(old, UserString):
  1055. old = old.data
  1056. if isinstance(new, UserString):
  1057. new = new.data
  1058. return self.__class__(self.data.replace(old, new, maxsplit))
  1059. def rfind(self, sub, start=0, end=_sys.maxsize):
  1060. if isinstance(sub, UserString):
  1061. sub = sub.data
  1062. return self.data.rfind(sub, start, end)
  1063. def rindex(self, sub, start=0, end=_sys.maxsize):
  1064. return self.data.rindex(sub, start, end)
  1065. def rjust(self, width, *args):
  1066. return self.__class__(self.data.rjust(width, *args))
  1067. def rpartition(self, sep):
  1068. return self.data.rpartition(sep)
  1069. def rstrip(self, chars=None):
  1070. return self.__class__(self.data.rstrip(chars))
  1071. def split(self, sep=None, maxsplit=-1):
  1072. return self.data.split(sep, maxsplit)
  1073. def rsplit(self, sep=None, maxsplit=-1):
  1074. return self.data.rsplit(sep, maxsplit)
  1075. def splitlines(self, keepends=False): return self.data.splitlines(keepends)
  1076. def startswith(self, prefix, start=0, end=_sys.maxsize):
  1077. return self.data.startswith(prefix, start, end)
  1078. def strip(self, chars=None): return self.__class__(self.data.strip(chars))
  1079. def swapcase(self): return self.__class__(self.data.swapcase())
  1080. def title(self): return self.__class__(self.data.title())
  1081. def translate(self, *args):
  1082. return self.__class__(self.data.translate(*args))
  1083. def upper(self): return self.__class__(self.data.upper())
  1084. def zfill(self, width): return self.__class__(self.data.zfill(width))