registry.go 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762
  1. // Copyright 2014 The Prometheus Authors
  2. // Licensed under the Apache License, Version 2.0 (the "License");
  3. // you may not use this file except in compliance with the License.
  4. // You may obtain a copy of the License at
  5. //
  6. // http://www.apache.org/licenses/LICENSE-2.0
  7. //
  8. // Unless required by applicable law or agreed to in writing, software
  9. // distributed under the License is distributed on an "AS IS" BASIS,
  10. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. // See the License for the specific language governing permissions and
  12. // limitations under the License.
  13. package prometheus
  14. import (
  15. "bytes"
  16. "errors"
  17. "fmt"
  18. "os"
  19. "sort"
  20. "sync"
  21. "unicode/utf8"
  22. "github.com/golang/protobuf/proto"
  23. dto "github.com/prometheus/client_model/go"
  24. )
  25. const (
  26. // Capacity for the channel to collect metrics and descriptors.
  27. capMetricChan = 1000
  28. capDescChan = 10
  29. )
  30. // DefaultRegisterer and DefaultGatherer are the implementations of the
  31. // Registerer and Gatherer interface a number of convenience functions in this
  32. // package act on. Initially, both variables point to the same Registry, which
  33. // has a process collector (see NewProcessCollector) and a Go collector (see
  34. // NewGoCollector) already registered. This approach to keep default instances
  35. // as global state mirrors the approach of other packages in the Go standard
  36. // library. Note that there are caveats. Change the variables with caution and
  37. // only if you understand the consequences. Users who want to avoid global state
  38. // altogether should not use the convenience function and act on custom
  39. // instances instead.
  40. var (
  41. defaultRegistry = NewRegistry()
  42. DefaultRegisterer Registerer = defaultRegistry
  43. DefaultGatherer Gatherer = defaultRegistry
  44. )
  45. func init() {
  46. MustRegister(NewProcessCollector(os.Getpid(), ""))
  47. MustRegister(NewGoCollector())
  48. }
  49. // NewRegistry creates a new vanilla Registry without any Collectors
  50. // pre-registered.
  51. func NewRegistry() *Registry {
  52. return &Registry{
  53. collectorsByID: map[uint64]Collector{},
  54. descIDs: map[uint64]struct{}{},
  55. dimHashesByName: map[string]uint64{},
  56. }
  57. }
  58. // NewPedanticRegistry returns a registry that checks during collection if each
  59. // collected Metric is consistent with its reported Desc, and if the Desc has
  60. // actually been registered with the registry.
  61. //
  62. // Usually, a Registry will be happy as long as the union of all collected
  63. // Metrics is consistent and valid even if some metrics are not consistent with
  64. // their own Desc or a Desc provided by their registered Collector. Well-behaved
  65. // Collectors and Metrics will only provide consistent Descs. This Registry is
  66. // useful to test the implementation of Collectors and Metrics.
  67. func NewPedanticRegistry() *Registry {
  68. r := NewRegistry()
  69. r.pedanticChecksEnabled = true
  70. return r
  71. }
  72. // Registerer is the interface for the part of a registry in charge of
  73. // registering and unregistering. Users of custom registries should use
  74. // Registerer as type for registration purposes (rather than the Registry type
  75. // directly). In that way, they are free to use custom Registerer implementation
  76. // (e.g. for testing purposes).
  77. type Registerer interface {
  78. // Register registers a new Collector to be included in metrics
  79. // collection. It returns an error if the descriptors provided by the
  80. // Collector are invalid or if they — in combination with descriptors of
  81. // already registered Collectors — do not fulfill the consistency and
  82. // uniqueness criteria described in the documentation of metric.Desc.
  83. //
  84. // If the provided Collector is equal to a Collector already registered
  85. // (which includes the case of re-registering the same Collector), the
  86. // returned error is an instance of AlreadyRegisteredError, which
  87. // contains the previously registered Collector.
  88. //
  89. // It is in general not safe to register the same Collector multiple
  90. // times concurrently.
  91. Register(Collector) error
  92. // MustRegister works like Register but registers any number of
  93. // Collectors and panics upon the first registration that causes an
  94. // error.
  95. MustRegister(...Collector)
  96. // Unregister unregisters the Collector that equals the Collector passed
  97. // in as an argument. (Two Collectors are considered equal if their
  98. // Describe method yields the same set of descriptors.) The function
  99. // returns whether a Collector was unregistered.
  100. //
  101. // Note that even after unregistering, it will not be possible to
  102. // register a new Collector that is inconsistent with the unregistered
  103. // Collector, e.g. a Collector collecting metrics with the same name but
  104. // a different help string. The rationale here is that the same registry
  105. // instance must only collect consistent metrics throughout its
  106. // lifetime.
  107. Unregister(Collector) bool
  108. }
  109. // Gatherer is the interface for the part of a registry in charge of gathering
  110. // the collected metrics into a number of MetricFamilies. The Gatherer interface
  111. // comes with the same general implication as described for the Registerer
  112. // interface.
  113. type Gatherer interface {
  114. // Gather calls the Collect method of the registered Collectors and then
  115. // gathers the collected metrics into a lexicographically sorted slice
  116. // of MetricFamily protobufs. Even if an error occurs, Gather attempts
  117. // to gather as many metrics as possible. Hence, if a non-nil error is
  118. // returned, the returned MetricFamily slice could be nil (in case of a
  119. // fatal error that prevented any meaningful metric collection) or
  120. // contain a number of MetricFamily protobufs, some of which might be
  121. // incomplete, and some might be missing altogether. The returned error
  122. // (which might be a MultiError) explains the details. In scenarios
  123. // where complete collection is critical, the returned MetricFamily
  124. // protobufs should be disregarded if the returned error is non-nil.
  125. Gather() ([]*dto.MetricFamily, error)
  126. }
  127. // Register registers the provided Collector with the DefaultRegisterer.
  128. //
  129. // Register is a shortcut for DefaultRegisterer.Register(c). See there for more
  130. // details.
  131. func Register(c Collector) error {
  132. return DefaultRegisterer.Register(c)
  133. }
  134. // MustRegister registers the provided Collectors with the DefaultRegisterer and
  135. // panics if any error occurs.
  136. //
  137. // MustRegister is a shortcut for DefaultRegisterer.MustRegister(cs...). See
  138. // there for more details.
  139. func MustRegister(cs ...Collector) {
  140. DefaultRegisterer.MustRegister(cs...)
  141. }
  142. // Unregister removes the registration of the provided Collector from the
  143. // DefaultRegisterer.
  144. //
  145. // Unregister is a shortcut for DefaultRegisterer.Unregister(c). See there for
  146. // more details.
  147. func Unregister(c Collector) bool {
  148. return DefaultRegisterer.Unregister(c)
  149. }
  150. // GathererFunc turns a function into a Gatherer.
  151. type GathererFunc func() ([]*dto.MetricFamily, error)
  152. // Gather implements Gatherer.
  153. func (gf GathererFunc) Gather() ([]*dto.MetricFamily, error) {
  154. return gf()
  155. }
  156. // AlreadyRegisteredError is returned by the Register method if the Collector to
  157. // be registered has already been registered before, or a different Collector
  158. // that collects the same metrics has been registered before. Registration fails
  159. // in that case, but you can detect from the kind of error what has
  160. // happened. The error contains fields for the existing Collector and the
  161. // (rejected) new Collector that equals the existing one. This can be used to
  162. // find out if an equal Collector has been registered before and switch over to
  163. // using the old one, as demonstrated in the example.
  164. type AlreadyRegisteredError struct {
  165. ExistingCollector, NewCollector Collector
  166. }
  167. func (err AlreadyRegisteredError) Error() string {
  168. return "duplicate metrics collector registration attempted"
  169. }
  170. // MultiError is a slice of errors implementing the error interface. It is used
  171. // by a Gatherer to report multiple errors during MetricFamily gathering.
  172. type MultiError []error
  173. func (errs MultiError) Error() string {
  174. if len(errs) == 0 {
  175. return ""
  176. }
  177. buf := &bytes.Buffer{}
  178. fmt.Fprintf(buf, "%d error(s) occurred:", len(errs))
  179. for _, err := range errs {
  180. fmt.Fprintf(buf, "\n* %s", err)
  181. }
  182. return buf.String()
  183. }
  184. // MaybeUnwrap returns nil if len(errs) is 0. It returns the first and only
  185. // contained error as error if len(errs is 1). In all other cases, it returns
  186. // the MultiError directly. This is helpful for returning a MultiError in a way
  187. // that only uses the MultiError if needed.
  188. func (errs MultiError) MaybeUnwrap() error {
  189. switch len(errs) {
  190. case 0:
  191. return nil
  192. case 1:
  193. return errs[0]
  194. default:
  195. return errs
  196. }
  197. }
  198. // Registry registers Prometheus collectors, collects their metrics, and gathers
  199. // them into MetricFamilies for exposition. It implements both Registerer and
  200. // Gatherer. The zero value is not usable. Create instances with NewRegistry or
  201. // NewPedanticRegistry.
  202. type Registry struct {
  203. mtx sync.RWMutex
  204. collectorsByID map[uint64]Collector // ID is a hash of the descIDs.
  205. descIDs map[uint64]struct{}
  206. dimHashesByName map[string]uint64
  207. pedanticChecksEnabled bool
  208. }
  209. // Register implements Registerer.
  210. func (r *Registry) Register(c Collector) error {
  211. var (
  212. descChan = make(chan *Desc, capDescChan)
  213. newDescIDs = map[uint64]struct{}{}
  214. newDimHashesByName = map[string]uint64{}
  215. collectorID uint64 // Just a sum of all desc IDs.
  216. duplicateDescErr error
  217. )
  218. go func() {
  219. c.Describe(descChan)
  220. close(descChan)
  221. }()
  222. r.mtx.Lock()
  223. defer r.mtx.Unlock()
  224. // Conduct various tests...
  225. for desc := range descChan {
  226. // Is the descriptor valid at all?
  227. if desc.err != nil {
  228. return fmt.Errorf("descriptor %s is invalid: %s", desc, desc.err)
  229. }
  230. // Is the descID unique?
  231. // (In other words: Is the fqName + constLabel combination unique?)
  232. if _, exists := r.descIDs[desc.id]; exists {
  233. duplicateDescErr = fmt.Errorf("descriptor %s already exists with the same fully-qualified name and const label values", desc)
  234. }
  235. // If it is not a duplicate desc in this collector, add it to
  236. // the collectorID. (We allow duplicate descs within the same
  237. // collector, but their existence must be a no-op.)
  238. if _, exists := newDescIDs[desc.id]; !exists {
  239. newDescIDs[desc.id] = struct{}{}
  240. collectorID += desc.id
  241. }
  242. // Are all the label names and the help string consistent with
  243. // previous descriptors of the same name?
  244. // First check existing descriptors...
  245. if dimHash, exists := r.dimHashesByName[desc.fqName]; exists {
  246. if dimHash != desc.dimHash {
  247. return fmt.Errorf("a previously registered descriptor with the same fully-qualified name as %s has different label names or a different help string", desc)
  248. }
  249. } else {
  250. // ...then check the new descriptors already seen.
  251. if dimHash, exists := newDimHashesByName[desc.fqName]; exists {
  252. if dimHash != desc.dimHash {
  253. return fmt.Errorf("descriptors reported by collector have inconsistent label names or help strings for the same fully-qualified name, offender is %s", desc)
  254. }
  255. } else {
  256. newDimHashesByName[desc.fqName] = desc.dimHash
  257. }
  258. }
  259. }
  260. // Did anything happen at all?
  261. if len(newDescIDs) == 0 {
  262. return errors.New("collector has no descriptors")
  263. }
  264. if existing, exists := r.collectorsByID[collectorID]; exists {
  265. return AlreadyRegisteredError{
  266. ExistingCollector: existing,
  267. NewCollector: c,
  268. }
  269. }
  270. // If the collectorID is new, but at least one of the descs existed
  271. // before, we are in trouble.
  272. if duplicateDescErr != nil {
  273. return duplicateDescErr
  274. }
  275. // Only after all tests have passed, actually register.
  276. r.collectorsByID[collectorID] = c
  277. for hash := range newDescIDs {
  278. r.descIDs[hash] = struct{}{}
  279. }
  280. for name, dimHash := range newDimHashesByName {
  281. r.dimHashesByName[name] = dimHash
  282. }
  283. return nil
  284. }
  285. // Unregister implements Registerer.
  286. func (r *Registry) Unregister(c Collector) bool {
  287. var (
  288. descChan = make(chan *Desc, capDescChan)
  289. descIDs = map[uint64]struct{}{}
  290. collectorID uint64 // Just a sum of the desc IDs.
  291. )
  292. go func() {
  293. c.Describe(descChan)
  294. close(descChan)
  295. }()
  296. for desc := range descChan {
  297. if _, exists := descIDs[desc.id]; !exists {
  298. collectorID += desc.id
  299. descIDs[desc.id] = struct{}{}
  300. }
  301. }
  302. r.mtx.RLock()
  303. if _, exists := r.collectorsByID[collectorID]; !exists {
  304. r.mtx.RUnlock()
  305. return false
  306. }
  307. r.mtx.RUnlock()
  308. r.mtx.Lock()
  309. defer r.mtx.Unlock()
  310. delete(r.collectorsByID, collectorID)
  311. for id := range descIDs {
  312. delete(r.descIDs, id)
  313. }
  314. // dimHashesByName is left untouched as those must be consistent
  315. // throughout the lifetime of a program.
  316. return true
  317. }
  318. // MustRegister implements Registerer.
  319. func (r *Registry) MustRegister(cs ...Collector) {
  320. for _, c := range cs {
  321. if err := r.Register(c); err != nil {
  322. panic(err)
  323. }
  324. }
  325. }
  326. // Gather implements Gatherer.
  327. func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
  328. var (
  329. metricChan = make(chan Metric, capMetricChan)
  330. metricHashes = map[uint64]struct{}{}
  331. dimHashes = map[string]uint64{}
  332. wg sync.WaitGroup
  333. errs MultiError // The collected errors to return in the end.
  334. registeredDescIDs map[uint64]struct{} // Only used for pedantic checks
  335. )
  336. r.mtx.RLock()
  337. metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName))
  338. // Scatter.
  339. // (Collectors could be complex and slow, so we call them all at once.)
  340. wg.Add(len(r.collectorsByID))
  341. go func() {
  342. wg.Wait()
  343. close(metricChan)
  344. }()
  345. for _, collector := range r.collectorsByID {
  346. go func(collector Collector) {
  347. defer wg.Done()
  348. collector.Collect(metricChan)
  349. }(collector)
  350. }
  351. // In case pedantic checks are enabled, we have to copy the map before
  352. // giving up the RLock.
  353. if r.pedanticChecksEnabled {
  354. registeredDescIDs = make(map[uint64]struct{}, len(r.descIDs))
  355. for id := range r.descIDs {
  356. registeredDescIDs[id] = struct{}{}
  357. }
  358. }
  359. r.mtx.RUnlock()
  360. // Drain metricChan in case of premature return.
  361. defer func() {
  362. for range metricChan {
  363. }
  364. }()
  365. // Gather.
  366. for metric := range metricChan {
  367. // This could be done concurrently, too, but it required locking
  368. // of metricFamiliesByName (and of metricHashes if checks are
  369. // enabled). Most likely not worth it.
  370. desc := metric.Desc()
  371. dtoMetric := &dto.Metric{}
  372. if err := metric.Write(dtoMetric); err != nil {
  373. errs = append(errs, fmt.Errorf(
  374. "error collecting metric %v: %s", desc, err,
  375. ))
  376. continue
  377. }
  378. metricFamily, ok := metricFamiliesByName[desc.fqName]
  379. if ok {
  380. if metricFamily.GetHelp() != desc.help {
  381. errs = append(errs, fmt.Errorf(
  382. "collected metric %s %s has help %q but should have %q",
  383. desc.fqName, dtoMetric, desc.help, metricFamily.GetHelp(),
  384. ))
  385. continue
  386. }
  387. // TODO(beorn7): Simplify switch once Desc has type.
  388. switch metricFamily.GetType() {
  389. case dto.MetricType_COUNTER:
  390. if dtoMetric.Counter == nil {
  391. errs = append(errs, fmt.Errorf(
  392. "collected metric %s %s should be a Counter",
  393. desc.fqName, dtoMetric,
  394. ))
  395. continue
  396. }
  397. case dto.MetricType_GAUGE:
  398. if dtoMetric.Gauge == nil {
  399. errs = append(errs, fmt.Errorf(
  400. "collected metric %s %s should be a Gauge",
  401. desc.fqName, dtoMetric,
  402. ))
  403. continue
  404. }
  405. case dto.MetricType_SUMMARY:
  406. if dtoMetric.Summary == nil {
  407. errs = append(errs, fmt.Errorf(
  408. "collected metric %s %s should be a Summary",
  409. desc.fqName, dtoMetric,
  410. ))
  411. continue
  412. }
  413. case dto.MetricType_UNTYPED:
  414. if dtoMetric.Untyped == nil {
  415. errs = append(errs, fmt.Errorf(
  416. "collected metric %s %s should be Untyped",
  417. desc.fqName, dtoMetric,
  418. ))
  419. continue
  420. }
  421. case dto.MetricType_HISTOGRAM:
  422. if dtoMetric.Histogram == nil {
  423. errs = append(errs, fmt.Errorf(
  424. "collected metric %s %s should be a Histogram",
  425. desc.fqName, dtoMetric,
  426. ))
  427. continue
  428. }
  429. default:
  430. panic("encountered MetricFamily with invalid type")
  431. }
  432. } else {
  433. metricFamily = &dto.MetricFamily{}
  434. metricFamily.Name = proto.String(desc.fqName)
  435. metricFamily.Help = proto.String(desc.help)
  436. // TODO(beorn7): Simplify switch once Desc has type.
  437. switch {
  438. case dtoMetric.Gauge != nil:
  439. metricFamily.Type = dto.MetricType_GAUGE.Enum()
  440. case dtoMetric.Counter != nil:
  441. metricFamily.Type = dto.MetricType_COUNTER.Enum()
  442. case dtoMetric.Summary != nil:
  443. metricFamily.Type = dto.MetricType_SUMMARY.Enum()
  444. case dtoMetric.Untyped != nil:
  445. metricFamily.Type = dto.MetricType_UNTYPED.Enum()
  446. case dtoMetric.Histogram != nil:
  447. metricFamily.Type = dto.MetricType_HISTOGRAM.Enum()
  448. default:
  449. errs = append(errs, fmt.Errorf(
  450. "empty metric collected: %s", dtoMetric,
  451. ))
  452. continue
  453. }
  454. metricFamiliesByName[desc.fqName] = metricFamily
  455. }
  456. if err := checkMetricConsistency(metricFamily, dtoMetric, metricHashes, dimHashes); err != nil {
  457. errs = append(errs, err)
  458. continue
  459. }
  460. if r.pedanticChecksEnabled {
  461. // Is the desc registered at all?
  462. if _, exist := registeredDescIDs[desc.id]; !exist {
  463. errs = append(errs, fmt.Errorf(
  464. "collected metric %s %s with unregistered descriptor %s",
  465. metricFamily.GetName(), dtoMetric, desc,
  466. ))
  467. continue
  468. }
  469. if err := checkDescConsistency(metricFamily, dtoMetric, desc); err != nil {
  470. errs = append(errs, err)
  471. continue
  472. }
  473. }
  474. metricFamily.Metric = append(metricFamily.Metric, dtoMetric)
  475. }
  476. return normalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap()
  477. }
  478. // Gatherers is a slice of Gatherer instances that implements the Gatherer
  479. // interface itself. Its Gather method calls Gather on all Gatherers in the
  480. // slice in order and returns the merged results. Errors returned from the
  481. // Gather calles are all returned in a flattened MultiError. Duplicate and
  482. // inconsistent Metrics are skipped (first occurrence in slice order wins) and
  483. // reported in the returned error.
  484. //
  485. // Gatherers can be used to merge the Gather results from multiple
  486. // Registries. It also provides a way to directly inject existing MetricFamily
  487. // protobufs into the gathering by creating a custom Gatherer with a Gather
  488. // method that simply returns the existing MetricFamily protobufs. Note that no
  489. // registration is involved (in contrast to Collector registration), so
  490. // obviously registration-time checks cannot happen. Any inconsistencies between
  491. // the gathered MetricFamilies are reported as errors by the Gather method, and
  492. // inconsistent Metrics are dropped. Invalid parts of the MetricFamilies
  493. // (e.g. syntactically invalid metric or label names) will go undetected.
  494. type Gatherers []Gatherer
  495. // Gather implements Gatherer.
  496. func (gs Gatherers) Gather() ([]*dto.MetricFamily, error) {
  497. var (
  498. metricFamiliesByName = map[string]*dto.MetricFamily{}
  499. metricHashes = map[uint64]struct{}{}
  500. dimHashes = map[string]uint64{}
  501. errs MultiError // The collected errors to return in the end.
  502. )
  503. for i, g := range gs {
  504. mfs, err := g.Gather()
  505. if err != nil {
  506. if multiErr, ok := err.(MultiError); ok {
  507. for _, err := range multiErr {
  508. errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err))
  509. }
  510. } else {
  511. errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err))
  512. }
  513. }
  514. for _, mf := range mfs {
  515. existingMF, exists := metricFamiliesByName[mf.GetName()]
  516. if exists {
  517. if existingMF.GetHelp() != mf.GetHelp() {
  518. errs = append(errs, fmt.Errorf(
  519. "gathered metric family %s has help %q but should have %q",
  520. mf.GetName(), mf.GetHelp(), existingMF.GetHelp(),
  521. ))
  522. continue
  523. }
  524. if existingMF.GetType() != mf.GetType() {
  525. errs = append(errs, fmt.Errorf(
  526. "gathered metric family %s has type %s but should have %s",
  527. mf.GetName(), mf.GetType(), existingMF.GetType(),
  528. ))
  529. continue
  530. }
  531. } else {
  532. existingMF = &dto.MetricFamily{}
  533. existingMF.Name = mf.Name
  534. existingMF.Help = mf.Help
  535. existingMF.Type = mf.Type
  536. metricFamiliesByName[mf.GetName()] = existingMF
  537. }
  538. for _, m := range mf.Metric {
  539. if err := checkMetricConsistency(existingMF, m, metricHashes, dimHashes); err != nil {
  540. errs = append(errs, err)
  541. continue
  542. }
  543. existingMF.Metric = append(existingMF.Metric, m)
  544. }
  545. }
  546. }
  547. return normalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap()
  548. }
  549. // metricSorter is a sortable slice of *dto.Metric.
  550. type metricSorter []*dto.Metric
  551. func (s metricSorter) Len() int {
  552. return len(s)
  553. }
  554. func (s metricSorter) Swap(i, j int) {
  555. s[i], s[j] = s[j], s[i]
  556. }
  557. func (s metricSorter) Less(i, j int) bool {
  558. if len(s[i].Label) != len(s[j].Label) {
  559. // This should not happen. The metrics are
  560. // inconsistent. However, we have to deal with the fact, as
  561. // people might use custom collectors or metric family injection
  562. // to create inconsistent metrics. So let's simply compare the
  563. // number of labels in this case. That will still yield
  564. // reproducible sorting.
  565. return len(s[i].Label) < len(s[j].Label)
  566. }
  567. for n, lp := range s[i].Label {
  568. vi := lp.GetValue()
  569. vj := s[j].Label[n].GetValue()
  570. if vi != vj {
  571. return vi < vj
  572. }
  573. }
  574. // We should never arrive here. Multiple metrics with the same
  575. // label set in the same scrape will lead to undefined ingestion
  576. // behavior. However, as above, we have to provide stable sorting
  577. // here, even for inconsistent metrics. So sort equal metrics
  578. // by their timestamp, with missing timestamps (implying "now")
  579. // coming last.
  580. if s[i].TimestampMs == nil {
  581. return false
  582. }
  583. if s[j].TimestampMs == nil {
  584. return true
  585. }
  586. return s[i].GetTimestampMs() < s[j].GetTimestampMs()
  587. }
  588. // normalizeMetricFamilies returns a MetricFamily slice with empty
  589. // MetricFamilies pruned and the remaining MetricFamilies sorted by name within
  590. // the slice, with the contained Metrics sorted within each MetricFamily.
  591. func normalizeMetricFamilies(metricFamiliesByName map[string]*dto.MetricFamily) []*dto.MetricFamily {
  592. for _, mf := range metricFamiliesByName {
  593. sort.Sort(metricSorter(mf.Metric))
  594. }
  595. names := make([]string, 0, len(metricFamiliesByName))
  596. for name, mf := range metricFamiliesByName {
  597. if len(mf.Metric) > 0 {
  598. names = append(names, name)
  599. }
  600. }
  601. sort.Strings(names)
  602. result := make([]*dto.MetricFamily, 0, len(names))
  603. for _, name := range names {
  604. result = append(result, metricFamiliesByName[name])
  605. }
  606. return result
  607. }
  608. // checkMetricConsistency checks if the provided Metric is consistent with the
  609. // provided MetricFamily. It also hashed the Metric labels and the MetricFamily
  610. // name. If the resulting hash is alread in the provided metricHashes, an error
  611. // is returned. If not, it is added to metricHashes. The provided dimHashes maps
  612. // MetricFamily names to their dimHash (hashed sorted label names). If dimHashes
  613. // doesn't yet contain a hash for the provided MetricFamily, it is
  614. // added. Otherwise, an error is returned if the existing dimHashes in not equal
  615. // the calculated dimHash.
  616. func checkMetricConsistency(
  617. metricFamily *dto.MetricFamily,
  618. dtoMetric *dto.Metric,
  619. metricHashes map[uint64]struct{},
  620. dimHashes map[string]uint64,
  621. ) error {
  622. // Type consistency with metric family.
  623. if metricFamily.GetType() == dto.MetricType_GAUGE && dtoMetric.Gauge == nil ||
  624. metricFamily.GetType() == dto.MetricType_COUNTER && dtoMetric.Counter == nil ||
  625. metricFamily.GetType() == dto.MetricType_SUMMARY && dtoMetric.Summary == nil ||
  626. metricFamily.GetType() == dto.MetricType_HISTOGRAM && dtoMetric.Histogram == nil ||
  627. metricFamily.GetType() == dto.MetricType_UNTYPED && dtoMetric.Untyped == nil {
  628. return fmt.Errorf(
  629. "collected metric %s %s is not a %s",
  630. metricFamily.GetName(), dtoMetric, metricFamily.GetType(),
  631. )
  632. }
  633. for _, labelPair := range dtoMetric.GetLabel() {
  634. if !utf8.ValidString(*labelPair.Value) {
  635. return fmt.Errorf("collected metric's label %s is not utf8: %#v", *labelPair.Name, *labelPair.Value)
  636. }
  637. }
  638. // Is the metric unique (i.e. no other metric with the same name and the same label values)?
  639. h := hashNew()
  640. h = hashAdd(h, metricFamily.GetName())
  641. h = hashAddByte(h, separatorByte)
  642. dh := hashNew()
  643. // Make sure label pairs are sorted. We depend on it for the consistency
  644. // check.
  645. sort.Sort(LabelPairSorter(dtoMetric.Label))
  646. for _, lp := range dtoMetric.Label {
  647. h = hashAdd(h, lp.GetValue())
  648. h = hashAddByte(h, separatorByte)
  649. dh = hashAdd(dh, lp.GetName())
  650. dh = hashAddByte(dh, separatorByte)
  651. }
  652. if _, exists := metricHashes[h]; exists {
  653. return fmt.Errorf(
  654. "collected metric %s %s was collected before with the same name and label values",
  655. metricFamily.GetName(), dtoMetric,
  656. )
  657. }
  658. if dimHash, ok := dimHashes[metricFamily.GetName()]; ok {
  659. if dimHash != dh {
  660. return fmt.Errorf(
  661. "collected metric %s %s has label dimensions inconsistent with previously collected metrics in the same metric family",
  662. metricFamily.GetName(), dtoMetric,
  663. )
  664. }
  665. } else {
  666. dimHashes[metricFamily.GetName()] = dh
  667. }
  668. metricHashes[h] = struct{}{}
  669. return nil
  670. }
  671. func checkDescConsistency(
  672. metricFamily *dto.MetricFamily,
  673. dtoMetric *dto.Metric,
  674. desc *Desc,
  675. ) error {
  676. // Desc help consistency with metric family help.
  677. if metricFamily.GetHelp() != desc.help {
  678. return fmt.Errorf(
  679. "collected metric %s %s has help %q but should have %q",
  680. metricFamily.GetName(), dtoMetric, metricFamily.GetHelp(), desc.help,
  681. )
  682. }
  683. // Is the desc consistent with the content of the metric?
  684. lpsFromDesc := make([]*dto.LabelPair, 0, len(dtoMetric.Label))
  685. lpsFromDesc = append(lpsFromDesc, desc.constLabelPairs...)
  686. for _, l := range desc.variableLabels {
  687. lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{
  688. Name: proto.String(l),
  689. })
  690. }
  691. if len(lpsFromDesc) != len(dtoMetric.Label) {
  692. return fmt.Errorf(
  693. "labels in collected metric %s %s are inconsistent with descriptor %s",
  694. metricFamily.GetName(), dtoMetric, desc,
  695. )
  696. }
  697. sort.Sort(LabelPairSorter(lpsFromDesc))
  698. for i, lpFromDesc := range lpsFromDesc {
  699. lpFromMetric := dtoMetric.Label[i]
  700. if lpFromDesc.GetName() != lpFromMetric.GetName() ||
  701. lpFromDesc.Value != nil && lpFromDesc.GetValue() != lpFromMetric.GetValue() {
  702. return fmt.Errorf(
  703. "labels in collected metric %s %s are inconsistent with descriptor %s",
  704. metricFamily.GetName(), dtoMetric, desc,
  705. )
  706. }
  707. }
  708. return nil
  709. }