View Javadoc
1   /*
2    * Copyright (C) 2012 The Guava Authors
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
5    * in compliance with the License. You may obtain a copy of the License at
6    *
7    * http://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software distributed under the License
10   * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
11   * or implied. See the License for the specific language governing permissions and limitations under
12   * the License.
13   */
14  package com.google.common.util.concurrent;
15  
16  import static com.google.common.base.Preconditions.checkArgument;
17  import static com.google.common.base.Preconditions.checkNotNull;
18  import static com.google.common.base.Preconditions.checkState;
19  import static com.google.common.base.Predicates.equalTo;
20  import static com.google.common.base.Predicates.in;
21  import static com.google.common.base.Predicates.instanceOf;
22  import static com.google.common.base.Predicates.not;
23  import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
24  import static com.google.common.util.concurrent.Service.State.FAILED;
25  import static com.google.common.util.concurrent.Service.State.NEW;
26  import static com.google.common.util.concurrent.Service.State.RUNNING;
27  import static com.google.common.util.concurrent.Service.State.STARTING;
28  import static com.google.common.util.concurrent.Service.State.STOPPING;
29  import static com.google.common.util.concurrent.Service.State.TERMINATED;
30  import static java.util.concurrent.TimeUnit.MILLISECONDS;
31  
32  import com.google.common.annotations.Beta;
33  import com.google.common.annotations.GwtIncompatible;
34  import com.google.common.base.Function;
35  import com.google.common.base.MoreObjects;
36  import com.google.common.base.Stopwatch;
37  import com.google.common.collect.Collections2;
38  import com.google.common.collect.ImmutableCollection;
39  import com.google.common.collect.ImmutableList;
40  import com.google.common.collect.ImmutableMap;
41  import com.google.common.collect.ImmutableMultimap;
42  import com.google.common.collect.ImmutableSet;
43  import com.google.common.collect.ImmutableSetMultimap;
44  import com.google.common.collect.Lists;
45  import com.google.common.collect.Maps;
46  import com.google.common.collect.MultimapBuilder;
47  import com.google.common.collect.Multimaps;
48  import com.google.common.collect.Multiset;
49  import com.google.common.collect.Ordering;
50  import com.google.common.collect.SetMultimap;
51  import com.google.common.util.concurrent.Service.State;
52  import com.google.errorprone.annotations.CanIgnoreReturnValue;
53  import com.google.j2objc.annotations.WeakOuter;
54  import java.lang.ref.WeakReference;
55  import java.util.Collections;
56  import java.util.EnumSet;
57  import java.util.List;
58  import java.util.Map;
59  import java.util.Map.Entry;
60  import java.util.concurrent.Executor;
61  import java.util.concurrent.TimeUnit;
62  import java.util.concurrent.TimeoutException;
63  import java.util.logging.Level;
64  import java.util.logging.Logger;
65  import javax.annotation.concurrent.GuardedBy;
66  
67  /**
68   * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
69   * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
70   * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
71   * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
72   * mechanism.
73   *
74   * <p>While it is recommended that service lifecycles be managed via this class, state transitions
75   * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
76   * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
77   * when appropriate and {@link #awaitHealthy} will still work as expected.
78   *
79   * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
80   * <pre>   {@code
81   * class Server {
82   *   public static void main(String[] args) {
83   *     Set<Service> services = ...;
84   *     ServiceManager manager = new ServiceManager(services);
85   *     manager.addListener(new Listener() {
86   *         public void stopped() {}
87   *         public void healthy() {
88   *           // Services have been initialized and are healthy, start accepting requests...
89   *         }
90   *         public void failure(Service service) {
91   *           // Something failed, at this point we could log it, notify a load balancer, or take
92   *           // some other action.  For now we will just exit.
93   *           System.exit(1);
94   *         }
95   *       },
96   *       MoreExecutors.directExecutor());
97   *
98   *     Runtime.getRuntime().addShutdownHook(new Thread() {
99   *       public void run() {
100  *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
101  *         // requests.
102  *         try {
103  *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
104  *         } catch (TimeoutException timeout) {
105  *           // stopping timed out
106  *         }
107  *       }
108  *     });
109  *     manager.startAsync();  // start all the services asynchronously
110  *   }
111  * }}</pre>
112  *
113  * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
114  * service failure and to ensure that when the JVM is shutting down all the services are stopped.
115  *
116  * @author Luke Sandberg
117  * @since 14.0
118  */
119 @Beta
120 @GwtIncompatible
121 public final class ServiceManager {
122   private static final Logger logger = Logger.getLogger(ServiceManager.class.getName());
123   private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
124       new ListenerCallQueue.Event<Listener>() {
125         @Override
126         public void call(Listener listener) {
127           listener.healthy();
128         }
129 
130         @Override
131         public String toString() {
132           return "healthy()";
133         }
134       };
135   private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
136       new ListenerCallQueue.Event<Listener>() {
137         @Override
138         public void call(Listener listener) {
139           listener.stopped();
140         }
141 
142         @Override
143         public String toString() {
144           return "stopped()";
145         }
146       };
147 
148   /**
149    * A listener for the aggregate state changes of the services that are under management. Users
150    * that need to listen to more fine-grained events (such as when each particular
151    * {@linkplain Service service} starts, or terminates), should attach {@linkplain Service.Listener
152    * service listeners} to each individual service.
153    *
154    * @author Luke Sandberg
155    * @since 15.0 (present as an interface in 14.0)
156    */
157   @Beta // Should come out of Beta when ServiceManager does
158   public abstract static class Listener {
159     /**
160      * Called when the service initially becomes healthy.
161      *
162      * <p>This will be called at most once after all the services have entered the
163      * {@linkplain State#RUNNING running} state. If any services fail during start up or
164      * {@linkplain State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other
165      * services have started {@linkplain State#RUNNING running} then this method will not be called.
166      */
167     public void healthy() {}
168 
169     /**
170      * Called when the all of the component services have reached a terminal state, either
171      * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
172      */
173     public void stopped() {}
174 
175     /**
176      * Called when a component service has {@linkplain State#FAILED failed}.
177      *
178      * @param service The service that failed.
179      */
180     public void failure(Service service) {}
181   }
182 
183   /**
184    * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
185    * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
186    * be made {@code static} and its instances can be safely constructed and added in the
187    * {@link ServiceManager} constructor without having to close over the partially constructed
188    * {@link ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
189    */
190   private final ServiceManagerState state;
191   private final ImmutableList<Service> services;
192 
193   /**
194    * Constructs a new instance for managing the given services.
195    *
196    * @param services The services to manage
197    *
198    * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
199    *     are any duplicate services.
200    */
201   public ServiceManager(Iterable<? extends Service> services) {
202     ImmutableList<Service> copy = ImmutableList.copyOf(services);
203     if (copy.isEmpty()) {
204       // Having no services causes the manager to behave strangely. Notably, listeners are never
205       // fired. To avoid this we substitute a placeholder service.
206       logger.log(
207           Level.WARNING,
208           "ServiceManager configured with no services.  Is your application configured properly?",
209           new EmptyServiceManagerWarning());
210       copy = ImmutableList.<Service>of(new NoOpService());
211     }
212     this.state = new ServiceManagerState(copy);
213     this.services = copy;
214     WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
215     for (Service service : copy) {
216       service.addListener(new ServiceListener(service, stateReference), directExecutor());
217       // We check the state after adding the listener as a way to ensure that our listener was added
218       // to a NEW service.
219       checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
220     }
221     // We have installed all of our listeners and after this point any state transition should be
222     // correct.
223     this.state.markReady();
224   }
225 
226   /**
227    * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
228    * executor. The listener will not have previous state changes replayed, so it is suggested that
229    * listeners are added before any of the managed services are {@linkplain Service#startAsync
230    * started}.
231    *
232    * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
233    * across calls to multiple listeners. Specifically, a given listener will have its callbacks
234    * invoked in the same order as the underlying service enters those states. Additionally, at most
235    * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
236    * may execute concurrently, and listeners may execute in an order different from the one in which
237    * they were registered.
238    *
239    * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
240    * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
241    * logged.
242    *
243    * <p>For fast, lightweight listeners that would be safe to execute in any thread, consider
244    * calling {@link #addListener(Listener)}.
245    *
246    * @param listener the listener to run when the manager changes state
247    * @param executor the executor in which the listeners callback methods will be run.
248    */
249   public void addListener(Listener listener, Executor executor) {
250     state.addListener(listener, executor);
251   }
252 
253   /**
254    * Registers a {@link Listener} to be run when this {@link ServiceManager} changes state. The
255    * listener will not have previous state changes replayed, so it is suggested that listeners are
256    * added before any of the managed services are {@linkplain Service#startAsync started}.
257    *
258    * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
259    * across calls to multiple listeners. Specifically, a given listener will have its callbacks
260    * invoked in the same order as the underlying service enters those states. Additionally, at most
261    * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
262    * may execute concurrently, and listeners may execute in an order different from the one in which
263    * they were registered.
264    *
265    * <p>RuntimeExceptions thrown by a listener will be caught and logged.
266    *
267    * @param listener the listener to run when the manager changes state
268    */
269   public void addListener(Listener listener) {
270     state.addListener(listener, directExecutor());
271   }
272 
273   /**
274    * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
275    * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
276    *
277    * @return this
278    * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
279    *     method is called.
280    */
281   @CanIgnoreReturnValue
282   public ServiceManager startAsync() {
283     for (Service service : services) {
284       State state = service.state();
285       checkState(state == NEW, "Service %s is %s, cannot start it.", service, state);
286     }
287     for (Service service : services) {
288       try {
289         state.tryStartTiming(service);
290         service.startAsync();
291       } catch (IllegalStateException e) {
292         // This can happen if the service has already been started or stopped (e.g. by another
293         // service or listener). Our contract says it is safe to call this method if
294         // all services were NEW when it was called, and this has already been verified above, so we
295         // don't propagate the exception.
296         logger.log(Level.WARNING, "Unable to start Service " + service, e);
297       }
298     }
299     return this;
300   }
301 
302   /**
303    * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
304    * will become healthy after all the component services have reached the {@linkplain State#RUNNING
305    * running} state.
306    *
307    * @throws IllegalStateException if the service manager reaches a state from which it cannot
308    *     become {@linkplain #isHealthy() healthy}.
309    */
310   public void awaitHealthy() {
311     state.awaitHealthy();
312   }
313 
314   /**
315    * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
316    * than the given time. The manager will become healthy after all the component services have
317    * reached the {@linkplain State#RUNNING running} state.
318    *
319    * @param timeout the maximum time to wait
320    * @param unit the time unit of the timeout argument
321    * @throws TimeoutException if not all of the services have finished starting within the deadline
322    * @throws IllegalStateException if the service manager reaches a state from which it cannot
323    *     become {@linkplain #isHealthy() healthy}.
324    */
325   public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
326     state.awaitHealthy(timeout, unit);
327   }
328 
329   /**
330    * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
331    * being managed.
332    *
333    * @return this
334    */
335   @CanIgnoreReturnValue
336   public ServiceManager stopAsync() {
337     for (Service service : services) {
338       service.stopAsync();
339     }
340     return this;
341   }
342 
343   /**
344    * Waits for the all the services to reach a terminal state. After this method returns all
345    * services will either be {@linkplain Service.State#TERMINATED terminated} or
346    * {@linkplain Service.State#FAILED failed}.
347    */
348   public void awaitStopped() {
349     state.awaitStopped();
350   }
351 
352   /**
353    * Waits for the all the services to reach a terminal state for no more than the given time. After
354    * this method returns all services will either be {@linkplain Service.State#TERMINATED
355    * terminated} or {@linkplain Service.State#FAILED failed}.
356    *
357    * @param timeout the maximum time to wait
358    * @param unit the time unit of the timeout argument
359    * @throws TimeoutException if not all of the services have stopped within the deadline
360    */
361   public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
362     state.awaitStopped(timeout, unit);
363   }
364 
365   /**
366    * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
367    *
368    * <p>Users who want more detailed information should use the {@link #servicesByState} method to
369    * get detailed information about which services are not running.
370    */
371   public boolean isHealthy() {
372     for (Service service : services) {
373       if (!service.isRunning()) {
374         return false;
375       }
376     }
377     return true;
378   }
379 
380   /**
381    * Provides a snapshot of the current state of all the services under management.
382    *
383    * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
384    * correspond to a point in time view of the services.
385    */
386   public ImmutableMultimap<State, Service> servicesByState() {
387     return state.servicesByState();
388   }
389 
390   /**
391    * Returns the service load times. This value will only return startup times for services that
392    * have finished starting.
393    *
394    * @return Map of services and their corresponding startup time in millis, the map entries will be
395    *     ordered by startup time.
396    */
397   public ImmutableMap<Service, Long> startupTimes() {
398     return state.startupTimes();
399   }
400 
401   @Override
402   public String toString() {
403     return MoreObjects.toStringHelper(ServiceManager.class)
404         .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
405         .toString();
406   }
407 
408   /**
409    * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
410    * accessed by instances of {@link ServiceListener}.
411    */
412   private static final class ServiceManagerState {
413     final Monitor monitor = new Monitor();
414 
415     @GuardedBy("monitor")
416     final SetMultimap<State, Service> servicesByState =
417         MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
418 
419     @GuardedBy("monitor")
420     final Multiset<State> states = servicesByState.keys();
421 
422     @GuardedBy("monitor")
423     final Map<Service, Stopwatch> startupTimers = Maps.newIdentityHashMap();
424 
425     /**
426      * These two booleans are used to mark the state as ready to start.
427      *
428      * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
429      * correctly installed
430      *
431      * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
432      * transition has been performed.
433      *
434      * <p>Together, they allow us to enforce that all services have their listeners installed prior
435      * to any service performing a transition, then we can fail in the ServiceManager constructor
436      * rather than in a Service.Listener callback.
437      */
438     @GuardedBy("monitor")
439     boolean ready;
440 
441     @GuardedBy("monitor")
442     boolean transitioned;
443 
444     final int numberOfServices;
445 
446     /**
447      * Controls how long to wait for all the services to either become healthy or reach a state from
448      * which it is guaranteed that it can never become healthy.
449      */
450     final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
451 
452     @WeakOuter
453     final class AwaitHealthGuard extends Monitor.Guard {
454       AwaitHealthGuard() {
455         super(ServiceManagerState.this.monitor);
456       }
457 
458       @Override
459       @GuardedBy("ServiceManagerState.this.monitor")
460       public boolean isSatisfied() {
461         // All services have started or some service has terminated/failed.
462         return states.count(RUNNING) == numberOfServices
463             || states.contains(STOPPING)
464             || states.contains(TERMINATED)
465             || states.contains(FAILED);
466       }
467     }
468 
469     /**
470      * Controls how long to wait for all services to reach a terminal state.
471      */
472     final Monitor.Guard stoppedGuard = new StoppedGuard();
473 
474     @WeakOuter
475     final class StoppedGuard extends Monitor.Guard {
476       StoppedGuard() {
477         super(ServiceManagerState.this.monitor);
478       }
479 
480       @Override
481       @GuardedBy("ServiceManagerState.this.monitor")
482       public boolean isSatisfied() {
483         return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
484       }
485     }
486 
487     /** The listeners to notify during a state transition. */
488     final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
489 
490     /**
491      * It is implicitly assumed that all the services are NEW and that they will all remain NEW
492      * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
493      * responsibility to only call {@link #markReady()} if all services were new at the time this
494      * method was called and when all the listeners were installed.
495      */
496     ServiceManagerState(ImmutableCollection<Service> services) {
497       this.numberOfServices = services.size();
498       servicesByState.putAll(NEW, services);
499     }
500 
501     /**
502      * Attempts to start the timer immediately prior to the service being started via
503      * {@link Service#startAsync()}.
504      */
505     void tryStartTiming(Service service) {
506       monitor.enter();
507       try {
508         Stopwatch stopwatch = startupTimers.get(service);
509         if (stopwatch == null) {
510           startupTimers.put(service, Stopwatch.createStarted());
511         }
512       } finally {
513         monitor.leave();
514       }
515     }
516 
517     /**
518      * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
519      * been observed yet.
520      */
521     void markReady() {
522       monitor.enter();
523       try {
524         if (!transitioned) {
525           // nothing has transitioned since construction, good.
526           ready = true;
527         } else {
528           // This should be an extremely rare race condition.
529           List<Service> servicesInBadStates = Lists.newArrayList();
530           for (Service service : servicesByState().values()) {
531             if (service.state() != NEW) {
532               servicesInBadStates.add(service);
533             }
534           }
535           throw new IllegalArgumentException(
536               "Services started transitioning asynchronously before "
537                   + "the ServiceManager was constructed: "
538                   + servicesInBadStates);
539         }
540       } finally {
541         monitor.leave();
542       }
543     }
544 
545     void addListener(Listener listener, Executor executor) {
546       listeners.addListener(listener, executor);
547     }
548 
549     void awaitHealthy() {
550       monitor.enterWhenUninterruptibly(awaitHealthGuard);
551       try {
552         checkHealthy();
553       } finally {
554         monitor.leave();
555       }
556     }
557 
558     void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
559       monitor.enter();
560       try {
561         if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
562           throw new TimeoutException(
563               "Timeout waiting for the services to become healthy. The "
564                   + "following services have not started: "
565                   + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
566         }
567         checkHealthy();
568       } finally {
569         monitor.leave();
570       }
571     }
572 
573     void awaitStopped() {
574       monitor.enterWhenUninterruptibly(stoppedGuard);
575       monitor.leave();
576     }
577 
578     void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
579       monitor.enter();
580       try {
581         if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
582           throw new TimeoutException(
583               "Timeout waiting for the services to stop. The following "
584                   + "services have not stopped: "
585                   + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
586         }
587       } finally {
588         monitor.leave();
589       }
590     }
591 
592     ImmutableMultimap<State, Service> servicesByState() {
593       ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
594       monitor.enter();
595       try {
596         for (Entry<State, Service> entry : servicesByState.entries()) {
597           if (!(entry.getValue() instanceof NoOpService)) {
598             builder.put(entry);
599           }
600         }
601       } finally {
602         monitor.leave();
603       }
604       return builder.build();
605     }
606 
607     ImmutableMap<Service, Long> startupTimes() {
608       List<Entry<Service, Long>> loadTimes;
609       monitor.enter();
610       try {
611         loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
612         // N.B. There will only be an entry in the map if the service has started
613         for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
614           Service service = entry.getKey();
615           Stopwatch stopWatch = entry.getValue();
616           if (!stopWatch.isRunning() && !(service instanceof NoOpService)) {
617             loadTimes.add(Maps.immutableEntry(service, stopWatch.elapsed(MILLISECONDS)));
618           }
619         }
620       } finally {
621         monitor.leave();
622       }
623       Collections.sort(
624           loadTimes,
625           Ordering.natural()
626               .onResultOf(
627                   new Function<Entry<Service, Long>, Long>() {
628                     @Override
629                     public Long apply(Map.Entry<Service, Long> input) {
630                       return input.getValue();
631                     }
632                   }));
633       return ImmutableMap.copyOf(loadTimes);
634     }
635 
636     /**
637      * Updates the state with the given service transition.
638      *
639      * <p>This method performs the main logic of ServiceManager in the following steps.
640      * <ol>
641      * <li>Update the {@link #servicesByState()}
642      * <li>Update the {@link #startupTimers}
643      * <li>Based on the new state queue listeners to run
644      * <li>Run the listeners (outside of the lock)
645      * </ol>
646      */
647     void transitionService(final Service service, State from, State to) {
648       checkNotNull(service);
649       checkArgument(from != to);
650       monitor.enter();
651       try {
652         transitioned = true;
653         if (!ready) {
654           return;
655         }
656         // Update state.
657         checkState(
658             servicesByState.remove(from, service),
659             "Service %s not at the expected location in the state map %s",
660             service,
661             from);
662         checkState(
663             servicesByState.put(to, service),
664             "Service %s in the state map unexpectedly at %s",
665             service,
666             to);
667         // Update the timer
668         Stopwatch stopwatch = startupTimers.get(service);
669         if (stopwatch == null) {
670           // This means the service was started by some means other than ServiceManager.startAsync
671           stopwatch = Stopwatch.createStarted();
672           startupTimers.put(service, stopwatch);
673         }
674         if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
675           // N.B. if we miss the STARTING event then we may never record a startup time.
676           stopwatch.stop();
677           if (!(service instanceof NoOpService)) {
678             logger.log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
679           }
680         }
681         // Queue our listeners
682 
683         // Did a service fail?
684         if (to == FAILED) {
685           enqueueFailedEvent(service);
686         }
687 
688         if (states.count(RUNNING) == numberOfServices) {
689           // This means that the manager is currently healthy. N.B. If other threads call isHealthy
690           // they are not guaranteed to get 'true', because any service could fail right now.
691           enqueueHealthyEvent();
692         } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
693           enqueueStoppedEvent();
694         }
695       } finally {
696         monitor.leave();
697         // Run our executors outside of the lock
698         dispatchListenerEvents();
699       }
700     }
701 
702     void enqueueStoppedEvent() {
703       listeners.enqueue(STOPPED_EVENT);
704     }
705 
706     void enqueueHealthyEvent() {
707       listeners.enqueue(HEALTHY_EVENT);
708     }
709 
710     void enqueueFailedEvent(final Service service) {
711       listeners.enqueue(
712           new ListenerCallQueue.Event<Listener>() {
713             @Override
714             public void call(Listener listener) {
715               listener.failure(service);
716             }
717 
718             @Override
719             public String toString() {
720               return "failed({service=" + service + "})";
721             }
722           });
723     }
724 
725     /** Attempts to execute all the listeners in {@link #listeners}. */
726     void dispatchListenerEvents() {
727       checkState(
728           !monitor.isOccupiedByCurrentThread(),
729           "It is incorrect to execute listeners with the monitor held.");
730       listeners.dispatch();
731     }
732 
733     @GuardedBy("monitor")
734     void checkHealthy() {
735       if (states.count(RUNNING) != numberOfServices) {
736         IllegalStateException exception =
737             new IllegalStateException(
738                 "Expected to be healthy after starting. The following services are not running: "
739                     + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
740         for (Service service : servicesByState.get(State.FAILED)) {
741           exception.addSuppressed(new FailedService(service));
742         }
743         throw exception;
744       }
745     }
746   }
747 
748   /**
749    * A {@link Service} that wraps another service and times how long it takes for it to start and
750    * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
751    * the state transitions.
752    */
753   private static final class ServiceListener extends Service.Listener {
754     final Service service;
755     // We store the state in a weak reference to ensure that if something went wrong while
756     // constructing the ServiceManager we don't pointlessly keep updating the state.
757     final WeakReference<ServiceManagerState> state;
758 
759     ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
760       this.service = service;
761       this.state = state;
762     }
763 
764     @Override
765     public void starting() {
766       ServiceManagerState state = this.state.get();
767       if (state != null) {
768         state.transitionService(service, NEW, STARTING);
769         if (!(service instanceof NoOpService)) {
770           logger.log(Level.FINE, "Starting {0}.", service);
771         }
772       }
773     }
774 
775     @Override
776     public void running() {
777       ServiceManagerState state = this.state.get();
778       if (state != null) {
779         state.transitionService(service, STARTING, RUNNING);
780       }
781     }
782 
783     @Override
784     public void stopping(State from) {
785       ServiceManagerState state = this.state.get();
786       if (state != null) {
787         state.transitionService(service, from, STOPPING);
788       }
789     }
790 
791     @Override
792     public void terminated(State from) {
793       ServiceManagerState state = this.state.get();
794       if (state != null) {
795         if (!(service instanceof NoOpService)) {
796           logger.log(
797               Level.FINE,
798               "Service {0} has terminated. Previous state was: {1}",
799               new Object[] {service, from});
800         }
801         state.transitionService(service, from, TERMINATED);
802       }
803     }
804 
805     @Override
806     public void failed(State from, Throwable failure) {
807       ServiceManagerState state = this.state.get();
808       if (state != null) {
809         // Log before the transition, so that if the process exits in response to server failure,
810         // there is a higher likelihood that the cause will be in the logs.
811         boolean log = !(service instanceof NoOpService);
812         /*
813          * We have already exposed startup exceptions to the user in the form of suppressed
814          * exceptions. We don't need to log those exceptions again.
815          */
816         log &= from != State.STARTING;
817         if (log) {
818           logger.log(
819               Level.SEVERE,
820               "Service " + service + " has failed in the " + from + " state.",
821               failure);
822         }
823         state.transitionService(service, from, FAILED);
824       }
825     }
826   }
827 
828   /**
829    * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
830    * that the {@link ServiceManager} functions properly even when it is managing no services.
831    *
832    * <p>The use of this class is considered an implementation detail of ServiceManager and as such
833    * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
834    * logging statements.
835    */
836   private static final class NoOpService extends AbstractService {
837     @Override
838     protected void doStart() {
839       notifyStarted();
840     }
841 
842     @Override
843     protected void doStop() {
844       notifyStopped();
845     }
846   }
847 
848   /** This is never thrown but only used for logging. */
849   private static final class EmptyServiceManagerWarning extends Throwable {}
850 
851   private static final class FailedService extends Throwable {
852     FailedService(Service service) {
853       super(
854           service.toString(),
855           service.failureCause(),
856           false /* don't enable suppression */,
857           false /* don't calculate a stack trace. */);
858     }
859   }
860 }