View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.handler;
20  
21  import java.io.IOException;
22  import java.util.concurrent.atomic.AtomicBoolean;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.HRegionInfo;
28  import org.apache.hadoop.hbase.HTableDescriptor;
29  import org.apache.hadoop.hbase.Server;
30  import org.apache.hadoop.hbase.coordination.OpenRegionCoordination;
31  import org.apache.hadoop.hbase.executor.EventHandler;
32  import org.apache.hadoop.hbase.executor.EventType;
33  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
34  import org.apache.hadoop.hbase.regionserver.HRegion;
35  import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
36  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
37  import org.apache.hadoop.hbase.util.CancelableProgressable;
38  import org.apache.hadoop.hbase.util.ConfigUtil;
39  /**
40   * Handles opening of a region on a region server.
41   * <p>
42   * This is executed after receiving an OPEN RPC from the master or client.
43   */
44  @InterfaceAudience.Private
45  public class OpenRegionHandler extends EventHandler {
46    private static final Log LOG = LogFactory.getLog(OpenRegionHandler.class);
47  
48    protected final RegionServerServices rsServices;
49  
50    private final HRegionInfo regionInfo;
51    private final HTableDescriptor htd;
52  
53    private OpenRegionCoordination coordination;
54    private OpenRegionCoordination.OpenRegionDetails ord;
55  
56    private final boolean useZKForAssignment;
57  
58    public OpenRegionHandler(final Server server,
59        final RegionServerServices rsServices, HRegionInfo regionInfo,
60        HTableDescriptor htd, OpenRegionCoordination coordination,
61        OpenRegionCoordination.OpenRegionDetails ord) {
62      this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION,
63          coordination, ord);
64    }
65  
66    protected OpenRegionHandler(final Server server,
67        final RegionServerServices rsServices, final HRegionInfo regionInfo,
68        final HTableDescriptor htd, EventType eventType,
69        OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord) {
70      super(server, eventType);
71      this.rsServices = rsServices;
72      this.regionInfo = regionInfo;
73      this.htd = htd;
74      this.coordination = coordination;
75      this.ord = ord;
76      useZKForAssignment = ConfigUtil.useZKForAssignment(server.getConfiguration());
77    }
78  
79    public HRegionInfo getRegionInfo() {
80      return regionInfo;
81    }
82  
83    @Override
84    public void process() throws IOException {
85      boolean openSuccessful = false;
86      boolean transitionedToOpening = false;
87      final String regionName = regionInfo.getRegionNameAsString();
88      HRegion region = null;
89  
90      try {
91        if (this.server.isStopped() || this.rsServices.isStopping()) {
92          return;
93        }
94        final String encodedName = regionInfo.getEncodedName();
95  
96        // 3 different difficult situations can occur
97        // 1) The opening was cancelled. This is an expected situation
98        // 2) The region was hijacked, we no longer have the znode
99        // 3) The region is now marked as online while we're suppose to open. This would be a bug.
100 
101       // Check that this region is not already online
102       if (this.rsServices.getFromOnlineRegions(encodedName) != null) {
103         LOG.error("Region " + encodedName +
104             " was already online when we started processing the opening. " +
105             "Marking this new attempt as failed");
106         return;
107       }
108 
109       // Check that we're still supposed to open the region and transition.
110       // If fails, just return.  Someone stole the region from under us.
111       // Calling transitionFromOfflineToOpening initializes this.version.
112       if (!isRegionStillOpening()){
113         LOG.error("Region " + encodedName + " opening cancelled");
114         return;
115       }
116 
117       if (useZKForAssignment
118           && !coordination.transitionFromOfflineToOpening(regionInfo, ord)) {
119         LOG.warn("Region was hijacked? Opening cancelled for encodedName=" + encodedName);
120         // This is a desperate attempt: the znode is unlikely to be ours. But we can't do more.
121         return;
122       }
123       transitionedToOpening = true;
124       // Open region.  After a successful open, failures in subsequent
125       // processing needs to do a close as part of cleanup.
126       region = openRegion();
127       if (region == null) {
128         return;
129       }
130 
131       boolean failed = true;
132       if (!useZKForAssignment ||
133           coordination.tickleOpening(ord, regionInfo, rsServices, "post_region_open")) {
134         if (updateMeta(region)) {
135           failed = false;
136         }
137       }
138       if (failed || this.server.isStopped() ||
139           this.rsServices.isStopping()) {
140         return;
141       }
142 
143       if (!isRegionStillOpening() ||
144           (useZKForAssignment && !coordination.transitionToOpened(region, ord))) {
145         // If we fail to transition to opened, it's because of one of two cases:
146         //    (a) we lost our ZK lease
147         // OR (b) someone else opened the region before us
148         // OR (c) someone cancelled the open
149         // In all cases, we try to transition to failed_open to be safe.
150         return;
151       }
152 
153       // We have a znode in the opened state now. We can't really delete it as the master job.
154       // Transitioning to failed open would create a race condition if the master has already
155       // acted the transition to opened.
156       // Cancelling the open is dangerous, because we would have a state where the master thinks
157       // the region is opened while the region is actually closed. It is a dangerous state
158       // to be in. For this reason, from now on, we're not going back. There is a message in the
159       // finally close to let the admin knows where we stand.
160 
161 
162       // Successful region open, and add it to OnlineRegions
163       this.rsServices.addToOnlineRegions(region);
164       openSuccessful = true;
165 
166       // Done!  Successful region open
167       LOG.debug("Opened " + regionName + " on " +
168         this.server.getServerName());
169 
170 
171     } finally {
172       // Do all clean up here
173       if (!openSuccessful) {
174         doCleanUpOnFailedOpen(region, transitionedToOpening, ord);
175       }
176       final Boolean current = this.rsServices.getRegionsInTransitionInRS().
177           remove(this.regionInfo.getEncodedNameAsBytes());
178 
179       // Let's check if we have met a race condition on open cancellation....
180       // A better solution would be to not have any race condition.
181       // this.rsServices.getRegionsInTransitionInRS().remove(
182       //  this.regionInfo.getEncodedNameAsBytes(), Boolean.TRUE);
183       // would help, but we would still have a consistency issue to manage with
184       // 1) this.rsServices.addToOnlineRegions(region);
185       // 2) the ZK state.
186       if (openSuccessful) {
187         if (current == null) { // Should NEVER happen, but let's be paranoid.
188           LOG.error("Bad state: we've just opened a region that was NOT in transition. Region="
189               + regionName);
190         } else if (Boolean.FALSE.equals(current)) { // Can happen, if we're
191                                                     // really unlucky.
192           LOG.error("Race condition: we've finished to open a region, while a close was requested "
193               + " on region=" + regionName + ". It can be a critical error, as a region that"
194               + " should be closed is now opened. Closing it now");
195           cleanupFailedOpen(region);
196         }
197       }
198     }
199   }
200 
201   private void doCleanUpOnFailedOpen(HRegion region, boolean transitionedToOpening,
202                                      OpenRegionCoordination.OpenRegionDetails ord)
203       throws IOException {
204     if (transitionedToOpening) {
205       try {
206         if (region != null) {
207           cleanupFailedOpen(region);
208         }
209       } finally {
210         if (!useZKForAssignment) {
211           rsServices.reportRegionStateTransition(TransitionCode.FAILED_OPEN, regionInfo);
212         } else {
213           // Even if cleanupFailed open fails we need to do this transition
214           // See HBASE-7698
215           coordination.tryTransitionFromOpeningToFailedOpen(regionInfo, ord);
216         }
217       }
218     } else if (!useZKForAssignment) {
219       rsServices.reportRegionStateTransition(TransitionCode.FAILED_OPEN, regionInfo);
220     } else {
221       // If still transition to OPENING is not done, we need to transition znode
222       // to FAILED_OPEN
223       coordination.tryTransitionFromOfflineToFailedOpen(this.rsServices, regionInfo, ord);
224     }
225   }
226 
227   /**
228    * Update ZK or META.  This can take a while if for example the
229    * hbase:meta is not available -- if server hosting hbase:meta crashed and we are
230    * waiting on it to come back -- so run in a thread and keep updating znode
231    * state meantime so master doesn't timeout our region-in-transition.
232    * Caller must cleanup region if this fails.
233    */
234   boolean updateMeta(final HRegion r) {
235     if (this.server.isStopped() || this.rsServices.isStopping()) {
236       return false;
237     }
238     // Object we do wait/notify on.  Make it boolean.  If set, we're done.
239     // Else, wait.
240     final AtomicBoolean signaller = new AtomicBoolean(false);
241     PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r,
242       this.server, this.rsServices, signaller);
243     t.start();
244     // Post open deploy task:
245     //   meta => update meta location in ZK
246     //   other region => update meta
247     // It could fail if ZK/meta is not available and
248     // the update runs out of retries.
249     long now = System.currentTimeMillis();
250     long lastUpdate = now;
251     boolean tickleOpening = true;
252     while (!signaller.get() && t.isAlive() && !this.server.isStopped() &&
253         !this.rsServices.isStopping() && isRegionStillOpening()) {
254       long elapsed = now - lastUpdate;
255       if (elapsed > 120000) { // 2 minutes, no need to tickleOpening too often
256         // Only tickle OPENING if postOpenDeployTasks is taking some time.
257         lastUpdate = now;
258         if (useZKForAssignment) {
259           tickleOpening = coordination.tickleOpening(
260             ord, regionInfo, rsServices, "post_open_deploy");
261         }
262       }
263       synchronized (signaller) {
264         try {
265           // Wait for 10 seconds, so that server shutdown
266           // won't take too long if this thread happens to run.
267           if (!signaller.get()) signaller.wait(10000);
268         } catch (InterruptedException e) {
269           // Go to the loop check.
270         }
271       }
272       now = System.currentTimeMillis();
273     }
274     // Is thread still alive?  We may have left above loop because server is
275     // stopping or we timed out the edit.  Is so, interrupt it.
276     if (t.isAlive()) {
277       if (!signaller.get()) {
278         // Thread still running; interrupt
279         LOG.debug("Interrupting thread " + t);
280         t.interrupt();
281       }
282       try {
283         t.join();
284       } catch (InterruptedException ie) {
285         LOG.warn("Interrupted joining " +
286           r.getRegionInfo().getRegionNameAsString(), ie);
287         Thread.currentThread().interrupt();
288       }
289     }
290 
291     // Was there an exception opening the region?  This should trigger on
292     // InterruptedException too.  If so, we failed.  Even if tickle opening fails
293     // then it is a failure.
294     return ((!Thread.interrupted() && t.getException() == null) && tickleOpening);
295   }
296 
297   /**
298    * Thread to run region post open tasks. Call {@link #getException()} after
299    * the thread finishes to check for exceptions running
300    * {@link RegionServerServices#postOpenDeployTasks(
301    * HRegion, org.apache.hadoop.hbase.catalog.CatalogTracker)}
302    * .
303    */
304   static class PostOpenDeployTasksThread extends Thread {
305     private Throwable exception = null;
306     private final Server server;
307     private final RegionServerServices services;
308     private final HRegion region;
309     private final AtomicBoolean signaller;
310 
311     PostOpenDeployTasksThread(final HRegion region, final Server server,
312         final RegionServerServices services, final AtomicBoolean signaller) {
313       super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName());
314       this.setDaemon(true);
315       this.server = server;
316       this.services = services;
317       this.region = region;
318       this.signaller = signaller;
319     }
320 
321     public void run() {
322       try {
323         this.services.postOpenDeployTasks(this.region,
324           this.server.getCatalogTracker());
325       } catch (IOException e) {
326         server.abort("Exception running postOpenDeployTasks; region=" +
327             this.region.getRegionInfo().getEncodedName(), e);
328       } catch (Throwable e) {
329         LOG.warn("Exception running postOpenDeployTasks; region=" +
330           this.region.getRegionInfo().getEncodedName(), e);
331         this.exception = e;
332       }
333       // We're done.  Set flag then wake up anyone waiting on thread to complete.
334       this.signaller.set(true);
335       synchronized (this.signaller) {
336         this.signaller.notify();
337       }
338     }
339 
340     /**
341      * @return Null or the run exception; call this method after thread is done.
342      */
343     Throwable getException() {
344       return this.exception;
345     }
346   }
347 
348   /**
349    * @return Instance of HRegion if successful open else null.
350    */
351   HRegion openRegion() {
352     HRegion region = null;
353     try {
354       // Instantiate the region.  This also periodically tickles OPENING
355       // state so master doesn't timeout this region in transition.
356       region = HRegion.openHRegion(this.regionInfo, this.htd,
357         this.rsServices.getWAL(this.regionInfo),
358         this.server.getConfiguration(),
359         this.rsServices,
360         new CancelableProgressable() {
361           public boolean progress() {
362             if (useZKForAssignment) {
363               // if tickle failed, we need to cancel opening region.
364               return coordination.tickleOpening(ord, regionInfo,
365                 rsServices, "open_region_progress");
366             }
367             if (!isRegionStillOpening()) {
368               LOG.warn("Open region aborted since it isn't opening any more");
369               return false;
370             }
371             return true;
372           }
373         });
374     } catch (Throwable t) {
375       // We failed open. Our caller will see the 'null' return value
376       // and transition the node back to FAILED_OPEN. If that fails,
377       // we rely on the Timeout Monitor in the master to reassign.
378       LOG.error(
379           "Failed open of region=" + this.regionInfo.getRegionNameAsString()
380               + ", starting to roll back the global memstore size.", t);
381       // Decrease the global memstore size.
382       if (this.rsServices != null) {
383         RegionServerAccounting rsAccounting =
384           this.rsServices.getRegionServerAccounting();
385         if (rsAccounting != null) {
386           rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
387         }
388       }
389     }
390     return region;
391   }
392 
393   void cleanupFailedOpen(final HRegion region) throws IOException {
394     if (region != null) {
395       this.rsServices.removeFromOnlineRegions(region, null);
396       region.close();
397     }
398   }
399 
400   private boolean isRegionStillOpening() {
401     byte[] encodedName = regionInfo.getEncodedNameAsBytes();
402     Boolean action = rsServices.getRegionsInTransitionInRS().get(encodedName);
403     return Boolean.TRUE.equals(action); // true means opening for RIT
404   }
405 }