View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master.handler;
21  
22  import java.io.IOException;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.Server;
28  import org.apache.hadoop.hbase.ServerName;
29  import org.apache.hadoop.hbase.master.DeadServer;
30  import org.apache.hadoop.hbase.master.MasterServices;
31  import org.apache.zookeeper.KeeperException;
32  
33  /**
34   * Shutdown handler for the server hosting <code>-ROOT-</code>,
35   * <code>.META.</code>, or both.
36   */
37  public class MetaServerShutdownHandler extends ServerShutdownHandler {
38    private final boolean carryingRoot;
39    private final boolean carryingMeta;
40    private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class);
41    public MetaServerShutdownHandler(final Server server,
42        final MasterServices services,
43        final DeadServer deadServers, final ServerName serverName,
44        final boolean carryingRoot, final boolean carryingMeta) {
45      super(server, services, deadServers, serverName,
46        EventType.M_META_SERVER_SHUTDOWN, true);
47      this.carryingRoot = carryingRoot;
48      this.carryingMeta = carryingMeta;
49    }
50  
51    @Override
52    public void process() throws IOException {
53  
54      boolean gotException = true;
55      try {
56        try {
57          if (this.shouldSplitHlog) {
58            if (this.services.shouldSplitMetaSeparately()) {
59              LOG.info("Splitting META logs for " + serverName);
60              this.services.getMasterFileSystem().splitMetaLog(serverName);
61            } else {
62              LOG.info("Splitting all logs for " + serverName);
63              this.services.getMasterFileSystem().splitAllLogs(serverName);
64            }
65          }
66        } catch (IOException ioe) {
67          this.services.getExecutorService().submit(this);
68          this.deadServers.add(serverName);
69          throw new IOException("failed log splitting for " +
70              serverName + ", will retry", ioe);
71        }
72    
73        // Assign root and meta if we were carrying them.
74        if (isCarryingRoot()) { // -ROOT-
75          // Check again: region may be assigned to other where because of RIT
76          // timeout
77          if (this.services.getAssignmentManager().isCarryingRoot(serverName)) {
78            LOG.info("Server " + serverName
79                + " was carrying ROOT. Trying to assign.");
80            this.services.getAssignmentManager().regionOffline(
81                HRegionInfo.ROOT_REGIONINFO);
82            verifyAndAssignRootWithRetries();
83          } else {
84            LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
85          }
86        }
87        
88        if(!this.services.isServerShutdownHandlerEnabled()) {
89          // resubmit in case we're in master initialization and SSH hasn't been enabled yet.
90          this.services.getExecutorService().submit(this);
91          this.deadServers.add(serverName);
92          return;
93        }
94    
95        // Carrying meta?
96        if (isCarryingMeta()) {
97          // Check again: region may be assigned to other where because of RIT
98          // timeout
99          if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
100           LOG.info("Server " + serverName
101               + " was carrying META. Trying to assign.");
102           this.services.getAssignmentManager().regionOffline(
103               HRegionInfo.FIRST_META_REGIONINFO);
104           this.services.getAssignmentManager().assignMeta();
105         } else {
106           LOG.info("META has been assigned to otherwhere, skip assigning.");
107         }
108       }
109       
110       gotException = false;
111     } finally {
112       if (gotException){
113         // If we had an exception, this.deadServers.finish will be skipped in super.process()
114         this.deadServers.finish(serverName);
115       }
116     }
117 
118     super.process();
119   }
120   /**
121    * Before assign the ROOT region, ensure it haven't
122    *  been assigned by other place
123    * <p>
124    * Under some scenarios, the ROOT region can be opened twice, so it seemed online
125    * in two regionserver at the same time.
126    * If the ROOT region has been assigned, so the operation can be canceled.
127    * @throws InterruptedException
128    * @throws IOException
129    * @throws KeeperException
130    */
131   private void verifyAndAssignRoot()
132   throws InterruptedException, IOException, KeeperException {
133     long timeout = this.server.getConfiguration().
134       getLong("hbase.catalog.verification.timeout", 1000);
135     if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
136       this.services.getAssignmentManager().assignRoot();
137     } else if (serverName.equals(server.getCatalogTracker().getRootLocation())) {
138       throw new IOException("-ROOT- is onlined on the dead server "
139           + serverName);
140     } else {
141       LOG.info("Skip assigning -ROOT-, because it is online on the "
142           + server.getCatalogTracker().getRootLocation());
143     }
144   }
145 
146   /**
147    * Failed many times, shutdown processing
148    * @throws IOException
149    */
150   private void verifyAndAssignRootWithRetries() throws IOException {
151     int iTimes = this.server.getConfiguration().getInt(
152         "hbase.catalog.verification.retries", 10);
153 
154     long waitTime = this.server.getConfiguration().getLong(
155         "hbase.catalog.verification.timeout", 1000);
156 
157     int iFlag = 0;
158     while (true) {
159       try {
160         verifyAndAssignRoot();
161         break;
162       } catch (KeeperException e) {
163         this.server.abort("In server shutdown processing, assigning root", e);
164         throw new IOException("Aborting", e);
165       } catch (Exception e) {
166         if (iFlag >= iTimes) {
167           this.server.abort("verifyAndAssignRoot failed after" + iTimes
168               + " times retries, aborting", e);
169           throw new IOException("Aborting", e);
170         }
171         try {
172           Thread.sleep(waitTime);
173         } catch (InterruptedException e1) {
174           LOG.warn("Interrupted when is the thread sleep", e1);
175           Thread.currentThread().interrupt();
176           throw new IOException("Interrupted", e1);
177         }
178         iFlag++;
179       }
180     }
181   }
182 
183   boolean isCarryingRoot() {
184     return this.carryingRoot;
185   }
186 
187   boolean isCarryingMeta() {
188     return this.carryingMeta;
189   }
190   
191   @Override
192   public String toString() {
193     String name = "UnknownServerName";
194     if(server != null && server.getServerName() != null) {
195       name = server.getServerName().toString();
196     }
197     return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
198   }
199 }