View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import java.io.IOException;
21  import java.io.InterruptedIOException;
22  import java.net.ConnectException;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
31  import org.apache.hadoop.hbase.ServerName;
32  import org.apache.hadoop.hbase.client.Delete;
33  import org.apache.hadoop.hbase.client.HTable;
34  import org.apache.hadoop.hbase.client.Mutation;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
39  import org.apache.hadoop.hbase.util.PairOfSameType;
40  import org.apache.hadoop.hbase.util.Threads;
41  import org.apache.hadoop.hbase.util.Writables;
42  
43  /**
44   * Writes region and assignment information to <code>.META.</code>.
45   * TODO: Put MetaReader and MetaEditor together; doesn't make sense having
46   * them distinct.
47   */
48  public class MetaEditor {
49    // TODO: Strip CatalogTracker from this class.  Its all over and in the end
50    // its only used to get its Configuration so we can get associated
51    // Connection.
52    private static final Log LOG = LogFactory.getLog(MetaEditor.class);
53  
54    private static Put makePutFromRegionInfo(HRegionInfo regionInfo)
55    throws IOException {
56      Put put = new Put(regionInfo.getRegionName());
57      put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
58          Writables.getBytes(regionInfo));
59      return put;
60    }
61  
62    /**
63     * Put the passed <code>p</code> to the <code>.META.</code> table.
64     * @param ct CatalogTracker on whose back we will ride the edit.
65     * @param p Put to add to .META.
66     * @throws IOException
67     */
68    static void putToMetaTable(final CatalogTracker ct, final Put p)
69    throws IOException {
70      put(MetaReader.getMetaHTable(ct), p);
71    }
72  
73    /**
74     * Put the passed <code>p</code> to the <code>.META.</code> table.
75     * @param ct CatalogTracker on whose back we will ride the edit.
76     * @param p Put to add to .META.
77     * @throws IOException
78     */
79    static void putToRootTable(final CatalogTracker ct, final Put p)
80    throws IOException {
81      put(MetaReader.getRootHTable(ct), p);
82    }
83  
84    /**
85     * Put the passed <code>p</code> to a catalog table.
86     * @param ct CatalogTracker on whose back we will ride the edit.
87     * @param p Put to add
88     * @throws IOException
89     */
90    static void putToCatalogTable(final CatalogTracker ct, final Put p)
91    throws IOException {
92      HTable t = MetaReader.getCatalogHTable(ct, p.getRow());
93      put(t, p);
94    }
95  
96    /**
97     * @param t Table to use (will be closed when done).
98     * @param p
99     * @throws IOException
100    */
101   private static void put(final HTable t, final Put p) throws IOException {
102     try {
103       t.put(p);
104     } finally {
105       t.close();
106     }
107   }
108 
109   /**
110    * Put the passed <code>ps</code> to the <code>.META.</code> table.
111    * @param ct CatalogTracker on whose back we will ride the edit.
112    * @param ps Put to add to .META.
113    * @throws IOException
114    */
115   static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
116   throws IOException {
117     HTable t = MetaReader.getMetaHTable(ct);
118     try {
119       t.put(ps);
120     } finally {
121       t.close();
122     }
123   }
124 
125   /**
126    * Delete the passed <code>d</code> from the <code>.META.</code> table.
127    * @param ct CatalogTracker on whose back we will ride the edit.
128    * @param d Delete to add to .META.
129    * @throws IOException
130    */
131   static void deleteFromMetaTable(final CatalogTracker ct, final Delete d)
132       throws IOException {
133     List<Delete> dels = new ArrayList<Delete>(1);
134     dels.add(d);
135     deleteFromMetaTable(ct, dels);
136   }
137 
138   /**
139    * Delete the passed <code>deletes</code> from the <code>.META.</code> table.
140    * @param ct CatalogTracker on whose back we will ride the edit.
141    * @param deletes Deletes to add to .META.  This list should support #remove.
142    * @throws IOException
143    */
144   public static void deleteFromMetaTable(final CatalogTracker ct, final List<Delete> deletes)
145       throws IOException {
146     HTable t = MetaReader.getMetaHTable(ct);
147     try {
148       t.delete(deletes);
149     } finally {
150       t.close();
151     }
152   }
153 
154   /**
155    * Execute the passed <code>mutations</code> against <code>.META.</code> table.
156    * @param ct CatalogTracker on whose back we will ride the edit.
157    * @param mutations Puts and Deletes to execute on .META.
158    * @throws IOException
159    */
160   static void mutateMetaTable(final CatalogTracker ct, final List<Mutation> mutations)
161       throws IOException {
162     HTable t = MetaReader.getMetaHTable(ct);
163     try {
164       t.batch(mutations);
165     } catch (InterruptedException e) {
166       InterruptedIOException ie = new InterruptedIOException(e.getMessage());
167       ie.initCause(e);
168       throw ie;
169     } finally {
170       t.close();
171     }
172   }
173 
174   /**
175    * Adds a META row for the specified new region.
176    * @param regionInfo region information
177    * @throws IOException if problem connecting or updating meta
178    */
179   public static void addRegionToMeta(CatalogTracker catalogTracker,
180       HRegionInfo regionInfo)
181   throws IOException {
182     putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo));
183     LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to META");
184   }
185 
186   /**
187    * Adds a META row for each of the specified new regions.
188    * @param catalogTracker CatalogTracker
189    * @param regionInfos region information list
190    * @throws IOException if problem connecting or updating meta
191    */
192   public static void addRegionsToMeta(CatalogTracker catalogTracker,
193       List<HRegionInfo> regionInfos)
194   throws IOException {
195     List<Put> puts = new ArrayList<Put>();
196     for (HRegionInfo regionInfo : regionInfos) {
197       puts.add(makePutFromRegionInfo(regionInfo));
198     }
199     putsToMetaTable(catalogTracker, puts);
200     LOG.info("Added " + puts.size() + " regions in META");
201   }
202 
203   /**
204    * Offline parent in meta.
205    * Used when splitting.
206    * @param catalogTracker
207    * @param parent
208    * @param a Split daughter region A
209    * @param b Split daughter region B
210    * @throws NotAllMetaRegionsOnlineException
211    * @throws IOException
212    */
213   public static void offlineParentInMeta(CatalogTracker catalogTracker,
214       HRegionInfo parent, final HRegionInfo a, final HRegionInfo b)
215   throws NotAllMetaRegionsOnlineException, IOException {
216     HRegionInfo copyOfParent = new HRegionInfo(parent);
217     copyOfParent.setOffline(true);
218     copyOfParent.setSplit(true);
219     Put put = new Put(copyOfParent.getRegionName());
220     addRegionInfo(put, copyOfParent);
221     put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
222       Writables.getBytes(a));
223     put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
224       Writables.getBytes(b));
225     putToMetaTable(catalogTracker, put);
226     LOG.info("Offlined parent region " + parent.getRegionNameAsString() +
227       " in META");
228   }
229 
230   public static void addDaughter(final CatalogTracker catalogTracker,
231       final HRegionInfo regionInfo, final ServerName sn)
232   throws NotAllMetaRegionsOnlineException, IOException {
233     Put put = new Put(regionInfo.getRegionName());
234     addRegionInfo(put, regionInfo);
235     if (sn != null) addLocation(put, sn);
236     putToMetaTable(catalogTracker, put);
237     LOG.info("Added daughter " + regionInfo.getRegionNameAsString() +
238       (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
239   }
240 
241   /**
242    * Updates the location of the specified META region in ROOT to be the
243    * specified server hostname and startcode.
244    * <p>
245    * Uses passed catalog tracker to get a connection to the server hosting
246    * ROOT and makes edits to that region.
247    *
248    * @param catalogTracker catalog tracker
249    * @param regionInfo region to update location of
250    * @param sn Server name
251    * @throws IOException
252    * @throws ConnectException Usually because the regionserver carrying .META.
253    * is down.
254    * @throws NullPointerException Because no -ROOT- server connection
255    */
256   public static void updateMetaLocation(CatalogTracker catalogTracker,
257       HRegionInfo regionInfo, ServerName sn)
258   throws IOException, ConnectException {
259     updateLocation(catalogTracker, regionInfo, sn);
260   }
261 
262   /**
263    * Updates the location of the specified region in META to be the specified
264    * server hostname and startcode.
265    * <p>
266    * Uses passed catalog tracker to get a connection to the server hosting
267    * META and makes edits to that region.
268    *
269    * @param catalogTracker catalog tracker
270    * @param regionInfo region to update location of
271    * @param sn Server name
272    * @throws IOException
273    */
274   public static void updateRegionLocation(CatalogTracker catalogTracker,
275       HRegionInfo regionInfo, ServerName sn)
276   throws IOException {
277     updateLocation(catalogTracker, regionInfo, sn);
278   }
279 
280   /**
281    * Updates the location of the specified region to be the specified server.
282    * <p>
283    * Connects to the specified server which should be hosting the specified
284    * catalog region name to perform the edit.
285    *
286    * @param catalogTracker
287    * @param regionInfo region to update location of
288    * @param sn Server name
289    * @throws IOException In particular could throw {@link java.net.ConnectException}
290    * if the server is down on other end.
291    */
292   private static void updateLocation(final CatalogTracker catalogTracker,
293       HRegionInfo regionInfo, ServerName sn)
294   throws IOException {
295     Put put = new Put(regionInfo.getRegionName());
296     addLocation(put, sn);
297     putToCatalogTable(catalogTracker, put);
298     LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
299       " with server=" + sn);
300   }
301 
302   /**
303    * Deletes the specified region from META.
304    * @param catalogTracker
305    * @param regionInfo region to be deleted from META
306    * @throws IOException
307    */
308   public static void deleteRegion(CatalogTracker catalogTracker,
309       HRegionInfo regionInfo)
310   throws IOException {
311     Delete delete = new Delete(regionInfo.getRegionName());
312     deleteFromMetaTable(catalogTracker, delete);
313     LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from META");
314   }
315 
316   /**
317    * Deletes the specified regions from META.
318    * @param catalogTracker
319    * @param regionsInfo list of regions to be deleted from META
320    * @throws IOException
321    */
322   public static void deleteRegions(CatalogTracker catalogTracker,
323       List<HRegionInfo> regionsInfo) throws IOException {
324     List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
325     for (HRegionInfo hri: regionsInfo) {
326       deletes.add(new Delete(hri.getRegionName()));
327     }
328     deleteFromMetaTable(catalogTracker, deletes);
329     LOG.info("Deleted from META, regions: " + regionsInfo);
330   }
331 
332   /**
333    * Adds and Removes the specified regions from .META.
334    * @param catalogTracker
335    * @param regionsToRemove list of regions to be deleted from META
336    * @param regionsToAdd list of regions to be added to META
337    * @throws IOException
338    */
339   public static void mutateRegions(CatalogTracker catalogTracker,
340       final List<HRegionInfo> regionsToRemove, final List<HRegionInfo> regionsToAdd)
341       throws IOException {
342     List<Mutation> mutation = new ArrayList<Mutation>();
343     if (regionsToRemove != null) {
344       for (HRegionInfo hri: regionsToRemove) {
345         mutation.add(new Delete(hri.getRegionName()));
346       }
347     }
348     if (regionsToAdd != null) {
349       for (HRegionInfo hri: regionsToAdd) {
350         mutation.add(makePutFromRegionInfo(hri));
351       }
352     }
353     mutateMetaTable(catalogTracker, mutation);
354     if (regionsToRemove != null && regionsToRemove.size() > 0) {
355       LOG.debug("Deleted from META, regions: " + regionsToRemove);
356     }
357     if (regionsToAdd != null && regionsToAdd.size() > 0) {
358       LOG.debug("Add to META, regions: " + regionsToAdd);
359     }
360   }
361 
362   /**
363    * Overwrites the specified regions from hbase:meta
364    * @param catalogTracker
365    * @param regionInfos list of regions to be added to META
366    * @throws IOException
367    */
368   public static void overwriteRegions(CatalogTracker catalogTracker,
369       List<HRegionInfo> regionInfos) throws IOException {
370     deleteRegions(catalogTracker, regionInfos);
371     // Why sleep? This is the easiest way to ensure that the previous deletes does not
372     // eclipse the following puts, that might happen in the same ts from the server.
373     // See HBASE-9906, and HBASE-9879. Once either HBASE-9879, HBASE-8770 is fixed,
374     // or HBASE-9905 is fixed and meta uses seqIds, we do not need the sleep.
375     Threads.sleep(20);
376     addRegionsToMeta(catalogTracker, regionInfos);
377     LOG.info("Overwritten " + regionInfos);
378   }
379 
380   public static HRegionInfo getHRegionInfo(
381       Result data) throws IOException {
382     byte [] bytes =
383       data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
384     if (bytes == null) return null;
385     HRegionInfo info = Writables.getHRegionInfo(bytes);
386     LOG.info("Current INFO from scan results = " + info);
387     return info;
388   }
389 
390   /**
391    * Returns the daughter regions by reading from the corresponding columns of the .META. table
392    * Result. If the region is not a split parent region, it returns PairOfSameType(null, null).
393    */
394   public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
395     HRegionInfo splitA = Writables.getHRegionInfoOrNull(
396         data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER));
397     HRegionInfo splitB = Writables.getHRegionInfoOrNull(
398         data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER));
399     return new PairOfSameType<HRegionInfo>(splitA, splitB);
400   }
401 
402   private static Put addRegionInfo(final Put p, final HRegionInfo hri)
403   throws IOException {
404     p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
405         Writables.getBytes(hri));
406     return p;
407   }
408 
409   private static Put addLocation(final Put p, final ServerName sn) {
410     // using regionserver's local time as the timestamp of Put.
411     // See: HBASE-11536
412     long now = EnvironmentEdgeManager.currentTimeMillis();
413     p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, now,
414       Bytes.toBytes(sn.getHostAndPort()));
415     p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, now,
416       Bytes.toBytes(sn.getStartcode()));
417     return p;
418   }
419 }