View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.util.Comparator;
24  import java.util.HashSet;
25  import java.util.Map;
26  import java.util.TreeMap;
27  import java.util.concurrent.atomic.AtomicBoolean;
28  import java.util.concurrent.atomic.AtomicInteger;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.MetaTableAccessor;
39  import org.apache.hadoop.hbase.ScheduledChore;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.backup.HFileArchiver;
42  import org.apache.hadoop.hbase.classification.InterfaceAudience;
43  import org.apache.hadoop.hbase.client.Connection;
44  import org.apache.hadoop.hbase.client.Result;
45  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.FSUtils;
48  import org.apache.hadoop.hbase.util.Pair;
49  import org.apache.hadoop.hbase.util.PairOfSameType;
50  import org.apache.hadoop.hbase.util.Threads;
51  import org.apache.hadoop.hbase.util.Triple;
52
53  /**
54   * A janitor for the catalog tables.  Scans the <code>hbase:meta</code> catalog
55   * table on a period looking for unused regions to garbage collect.
56   */
57  @InterfaceAudience.Private
58  public class CatalogJanitor extends ScheduledChore {
59    private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
60
61    private final AtomicBoolean alreadyRunning = new AtomicBoolean(false);
62    private final AtomicBoolean enabled = new AtomicBoolean(true);
63    private final MasterServices services;
64    private final Connection connection;
65
66    CatalogJanitor(final MasterServices services) {
67      super("CatalogJanitor-" + services.getServerName().toShortString(), services,
68        services.getConfiguration().getInt("hbase.catalogjanitor.interval", 300000));
69      this.services = services;
70      this.connection = services.getConnection();
71    }
72
73    @Override
74    protected boolean initialChore() {
75      try {
76        if (this.enabled.get()) scan();
77      } catch (IOException e) {
78        LOG.warn("Failed initial scan of catalog table", e);
79        return false;
80      }
81      return true;
82    }
83
84    /**
85     * @param enabled
86     */
87    public boolean setEnabled(final boolean enabled) {
88      boolean alreadyEnabled = this.enabled.getAndSet(enabled);
89      // If disabling is requested on an already enabled chore, we could have an active
90      // scan still going on, callers might not be aware of that and do further action thinkng
91      // that no action would be from this chore.  In this case, the right action is to wait for
92      // the active scan to complete before exiting this function.
93      if (!enabled && alreadyEnabled) {
94        while (alreadyRunning.get()) {
95          Threads.sleepWithoutInterrupt(100);
96        }
97      }
98      return alreadyEnabled;
99    }
100
101   boolean getEnabled() {
102     return this.enabled.get();
103   }
104
105   @Override
106   protected void chore() {
107     try {
108       AssignmentManager am = this.services.getAssignmentManager();
109       if (this.enabled.get()
110           && !this.services.isInMaintenanceMode()
111           && am != null
112           && am.isFailoverCleanupDone()
113           && am.getRegionStates().getRegionsInTransition().size() == 0) {
114         scan();
115       } else {
116         LOG.warn("CatalogJanitor disabled! Not running scan.");
117       }
118     } catch (IOException e) {
119       LOG.warn("Failed scan of catalog table", e);
120     }
121   }
122
123   /**
124    * Scans hbase:meta and returns a number of scanned rows, and a map of merged
125    * regions, and an ordered map of split parents.
126    * @return triple of scanned rows, map of merged regions and map of split
127    *         parent regioninfos
128    * @throws IOException
129    */
130   Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>
131     getMergedRegionsAndSplitParents() throws IOException {
132     return getMergedRegionsAndSplitParents(null);
133   }
134
135   /**
136    * Scans hbase:meta and returns a number of scanned rows, and a map of merged
137    * regions, and an ordered map of split parents. if the given table name is
138    * null, return merged regions and split parents of all tables, else only the
139    * specified table
140    * @param tableName null represents all tables
141    * @return triple of scanned rows, and map of merged regions, and map of split
142    *         parent regioninfos
143    * @throws IOException
144    */
145   Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>
146     getMergedRegionsAndSplitParents(final TableName tableName) throws IOException {
147     final boolean isTableSpecified = (tableName != null);
148     // TODO: Only works with single hbase:meta region currently.  Fix.
149     final AtomicInteger count = new AtomicInteger(0);
150     // Keep Map of found split parents.  There are candidates for cleanup.
151     // Use a comparator that has split parents come before its daughters.
152     final Map<HRegionInfo, Result> splitParents =
153       new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
154     final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
155     // This visitor collects split parents and counts rows in the hbase:meta table
156
157     MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
158       @Override
159       public boolean visit(Result r) throws IOException {
160         if (r == null || r.isEmpty()) return true;
161         count.incrementAndGet();
162         HRegionInfo info = MetaTableAccessor.getHRegionInfo(r);
163         if (info == null) return true; // Keep scanning
164         if (isTableSpecified
165             && info.getTable().compareTo(tableName) > 0) {
166           // Another table, stop scanning
167           return false;
168         }
169         if (info.isSplitParent()) splitParents.put(info, r);
170         if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {
171           mergedRegions.put(info, r);
172         }
173         // Returning true means "keep scanning"
174         return true;
175       }
176     };
177
178     // Run full scan of hbase:meta catalog table passing in our custom visitor with
179     // the start row
180     MetaTableAccessor.scanMetaForTableRegions(this.connection, visitor, tableName);
181
182     return new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
183         count.get(), mergedRegions, splitParents);
184   }
185
186   /**
187    * If merged region no longer holds reference to the merge regions, archive
188    * merge region on hdfs and perform deleting references in hbase:meta
189    * @param mergedRegion
190    * @param regionA
191    * @param regionB
192    * @return true if we delete references in merged region on hbase:meta and archive
193    *         the files on the file system
194    * @throws IOException
195    */
196   boolean cleanMergeRegion(final HRegionInfo mergedRegion,
197       final HRegionInfo regionA, final HRegionInfo regionB) throws IOException {
198     FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
199     Path rootdir = this.services.getMasterFileSystem().getRootDir();
200     Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
201     HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
202     HRegionFileSystem regionFs = null;
203     try {
204       regionFs = HRegionFileSystem.openRegionFromFileSystem(
205           this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
206     } catch (IOException e) {
207       LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
208     }
209     if (regionFs == null || !regionFs.hasReferences(htd)) {
210       LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
211           + regionB.getRegionNameAsString()
212           + " from fs because merged region no longer holds references");
213       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
214       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
215       MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
216       services.getServerManager().removeRegion(regionA);
217       services.getServerManager().removeRegion(regionB);
218       return true;
219     }
220     return false;
221   }
222
223   /**
224    * Run janitorial scan of catalog <code>hbase:meta</code> table looking for
225    * garbage to collect.
226    * @return number of cleaned regions
227    * @throws IOException
228    */
229   int scan() throws IOException {
230     try {
231       if (!alreadyRunning.compareAndSet(false, true)) {
232         LOG.debug("CatalogJanitor already running");
233         return 0;
234       }
235       Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> scanTriple =
236         getMergedRegionsAndSplitParents();
237       int count = scanTriple.getFirst();
238       /**
239        * clean merge regions first
240        */
241       int mergeCleaned = 0;
242       Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond();
243       for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) {
244         if (this.services.isInMaintenanceMode()) {
245           // Stop cleaning if the master is in maintenance mode
246           break;
247         }
248
249         PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(e.getValue());
250         HRegionInfo regionA = p.getFirst();
251         HRegionInfo regionB = p.getSecond();
252         if (regionA == null || regionB == null) {
253           LOG.warn("Unexpected references regionA="
254               + (regionA == null ? "null" : regionA.getRegionNameAsString())
255               + ",regionB="
256               + (regionB == null ? "null" : regionB.getRegionNameAsString())
257               + " in merged region " + e.getKey().getRegionNameAsString());
258         } else {
259           if (cleanMergeRegion(e.getKey(), regionA, regionB)) {
260             mergeCleaned++;
261           }
262         }
263       }
264       /**
265        * clean split parents
266        */
267       Map<HRegionInfo, Result> splitParents = scanTriple.getThird();
268
269       // Now work on our list of found parents. See if any we can clean up.
270       int splitCleaned = 0;
271       // regions whose parents are still around
272       HashSet<String> parentNotCleaned = new HashSet<String>();
273       for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
274         if (this.services.isInMaintenanceMode()) {
275           // Stop cleaning if the master is in maintenance mode
276           break;
277         }
278
279         if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
280             cleanParent(e.getKey(), e.getValue())) {
281           splitCleaned++;
282         } else {
283           // We could not clean the parent, so it's daughters should not be
284           // cleaned either (HBASE-6160)
285           PairOfSameType<HRegionInfo> daughters =
286               MetaTableAccessor.getDaughterRegions(e.getValue());
287           parentNotCleaned.add(daughters.getFirst().getEncodedName());
288           parentNotCleaned.add(daughters.getSecond().getEncodedName());
289         }
290       }
291       if ((mergeCleaned + splitCleaned) != 0) {
292         LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
293             + " unreferenced merged region(s) and " + splitCleaned
294             + " unreferenced parent region(s)");
295       } else if (LOG.isTraceEnabled()) {
296         LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
297             + " unreferenced merged region(s) and " + splitCleaned
298             + " unreferenced parent region(s)");
299       }
300       return mergeCleaned + splitCleaned;
301     } finally {
302       alreadyRunning.set(false);
303     }
304   }
305
306   /**
307    * Compare HRegionInfos in a way that has split parents sort BEFORE their
308    * daughters.
309    */
310   static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
311     Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
312     @Override
313     public int compare(HRegionInfo left, HRegionInfo right) {
314       // This comparator differs from the one HRegionInfo in that it sorts
315       // parent before daughters.
316       if (left == null) return -1;
317       if (right == null) return 1;
318       // Same table name.
319       int result = left.getTable().compareTo(right.getTable());
320       if (result != 0) return result;
321       // Compare start keys.
322       result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
323       if (result != 0) return result;
324       // Compare end keys, but flip the operands so parent comes first
325       result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
326
327       return result;
328     }
329   }
330
331   /**
332    * If daughters no longer hold reference to the parents, delete the parent.
333    * @param parent HRegionInfo of split offlined parent
334    * @param rowContent Content of <code>parent</code> row in
335    * <code>metaRegionName</code>
336    * @return True if we removed <code>parent</code> from meta table and from
337    * the filesystem.
338    * @throws IOException
339    */
340   boolean cleanParent(final HRegionInfo parent, Result rowContent)
341   throws IOException {
342     boolean result = false;
343     // Check whether it is a merged region and not clean reference
344     // No necessary to check MERGEB_QUALIFIER because these two qualifiers will
345     // be inserted/deleted together
346     if (rowContent.getValue(HConstants.CATALOG_FAMILY,
347         HConstants.MERGEA_QUALIFIER) != null) {
348       // wait cleaning merge region first
349       return result;
350     }
351     // Run checks on each daughter split.
352     PairOfSameType<HRegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
353     Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
354     Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
355     if (hasNoReferences(a) && hasNoReferences(b)) {
356       LOG.debug("Deleting region " + parent.getRegionNameAsString() +
357         " because daughter splits no longer hold references");
358       FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
359       if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
360       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
361       MetaTableAccessor.deleteRegion(this.connection, parent);
362       services.getServerManager().removeRegion(parent);
363       result = true;
364     }
365     return result;
366   }
367
368   /**
369    * @param p A pair where the first boolean says whether or not the daughter
370    * region directory exists in the filesystem and then the second boolean says
371    * whether the daughter has references to the parent.
372    * @return True the passed <code>p</code> signifies no references.
373    */
374   private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
375     return !p.getFirst() || !p.getSecond();
376   }
377
378   /**
379    * Checks if a daughter region -- either splitA or splitB -- still holds
380    * references to parent.
381    * @param parent Parent region
382    * @param daughter Daughter region
383    * @return A pair where the first boolean says whether or not the daughter
384    * region directory exists in the filesystem and then the second boolean says
385    * whether the daughter has references to the parent.
386    * @throws IOException
387    */
388   Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter)
389   throws IOException {
390     if (daughter == null)  {
391       return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
392     }
393
394     FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
395     Path rootdir = this.services.getMasterFileSystem().getRootDir();
396     Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
397
398     Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
399
400     HRegionFileSystem regionFs = null;
401
402     try {
403       if (!FSUtils.isExists(fs, daughterRegionDir)) {
404         return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
405       }
406     } catch (IOException ioe) {
407       LOG.error("Error trying to determine if daughter region exists, " +
408                "assuming exists and has references", ioe);
409       return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
410     }
411
412     boolean references = false;
413     HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
414     try {
415       regionFs = HRegionFileSystem.openRegionFromFileSystem(
416           this.services.getConfiguration(), fs, tabledir, daughter, true);
417
418       for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
419         if ((references = regionFs.hasReferences(family.getNameAsString()))) {
420           break;
421         }
422       }
423     } catch (IOException e) {
424       LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName()
425           + ", to: " + parent.getEncodedName() + " assuming has references", e);
426       return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
427     }
428     return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references));
429   }
430
431   private HTableDescriptor getTableDescriptor(final TableName tableName)
432       throws FileNotFoundException, IOException {
433     return this.services.getTableDescriptors().get(tableName);
434   }
435
436   /**
437    * Checks if the specified region has merge qualifiers, if so, try to clean
438    * them
439    * @param region
440    * @return true if the specified region doesn't have merge qualifier now
441    * @throws IOException
442    */
443   public boolean cleanMergeQualifier(final HRegionInfo region)
444       throws IOException {
445     // Get merge regions if it is a merged region and already has merge
446     // qualifier
447     Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor
448         .getRegionsFromMergeQualifier(this.services.getConnection(),
449           region.getRegionName());
450     if (mergeRegions == null
451         || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) {
452       // It doesn't have merge qualifier, no need to clean
453       return true;
454     }
455     // It shouldn't happen, we must insert/delete these two qualifiers together
456     if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {
457       LOG.error("Merged region " + region.getRegionNameAsString()
458           + " has only one merge qualifier in META.");
459       return false;
460     }
461     return cleanMergeRegion(region, mergeRegions.getFirst(),
462         mergeRegions.getSecond());
463   }
464 }