View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.util.Comparator;
24  import java.util.HashSet;
25  import java.util.Map;
26  import java.util.TreeMap;
27  import java.util.concurrent.atomic.AtomicBoolean;
28  import java.util.concurrent.atomic.AtomicInteger;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.Chore;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HTableDescriptor;
41  import org.apache.hadoop.hbase.Server;
42  import org.apache.hadoop.hbase.backup.HFileArchiver;
43  import org.apache.hadoop.hbase.MetaTableAccessor;
44  import org.apache.hadoop.hbase.client.MetaScanner;
45  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
46  import org.apache.hadoop.hbase.client.Result;
47  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.FSUtils;
50  import org.apache.hadoop.hbase.util.Pair;
51  import org.apache.hadoop.hbase.util.PairOfSameType;
52  import org.apache.hadoop.hbase.util.Triple;
53  
54  /**
55   * A janitor for the catalog tables.  Scans the <code>hbase:meta</code> catalog
56   * table on a period looking for unused regions to garbage collect.
57   */
58  @InterfaceAudience.Private
59  public class CatalogJanitor extends Chore {
60    private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
61    private final Server server;
62    private final MasterServices services;
63    private AtomicBoolean enabled = new AtomicBoolean(true);
64    private AtomicBoolean alreadyRunning = new AtomicBoolean(false);
65  
66    CatalogJanitor(final Server server, final MasterServices services) {
67      super("CatalogJanitor-" + server.getServerName().toShortString(),
68        server.getConfiguration().getInt("hbase.catalogjanitor.interval", 300000),
69        server);
70      this.server = server;
71      this.services = services;
72    }
73  
74    @Override
75    protected boolean initialChore() {
76      try {
77        if (this.enabled.get()) scan();
78      } catch (IOException e) {
79        LOG.warn("Failed initial scan of catalog table", e);
80        return false;
81      }
82      return true;
83    }
84  
85    /**
86     * @param enabled
87     */
88    public boolean setEnabled(final boolean enabled) {
89      return this.enabled.getAndSet(enabled);
90    }
91  
92    boolean getEnabled() {
93      return this.enabled.get();
94    }
95  
96    @Override
97    protected void chore() {
98      try {
99        if (this.enabled.get()) {
100         scan();
101       } else {
102         LOG.warn("CatalogJanitor disabled! Not running scan.");
103       }
104     } catch (IOException e) {
105       LOG.warn("Failed scan of catalog table", e);
106     }
107   }
108 
109   /**
110    * Scans hbase:meta and returns a number of scanned rows, and a map of merged
111    * regions, and an ordered map of split parents.
112    * @return triple of scanned rows, map of merged regions and map of split
113    *         parent regioninfos
114    * @throws IOException
115    */
116   Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents()
117       throws IOException {
118     return getMergedRegionsAndSplitParents(null);
119   }
120 
121   /**
122    * Scans hbase:meta and returns a number of scanned rows, and a map of merged
123    * regions, and an ordered map of split parents. if the given table name is
124    * null, return merged regions and split parents of all tables, else only the
125    * specified table
126    * @param tableName null represents all tables
127    * @return triple of scanned rows, and map of merged regions, and map of split
128    *         parent regioninfos
129    * @throws IOException
130    */
131   Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents(
132       final TableName tableName) throws IOException {
133     final boolean isTableSpecified = (tableName != null);
134     // TODO: Only works with single hbase:meta region currently.  Fix.
135     final AtomicInteger count = new AtomicInteger(0);
136     // Keep Map of found split parents.  There are candidates for cleanup.
137     // Use a comparator that has split parents come before its daughters.
138     final Map<HRegionInfo, Result> splitParents =
139       new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
140     final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
141     // This visitor collects split parents and counts rows in the hbase:meta table
142 
143     MetaScannerVisitor visitor = new MetaScanner.MetaScannerVisitorBase() {
144       @Override
145       public boolean processRow(Result r) throws IOException {
146         if (r == null || r.isEmpty()) return true;
147         count.incrementAndGet();
148         HRegionInfo info = HRegionInfo.getHRegionInfo(r);
149         if (info == null) return true; // Keep scanning
150         if (isTableSpecified
151             && info.getTable().compareTo(tableName) > 0) {
152           // Another table, stop scanning
153           return false;
154         }
155         if (info.isSplitParent()) splitParents.put(info, r);
156         if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {
157           mergedRegions.put(info, r);
158         }
159         // Returning true means "keep scanning"
160         return true;
161       }
162     };
163 
164     // Run full scan of hbase:meta catalog table passing in our custom visitor with
165     // the start row
166     MetaScanner.metaScan(server.getConfiguration(), null, visitor, tableName);
167 
168     return new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
169         count.get(), mergedRegions, splitParents);
170   }
171 
172   /**
173    * If merged region no longer holds reference to the merge regions, archive
174    * merge region on hdfs and perform deleting references in hbase:meta
175    * @param mergedRegion
176    * @param regionA
177    * @param regionB
178    * @return true if we delete references in merged region on hbase:meta and archive
179    *         the files on the file system
180    * @throws IOException
181    */
182   boolean cleanMergeRegion(final HRegionInfo mergedRegion,
183       final HRegionInfo regionA, final HRegionInfo regionB) throws IOException {
184     FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
185     Path rootdir = this.services.getMasterFileSystem().getRootDir();
186     Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
187     HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
188     HRegionFileSystem regionFs = null;
189     try {
190       regionFs = HRegionFileSystem.openRegionFromFileSystem(
191           this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
192     } catch (IOException e) {
193       LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
194     }
195     if (regionFs == null || !regionFs.hasReferences(htd)) {
196       LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
197           + regionB.getRegionNameAsString()
198           + " from fs because merged region no longer holds references");
199       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
200       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
201       MetaTableAccessor.deleteMergeQualifiers(server.getShortCircuitConnection(),
202         mergedRegion);
203       return true;
204     }
205     return false;
206   }
207 
208   /**
209    * Run janitorial scan of catalog <code>hbase:meta</code> table looking for
210    * garbage to collect.
211    * @return number of cleaned regions
212    * @throws IOException
213    */
214   int scan() throws IOException {
215     try {
216       if (!alreadyRunning.compareAndSet(false, true)) {
217         return 0;
218       }
219       Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> scanTriple =
220         getMergedRegionsAndSplitParents();
221       int count = scanTriple.getFirst();
222       /**
223        * clean merge regions first
224        */
225       int mergeCleaned = 0;
226       Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond();
227       for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) {
228         HRegionInfo regionA = HRegionInfo.getHRegionInfo(e.getValue(),
229             HConstants.MERGEA_QUALIFIER);
230         HRegionInfo regionB = HRegionInfo.getHRegionInfo(e.getValue(),
231             HConstants.MERGEB_QUALIFIER);
232         if (regionA == null || regionB == null) {
233           LOG.warn("Unexpected references regionA="
234               + (regionA == null ? "null" : regionA.getRegionNameAsString())
235               + ",regionB="
236               + (regionB == null ? "null" : regionB.getRegionNameAsString())
237               + " in merged region " + e.getKey().getRegionNameAsString());
238         } else {
239           if (cleanMergeRegion(e.getKey(), regionA, regionB)) {
240             mergeCleaned++;
241           }
242         }
243       }
244       /**
245        * clean split parents
246        */
247       Map<HRegionInfo, Result> splitParents = scanTriple.getThird();
248 
249       // Now work on our list of found parents. See if any we can clean up.
250       int splitCleaned = 0;
251       // regions whose parents are still around
252       HashSet<String> parentNotCleaned = new HashSet<String>();
253       for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
254         if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
255             cleanParent(e.getKey(), e.getValue())) {
256           splitCleaned++;
257         } else {
258           // We could not clean the parent, so it's daughters should not be cleaned either (HBASE-6160)
259           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(e.getValue());
260           parentNotCleaned.add(daughters.getFirst().getEncodedName());
261           parentNotCleaned.add(daughters.getSecond().getEncodedName());
262         }
263       }
264       if ((mergeCleaned + splitCleaned) != 0) {
265         LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
266             + " unreferenced merged region(s) and " + splitCleaned
267             + " unreferenced parent region(s)");
268       } else if (LOG.isTraceEnabled()) {
269         LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
270             + " unreferenced merged region(s) and " + splitCleaned
271             + " unreferenced parent region(s)");
272       }
273       return mergeCleaned + splitCleaned;
274     } finally {
275       alreadyRunning.set(false);
276     }
277   }
278 
279   /**
280    * Compare HRegionInfos in a way that has split parents sort BEFORE their
281    * daughters.
282    */
283   static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
284     Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
285     @Override
286     public int compare(HRegionInfo left, HRegionInfo right) {
287       // This comparator differs from the one HRegionInfo in that it sorts
288       // parent before daughters.
289       if (left == null) return -1;
290       if (right == null) return 1;
291       // Same table name.
292       int result = left.getTable().compareTo(right.getTable());
293       if (result != 0) return result;
294       // Compare start keys.
295       result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
296       if (result != 0) return result;
297       // Compare end keys, but flip the operands so parent comes first
298       result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
299 
300       return result;
301     }
302   }
303 
304   /**
305    * If daughters no longer hold reference to the parents, delete the parent.
306    * @param parent HRegionInfo of split offlined parent
307    * @param rowContent Content of <code>parent</code> row in
308    * <code>metaRegionName</code>
309    * @return True if we removed <code>parent</code> from meta table and from
310    * the filesystem.
311    * @throws IOException
312    */
313   boolean cleanParent(final HRegionInfo parent, Result rowContent)
314   throws IOException {
315     boolean result = false;
316     // Check whether it is a merged region and not clean reference
317     // No necessary to check MERGEB_QUALIFIER because these two qualifiers will
318     // be inserted/deleted together
319     if (rowContent.getValue(HConstants.CATALOG_FAMILY,
320         HConstants.MERGEA_QUALIFIER) != null) {
321       // wait cleaning merge region first
322       return result;
323     }
324     // Run checks on each daughter split.
325     PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(rowContent);
326     Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
327     Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
328     if (hasNoReferences(a) && hasNoReferences(b)) {
329       LOG.debug("Deleting region " + parent.getRegionNameAsString() +
330         " because daughter splits no longer hold references");
331       FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
332       if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
333       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
334       MetaTableAccessor.deleteRegion(this.server.getShortCircuitConnection(), parent);
335       result = true;
336     }
337     return result;
338   }
339 
340   /**
341    * @param p A pair where the first boolean says whether or not the daughter
342    * region directory exists in the filesystem and then the second boolean says
343    * whether the daughter has references to the parent.
344    * @return True the passed <code>p</code> signifies no references.
345    */
346   private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
347     return !p.getFirst() || !p.getSecond();
348   }
349 
350   /**
351    * Checks if a daughter region -- either splitA or splitB -- still holds
352    * references to parent.
353    * @param parent Parent region
354    * @param daughter Daughter region
355    * @return A pair where the first boolean says whether or not the daughter
356    * region directory exists in the filesystem and then the second boolean says
357    * whether the daughter has references to the parent.
358    * @throws IOException
359    */
360   Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter)
361   throws IOException {
362     if (daughter == null)  {
363       return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
364     }
365 
366     FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
367     Path rootdir = this.services.getMasterFileSystem().getRootDir();
368     Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
369 
370     HRegionFileSystem regionFs = null;
371     try {
372       regionFs = HRegionFileSystem.openRegionFromFileSystem(
373           this.services.getConfiguration(), fs, tabledir, daughter, true);
374     } catch (IOException e) {
375       LOG.warn("Daughter region does not exist: " + daughter.getEncodedName()
376         + ", parent is: " + parent.getEncodedName());
377       return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
378     }
379 
380     boolean references = false;
381     HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
382     for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
383       if ((references = regionFs.hasReferences(family.getNameAsString()))) {
384         break;
385       }
386     }
387     return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references));
388   }
389 
390   private HTableDescriptor getTableDescriptor(final TableName tableName)
391       throws FileNotFoundException, IOException {
392     return this.services.getTableDescriptors().get(tableName);
393   }
394 
395   /**
396    * Checks if the specified region has merge qualifiers, if so, try to clean
397    * them
398    * @param region
399    * @return true if the specified region doesn't have merge qualifier now
400    * @throws IOException
401    */
402   public boolean cleanMergeQualifier(final HRegionInfo region)
403       throws IOException {
404     // Get merge regions if it is a merged region and already has merge
405     // qualifier
406     Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor
407         .getRegionsFromMergeQualifier(this.services.getShortCircuitConnection(),
408           region.getRegionName());
409     if (mergeRegions == null
410         || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) {
411       // It doesn't have merge qualifier, no need to clean
412       return true;
413     }
414     // It shouldn't happen, we must insert/delete these two qualifiers together
415     if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {
416       LOG.error("Merged region " + region.getRegionNameAsString()
417           + " has only one merge qualifier in META.");
418       return false;
419     }
420     return cleanMergeRegion(region, mergeRegions.getFirst(),
421         mergeRegions.getSecond());
422   }
423 }