View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver.compactions;
19  
20  import java.io.IOException;
21  import java.io.InterruptedIOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.List;
25  
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.regionserver.InternalScanner;
30  import org.apache.hadoop.hbase.regionserver.ScanType;
31  import org.apache.hadoop.hbase.regionserver.Store;
32  import org.apache.hadoop.hbase.regionserver.StoreFile;
33  import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
34  
35  /**
36   * Compact passed set of files. Create an instance and then call {@link #compact(CompactionRequest)}
37   */
38  @InterfaceAudience.Private
39  public class DefaultCompactor extends Compactor {
40    public DefaultCompactor(final Configuration conf, final Store store) {
41      super(conf, store);
42    }
43  
44    /**
45     * Do a minor/major compaction on an explicit set of storefiles from a Store.
46     */
47    public List<Path> compact(final CompactionRequest request) throws IOException {
48      FileDetails fd = getFileDetails(request.getFiles(), request.isAllFiles());
49      this.progress = new CompactionProgress(fd.maxKeyCount);
50  
51      // Find the smallest read point across all the Scanners.
52      long smallestReadPoint = getSmallestReadPoint();
53      List<StoreFileScanner> scanners = createFileScanners(request.getFiles(), smallestReadPoint);
54  
55      StoreFile.Writer writer = null;
56      List<Path> newFiles = new ArrayList<Path>();
57      boolean cleanSeqId = false;
58      IOException e = null;
59      try {
60        InternalScanner scanner = null;
61        try {
62          /* Include deletes, unless we are doing a compaction of all files */
63          ScanType scanType =
64              request.isAllFiles() ? ScanType.COMPACT_DROP_DELETES : ScanType.COMPACT_RETAIN_DELETES;
65          scanner = preCreateCoprocScanner(request, scanType, fd.earliestPutTs, scanners);
66          if (scanner == null) {
67            scanner = createScanner(store, scanners, scanType, smallestReadPoint, fd.earliestPutTs);
68          }
69          scanner = postCreateCoprocScanner(request, scanType, scanner);
70          if (scanner == null) {
71            // NULL scanner returned from coprocessor hooks means skip normal processing.
72            return newFiles;
73          }
74          // Create the writer even if no kv(Empty store file is also ok),
75          // because we need record the max seq id for the store file, see HBASE-6059
76          if(fd.minSeqIdToKeep > 0) {
77            smallestReadPoint = Math.min(fd.minSeqIdToKeep, smallestReadPoint);
78            cleanSeqId = true;
79          }
80          writer = store.createWriterInTmp(fd.maxKeyCount, this.compactionCompression, true,
81              fd.maxMVCCReadpoint >= smallestReadPoint, fd.maxTagsLength > 0);
82          boolean finished = performCompaction(scanner, writer, smallestReadPoint, cleanSeqId);
83          if (!finished) {
84            writer.close();
85            store.getFileSystem().delete(writer.getPath(), false);
86            writer = null;
87            throw new InterruptedIOException( "Aborting compaction of store " + store +
88                " in region " + store.getRegionInfo().getRegionNameAsString() +
89                " because it was interrupted.");
90           }
91         } finally {
92           if (scanner != null) {
93             scanner.close();
94           }
95        }
96      } catch (IOException ioe) {
97        e = ioe;
98        // Throw the exception
99        throw ioe;
100     }
101     finally {
102       if (writer != null) {
103         if (e != null) {
104           writer.close();
105         } else {
106           writer.appendMetadata(fd.maxSeqId, request.isAllFiles());
107           writer.close();
108           newFiles.add(writer.getPath());
109         }
110       }
111     }
112     return newFiles;
113   }
114 
115   /**
116    * Compact a list of files for testing. Creates a fake {@link CompactionRequest} to pass to
117    * {@link #compact(CompactionRequest)};
118    * @param filesToCompact the files to compact. These are used as the compactionSelection for
119    *          the generated {@link CompactionRequest}.
120    * @param isMajor true to major compact (prune all deletes, max versions, etc)
121    * @return Product of compaction or an empty list if all cells expired or deleted and nothing \
122    *         made it through the compaction.
123    * @throws IOException
124    */
125   public List<Path> compactForTesting(final Collection<StoreFile> filesToCompact, boolean isMajor)
126       throws IOException {
127     CompactionRequest cr = new CompactionRequest(filesToCompact);
128     cr.setIsMajor(isMajor, isMajor);
129     return this.compact(cr);
130   }
131 }