View Javadoc

1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver.compactions;
21  
22  import java.util.ArrayList;
23  import java.util.Calendar;
24  import java.util.GregorianCalendar;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.regionserver.StoreFile;
31  
32  public class CompactSelection {
33    private static final long serialVersionUID = 1L;
34    static final Log LOG = LogFactory.getLog(CompactSelection.class);
35    // the actual list - this is needed to handle methods like "sublist"
36    // correctly
37    List<StoreFile> filesToCompact = new ArrayList<StoreFile>();
38  
39    /**
40     * Number of off peak compactions either in the compaction queue or
41     * happening now. Please lock compactionCountLock before modifying.
42     */
43    static long numOutstandingOffPeakCompactions = 0;
44  
45    /**
46     * Lock object for numOutstandingOffPeakCompactions
47     */
48    private final static Object compactionCountLock = new Object();
49  
50    // HBase conf object
51    Configuration conf;
52    // was this compaction promoted to an off-peak
53    boolean isOffPeakCompaction = false;
54    // compactRatio: double on purpose!  Float.MAX < Long.MAX < Double.MAX
55    // With float, java will downcast your long to float for comparisons (bad)
56    private double compactRatio;
57    // compaction ratio off-peak
58    private double compactRatioOffPeak;
59    // offpeak start time
60    private int offPeakStartHour = -1;
61    // off peak end time
62    private int offPeakEndHour = -1;
63  
64    public CompactSelection(Configuration conf, List<StoreFile> filesToCompact) {
65      this.filesToCompact = filesToCompact;
66      this.conf = conf;
67      this.compactRatio = conf.getFloat("hbase.hstore.compaction.ratio", 1.2F);
68      this.compactRatioOffPeak = conf.getFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
69  
70      // Peak time is from [offPeakStartHour, offPeakEndHour). Valid numbers are [0, 23]
71      this.offPeakStartHour = conf.getInt("hbase.offpeak.start.hour", -1);
72      this.offPeakEndHour = conf.getInt("hbase.offpeak.end.hour", -1);
73      if (!isValidHour(this.offPeakStartHour) || !isValidHour(this.offPeakEndHour)) {
74        if (!(this.offPeakStartHour == -1 && this.offPeakEndHour == -1)) {
75          LOG.warn("Invalid start/end hour for peak hour : start = " +
76              this.offPeakStartHour + " end = " + this.offPeakEndHour +
77              ". Valid numbers are [0-23]");
78        }
79        this.offPeakStartHour = this.offPeakEndHour = -1;
80      }
81    }
82  
83    /**
84     * Select the expired store files to compact
85     * 
86     * @param maxExpiredTimeStamp
87     *          The store file will be marked as expired if its max time stamp is
88     *          less than this maxExpiredTimeStamp.
89     * @return A CompactSelection contains the expired store files as
90     *         filesToCompact
91     */
92    public CompactSelection selectExpiredStoreFilesToCompact(
93        long maxExpiredTimeStamp) {
94      if (filesToCompact == null || filesToCompact.size() == 0)
95        return null;
96      ArrayList<StoreFile> expiredStoreFiles = null;
97      boolean hasExpiredStoreFiles = false;
98      CompactSelection expiredSFSelection = null;
99  
100     for (StoreFile storeFile : this.filesToCompact) {
101       if (storeFile.getReader().getMaxTimestamp() < maxExpiredTimeStamp) {
102         LOG.info("Deleting the expired store file by compaction: "
103             + storeFile.getPath() + " whose maxTimeStamp is "
104             + storeFile.getReader().getMaxTimestamp()
105             + " while the max expired timestamp is " + maxExpiredTimeStamp);
106         if (!hasExpiredStoreFiles) {
107           expiredStoreFiles = new ArrayList<StoreFile>();
108           hasExpiredStoreFiles = true;
109         }
110         expiredStoreFiles.add(storeFile);
111       }
112     }
113 
114     if (hasExpiredStoreFiles) {
115       if (expiredStoreFiles.size() == 1
116           && expiredStoreFiles.get(0).getReader().getEntries() == 0) {
117         // If just one empty store file, do not select for compaction.
118         return expiredSFSelection;
119       }
120       expiredSFSelection = new CompactSelection(conf, expiredStoreFiles);
121     }
122     return expiredSFSelection;
123   }
124 
125   /**
126    * If the current hour falls in the off peak times and there are no 
127    * outstanding off peak compactions, the current compaction is 
128    * promoted to an off peak compaction. Currently only one off peak 
129    * compaction is present in the compaction queue.
130    *
131    * @param currentHour
132    * @return
133    */
134   public double getCompactSelectionRatio() {
135     double r = this.compactRatio;
136     synchronized(compactionCountLock) {
137       if (isOffPeakHour() && numOutstandingOffPeakCompactions == 0) {
138         r = this.compactRatioOffPeak;
139         numOutstandingOffPeakCompactions++;
140         isOffPeakCompaction = true;
141       }
142     }
143     if(isOffPeakCompaction) {
144       LOG.info("Running an off-peak compaction, selection ratio = " +
145           compactRatioOffPeak + ", numOutstandingOffPeakCompactions is now " +
146           numOutstandingOffPeakCompactions);
147     }
148     return r;
149   }
150 
151   /**
152    * The current compaction finished, so reset the off peak compactions count
153    * if this was an off peak compaction.
154    */
155   public void finishRequest() {
156     if (isOffPeakCompaction) {
157       synchronized(compactionCountLock) {
158         numOutstandingOffPeakCompactions--;
159         isOffPeakCompaction = false;
160       }
161       LOG.info("Compaction done, numOutstandingOffPeakCompactions is now " +
162           numOutstandingOffPeakCompactions);
163     }
164   }
165 
166   public List<StoreFile> getFilesToCompact() {
167     return filesToCompact;
168   }
169 
170   /**
171    * Removes all files from the current compaction list, and resets off peak
172    * compactions is set.
173    */
174   public void emptyFileList() {
175     filesToCompact.clear();
176     if (isOffPeakCompaction) {
177       synchronized(compactionCountLock) {
178         // reset the off peak count
179         numOutstandingOffPeakCompactions--;
180         isOffPeakCompaction = false;
181       }
182       LOG.info("Nothing to compact, numOutstandingOffPeakCompactions is now " +
183           numOutstandingOffPeakCompactions);
184     }
185   }
186 
187   public boolean isOffPeakCompaction() {
188     return this.isOffPeakCompaction;
189   }
190 
191   private boolean isOffPeakHour() {
192     int currentHour = (new GregorianCalendar()).get(Calendar.HOUR_OF_DAY);
193     // If offpeak time checking is disabled just return false.
194     if (this.offPeakStartHour == this.offPeakEndHour) {
195       return false;
196     }
197     if (this.offPeakStartHour < this.offPeakEndHour) {
198       return (currentHour >= this.offPeakStartHour && currentHour < this.offPeakEndHour);
199     }
200     return (currentHour >= this.offPeakStartHour || currentHour < this.offPeakEndHour);
201   }
202 
203   public CompactSelection subList(int start, int end) {
204     throw new UnsupportedOperationException();
205   }
206 
207   public CompactSelection getSubList(int start, int end) {
208     filesToCompact = filesToCompact.subList(start, end);
209     return this;
210   }
211 
212   public void clearSubList(int start, int end) {
213     filesToCompact.subList(start, end).clear();
214   }
215 
216   private boolean isValidHour(int hour) {
217     return (hour >= 0 && hour <= 23);
218   }
219 }