001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY;
021
022import java.io.IOException;
023import java.util.List;
024import org.apache.hadoop.conf.Configuration;
025import org.apache.hadoop.fs.Path;
026import org.apache.hadoop.hbase.CellComparator;
027import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
028import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
029import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy;
030import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest;
031import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactor;
032import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
033import org.apache.hadoop.hbase.security.User;
034import org.apache.hadoop.hbase.util.ReflectionUtils;
035import org.apache.yetus.audience.InterfaceAudience;
036
037/**
038 * HBASE-15400 This store engine allows us to store data in date tiered layout with exponential
039 * sizing so that the more recent data has more granularity. Time-range scan will perform the best
040 * with most recent data. When data reach maxAge, they are compacted in fixed-size time windows for
041 * TTL and archiving. Please refer to design spec for more details.
042 * https://docs.google.com/document/d/1_AmlNb2N8Us1xICsTeGDLKIqL6T-oHoRLZ323MG_uy8/edit#heading=h.uk6y5pd3oqgx
043 */
044@InterfaceAudience.Private
045public class DateTieredStoreEngine extends StoreEngine<DefaultStoreFlusher,
046  DateTieredCompactionPolicy, DateTieredCompactor, DefaultStoreFileManager> {
047
048  public static final String DATE_TIERED_STORE_ENGINE = DateTieredStoreEngine.class.getName();
049
050  protected void createCompactionPolicy(Configuration conf, HStore store) throws IOException {
051    String className =
052      conf.get(DEFAULT_COMPACTION_POLICY_CLASS_KEY, DateTieredCompactionPolicy.class.getName());
053    try {
054      compactionPolicy = ReflectionUtils.instantiateWithCustomCtor(className,
055        new Class[] { Configuration.class, StoreConfigInformation.class },
056        new Object[] { conf, store });
057    } catch (Exception e) {
058      throw new IOException("Unable to load configured compaction policy '" + className + "'", e);
059    }
060  }
061
062  @Override
063  public boolean needsCompaction(List<HStoreFile> filesCompacting) {
064    return compactionPolicy.needsCompaction(storeFileManager.getStoreFiles(), filesCompacting);
065  }
066
067  @Override
068  public CompactionContext createCompaction() throws IOException {
069    return new DateTieredCompactionContext();
070  }
071
072  @Override
073  protected void createComponents(Configuration conf, HStore store, CellComparator kvComparator)
074    throws IOException {
075    createCompactionPolicy(conf, store);
076    this.storeFileManager = new DefaultStoreFileManager(kvComparator,
077      StoreFileComparators.SEQ_ID_MAX_TIMESTAMP, conf, compactionPolicy.getConf());
078    this.storeFlusher = new DefaultStoreFlusher(conf, store);
079    this.compactor = new DateTieredCompactor(conf, store);
080  }
081
082  private final class DateTieredCompactionContext extends CompactionContext {
083
084    @Override
085    public List<HStoreFile> preSelect(List<HStoreFile> filesCompacting) {
086      return compactionPolicy.preSelectCompactionForCoprocessor(storeFileManager.getStoreFiles(),
087        filesCompacting);
088    }
089
090    @Override
091    public boolean select(List<HStoreFile> filesCompacting, boolean isUserCompaction,
092      boolean mayUseOffPeak, boolean forceMajor) throws IOException {
093      request = compactionPolicy.selectCompaction(storeFileManager.getStoreFiles(), filesCompacting,
094        isUserCompaction, mayUseOffPeak, forceMajor);
095      return request != null;
096    }
097
098    @Override
099    public void forceSelect(CompactionRequestImpl request) {
100      if (!(request instanceof DateTieredCompactionRequest)) {
101        throw new IllegalArgumentException("DateTieredCompactionRequest is expected. Actual: "
102          + request.getClass().getCanonicalName());
103      }
104      super.forceSelect(request);
105    }
106
107    @Override
108    public List<Path> compact(ThroughputController throughputController, User user)
109      throws IOException {
110      if (request instanceof DateTieredCompactionRequest) {
111        DateTieredCompactionRequest compactionRequest = (DateTieredCompactionRequest) request;
112        return compactor.compact(request, compactionRequest.getBoundaries(),
113          compactionRequest.getBoundariesPolicies(), throughputController, user);
114      } else {
115        throw new IllegalArgumentException("DateTieredCompactionRequest is expected. Actual: "
116          + request.getClass().getCanonicalName());
117      }
118    }
119  }
120}