View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import java.io.IOException;
21  import java.text.MessageFormat;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.classification.InterfaceAudience;
28  import org.apache.hadoop.classification.InterfaceStability;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.HRegionLocation;
31  import org.apache.hadoop.hbase.client.HTable;
32  import org.apache.hadoop.hbase.client.Result;
33  import org.apache.hadoop.hbase.client.Scan;
34  import org.apache.hadoop.hbase.client.Table;
35  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.hbase.util.Pair;
38  import org.apache.hadoop.hbase.util.RegionSizeCalculator;
39  import org.apache.hadoop.mapreduce.InputFormat;
40  import org.apache.hadoop.mapreduce.InputSplit;
41  import org.apache.hadoop.mapreduce.JobContext;
42  import org.apache.hadoop.mapreduce.RecordReader;
43  import org.apache.hadoop.mapreduce.TaskAttemptContext;
44  
45  /**
46   * A base for {@link MultiTableInputFormat}s. Receives a list of
47   * {@link Scan} instances that define the input tables and
48   * filters etc. Subclasses may use other TableRecordReader implementations.
49   */
50  @InterfaceAudience.Public
51  @InterfaceStability.Evolving
52  public abstract class MultiTableInputFormatBase extends
53      InputFormat<ImmutableBytesWritable, Result> {
54  
55    final Log LOG = LogFactory.getLog(MultiTableInputFormatBase.class);
56  
57    /** Holds the set of scans used to define the input. */
58    private List<Scan> scans;
59  
60    /** The reader scanning the table, can be a custom one. */
61    private TableRecordReader tableRecordReader = null;
62  
63    /**
64     * Builds a TableRecordReader. If no TableRecordReader was provided, uses the
65     * default.
66     *
67     * @param split The split to work with.
68     * @param context The current context.
69     * @return The newly created record reader.
70     * @throws IOException When creating the reader fails.
71     * @throws InterruptedException when record reader initialization fails
72     * @see org.apache.hadoop.mapreduce.InputFormat#createRecordReader(
73     *      org.apache.hadoop.mapreduce.InputSplit,
74     *      org.apache.hadoop.mapreduce.TaskAttemptContext)
75     */
76    @Override
77    public RecordReader<ImmutableBytesWritable, Result> createRecordReader(
78        InputSplit split, TaskAttemptContext context)
79        throws IOException, InterruptedException {
80      TableSplit tSplit = (TableSplit) split;
81      LOG.info(MessageFormat.format("Input split length: {0} bytes.", tSplit.getLength()));
82  
83      if (tSplit.getTableName() == null) {
84        throw new IOException("Cannot create a record reader because of a"
85            + " previous error. Please look at the previous logs lines from"
86            + " the task's full log for more details.");
87      }
88      Table table =
89          new HTable(context.getConfiguration(), tSplit.getTableName());
90  
91      TableRecordReader trr = this.tableRecordReader;
92  
93      try {
94        // if no table record reader was provided use default
95        if (trr == null) {
96          trr = new TableRecordReader();
97        }
98        Scan sc = tSplit.getScan();
99        sc.setStartRow(tSplit.getStartRow());
100       sc.setStopRow(tSplit.getEndRow());
101       trr.setScan(sc);
102       trr.setHTable(table);
103     } catch (IOException ioe) {
104       // If there is an exception make sure that all
105       // resources are closed and released.
106       table.close();
107       trr.close();
108       throw ioe;
109     }
110     return trr;
111   }
112 
113   /**
114    * Calculates the splits that will serve as input for the map tasks. The
115    * number of splits matches the number of regions in a table.
116    *
117    * @param context The current job context.
118    * @return The list of input splits.
119    * @throws IOException When creating the list of splits fails.
120    * @see org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce.JobContext)
121    */
122   @Override
123   public List<InputSplit> getSplits(JobContext context) throws IOException {
124     if (scans.isEmpty()) {
125       throw new IOException("No scans were provided.");
126     }
127     List<InputSplit> splits = new ArrayList<InputSplit>();
128 
129     for (Scan scan : scans) {
130       byte[] tableName = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME);
131       if (tableName == null) 
132         throw new IOException("A scan object did not have a table name");
133 
134       HTable table = null;
135       try {
136         table = new HTable(context.getConfiguration(), tableName);
137         Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
138         if (keys == null || keys.getFirst() == null ||
139             keys.getFirst().length == 0) {
140           throw new IOException("Expecting at least one region for table : "
141               + Bytes.toString(tableName));
142         }
143         int count = 0;
144 
145         byte[] startRow = scan.getStartRow();
146         byte[] stopRow = scan.getStopRow();
147 
148         RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(table);
149 
150         for (int i = 0; i < keys.getFirst().length; i++) {
151           if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
152             continue;
153           }
154           HRegionLocation hregionLocation = table.getRegionLocation(keys.getFirst()[i], false);
155           String regionHostname = hregionLocation.getHostname();
156           HRegionInfo regionInfo = hregionLocation.getRegionInfo();
157         
158           // determine if the given start and stop keys fall into the range
159           if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
160               Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
161               (stopRow.length == 0 ||
162                   Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
163             byte[] splitStart =
164                 startRow.length == 0 ||
165                     Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys
166                     .getFirst()[i] : startRow;
167             byte[] splitStop =
168                 (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i],
169                     stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys
170                     .getSecond()[i] : stopRow;
171 
172             long regionSize = sizeCalculator.getRegionSize(regionInfo.getRegionName());
173             TableSplit split =
174                 new TableSplit(table.getName(),
175                     scan, splitStart, splitStop, regionHostname, regionSize);
176 
177             splits.add(split);
178             if (LOG.isDebugEnabled())
179               LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
180           }
181         }
182       } finally {
183         if (null != table) table.close();
184       }
185     }
186     return splits;
187   }
188 
189   /**
190    * Test if the given region is to be included in the InputSplit while
191    * splitting the regions of a table.
192    * <p>
193    * This optimization is effective when there is a specific reasoning to
194    * exclude an entire region from the M-R job, (and hence, not contributing to
195    * the InputSplit), given the start and end keys of the same. <br>
196    * Useful when we need to remember the last-processed top record and revisit
197    * the [last, current) interval for M-R processing, continuously. In addition
198    * to reducing InputSplits, reduces the load on the region server as well, due
199    * to the ordering of the keys. <br>
200    * <br>
201    * Note: It is possible that <code>endKey.length() == 0 </code> , for the last
202    * (recent) region. <br>
203    * Override this method, if you want to bulk exclude regions altogether from
204    * M-R. By default, no region is excluded( i.e. all regions are included).
205    *
206    * @param startKey Start key of the region
207    * @param endKey End key of the region
208    * @return true, if this region needs to be included as part of the input
209    *         (default).
210    */
211   protected boolean includeRegionInSplit(final byte[] startKey,
212       final byte[] endKey) {
213     return true;
214   }
215 
216   /**
217    * Allows subclasses to get the list of {@link Scan} objects.
218    */
219   protected List<Scan> getScans() {
220     return this.scans;
221   }
222 
223   /**
224    * Allows subclasses to set the list of {@link Scan} objects.
225    *
226    * @param scans The list of {@link Scan} used to define the input
227    */
228   protected void setScans(List<Scan> scans) {
229     this.scans = scans;
230   }
231 
232   /**
233    * Allows subclasses to set the {@link TableRecordReader}.
234    *
235    * @param tableRecordReader A different {@link TableRecordReader}
236    *          implementation.
237    */
238   protected void setTableRecordReader(TableRecordReader tableRecordReader) {
239     this.tableRecordReader = tableRecordReader;
240   }
241 }