View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import java.io.IOException;
21  import java.text.MessageFormat;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.classification.InterfaceStability;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.HRegionLocation;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.client.Connection;
34  import org.apache.hadoop.hbase.client.ConnectionFactory;
35  import org.apache.hadoop.hbase.client.HTable;
36  import org.apache.hadoop.hbase.client.RegionLocator;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.client.Table;
40  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.Pair;
43  import org.apache.hadoop.hbase.util.RegionSizeCalculator;
44  import org.apache.hadoop.mapreduce.InputFormat;
45  import org.apache.hadoop.mapreduce.InputSplit;
46  import org.apache.hadoop.mapreduce.JobContext;
47  import org.apache.hadoop.mapreduce.RecordReader;
48  import org.apache.hadoop.mapreduce.TaskAttemptContext;
49  
50  /**
51   * A base for {@link MultiTableInputFormat}s. Receives a list of
52   * {@link Scan} instances that define the input tables and
53   * filters etc. Subclasses may use other TableRecordReader implementations.
54   */
55  @InterfaceAudience.Public
56  @InterfaceStability.Evolving
57  public abstract class MultiTableInputFormatBase extends
58      InputFormat<ImmutableBytesWritable, Result> {
59  
60    final Log LOG = LogFactory.getLog(MultiTableInputFormatBase.class);
61  
62    /** Holds the set of scans used to define the input. */
63    private List<Scan> scans;
64  
65    /** The reader scanning the table, can be a custom one. */
66    private TableRecordReader tableRecordReader = null;
67  
68    /**
69     * Builds a TableRecordReader. If no TableRecordReader was provided, uses the
70     * default.
71     *
72     * @param split The split to work with.
73     * @param context The current context.
74     * @return The newly created record reader.
75     * @throws IOException When creating the reader fails.
76     * @throws InterruptedException when record reader initialization fails
77     * @see org.apache.hadoop.mapreduce.InputFormat#createRecordReader(
78     *      org.apache.hadoop.mapreduce.InputSplit,
79     *      org.apache.hadoop.mapreduce.TaskAttemptContext)
80     */
81    @Override
82    public RecordReader<ImmutableBytesWritable, Result> createRecordReader(
83        InputSplit split, TaskAttemptContext context)
84        throws IOException, InterruptedException {
85      TableSplit tSplit = (TableSplit) split;
86      LOG.info(MessageFormat.format("Input split length: {0} bytes.", tSplit.getLength()));
87  
88      if (tSplit.getTable() == null) {
89        throw new IOException("Cannot create a record reader because of a"
90            + " previous error. Please look at the previous logs lines from"
91            + " the task's full log for more details.");
92      }
93      Connection connection = ConnectionFactory.createConnection(context.getConfiguration());
94      Table table = connection.getTable(tSplit.getTable());
95  
96      TableRecordReader trr = this.tableRecordReader;
97  
98      try {
99        // if no table record reader was provided use default
100       if (trr == null) {
101         trr = new TableRecordReader();
102       }
103       Scan sc = tSplit.getScan();
104       sc.setStartRow(tSplit.getStartRow());
105       sc.setStopRow(tSplit.getEndRow());
106       trr.setScan(sc);
107       trr.setTable(table);
108     } catch (IOException ioe) {
109       // If there is an exception make sure that all
110       // resources are closed and released.
111       connection.close();
112       table.close();
113       trr.close();
114       throw ioe;
115     }
116     return trr;
117   }
118 
119   /**
120    * Calculates the splits that will serve as input for the map tasks. The
121    * number of splits matches the number of regions in a table.
122    *
123    * @param context The current job context.
124    * @return The list of input splits.
125    * @throws IOException When creating the list of splits fails.
126    * @see org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce.JobContext)
127    */
128   @Override
129   public List<InputSplit> getSplits(JobContext context) throws IOException {
130     if (scans.isEmpty()) {
131       throw new IOException("No scans were provided.");
132     }
133     List<InputSplit> splits = new ArrayList<InputSplit>();
134 
135     for (Scan scan : scans) {
136       byte[] tableNameBytes = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME);
137       if (tableNameBytes == null)
138         throw new IOException("A scan object did not have a table name");
139 
140       TableName tableName = TableName.valueOf(tableNameBytes);
141       Table table = null;
142       RegionLocator regionLocator = null;
143       Connection conn = null;
144       try {
145         conn = ConnectionFactory.createConnection(context.getConfiguration());
146         table = conn.getTable(tableName);
147         regionLocator = conn.getRegionLocator(tableName);
148         regionLocator = (RegionLocator) table;
149         Pair<byte[][], byte[][]> keys = regionLocator.getStartEndKeys();
150         if (keys == null || keys.getFirst() == null ||
151             keys.getFirst().length == 0) {
152           throw new IOException("Expecting at least one region for table : "
153               + tableName.getNameAsString());
154         }
155         int count = 0;
156 
157         byte[] startRow = scan.getStartRow();
158         byte[] stopRow = scan.getStopRow();
159 
160         RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(
161             regionLocator, conn.getAdmin());
162 
163         for (int i = 0; i < keys.getFirst().length; i++) {
164           if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
165             continue;
166           }
167           HRegionLocation hregionLocation = regionLocator.getRegionLocation(
168               keys.getFirst()[i], false);
169           String regionHostname = hregionLocation.getHostname();
170           HRegionInfo regionInfo = hregionLocation.getRegionInfo();
171 
172           // determine if the given start and stop keys fall into the range
173           if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
174               Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
175               (stopRow.length == 0 ||
176                   Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
177             byte[] splitStart =
178                 startRow.length == 0 ||
179                     Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys
180                     .getFirst()[i] : startRow;
181             byte[] splitStop =
182                 (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i],
183                     stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys
184                     .getSecond()[i] : stopRow;
185 
186             long regionSize = sizeCalculator.getRegionSize(regionInfo.getRegionName());
187             TableSplit split =
188                 new TableSplit(regionLocator.getName(),
189                     scan, splitStart, splitStop, regionHostname, regionSize);
190 
191             splits.add(split);
192             if (LOG.isDebugEnabled())
193               LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
194           }
195         }
196       } finally {
197         if (null != table) table.close();
198         if (null != regionLocator) regionLocator.close();
199         if (null != conn) conn.close();
200       }
201     }
202     return splits;
203   }
204 
205   /**
206    * Test if the given region is to be included in the InputSplit while
207    * splitting the regions of a table.
208    * <p>
209    * This optimization is effective when there is a specific reasoning to
210    * exclude an entire region from the M-R job, (and hence, not contributing to
211    * the InputSplit), given the start and end keys of the same. <br>
212    * Useful when we need to remember the last-processed top record and revisit
213    * the [last, current) interval for M-R processing, continuously. In addition
214    * to reducing InputSplits, reduces the load on the region server as well, due
215    * to the ordering of the keys. <br>
216    * <br>
217    * Note: It is possible that <code>endKey.length() == 0 </code> , for the last
218    * (recent) region. <br>
219    * Override this method, if you want to bulk exclude regions altogether from
220    * M-R. By default, no region is excluded( i.e. all regions are included).
221    *
222    * @param startKey Start key of the region
223    * @param endKey End key of the region
224    * @return true, if this region needs to be included as part of the input
225    *         (default).
226    */
227   protected boolean includeRegionInSplit(final byte[] startKey,
228       final byte[] endKey) {
229     return true;
230   }
231 
232   /**
233    * Allows subclasses to get the list of {@link Scan} objects.
234    */
235   protected List<Scan> getScans() {
236     return this.scans;
237   }
238 
239   /**
240    * Allows subclasses to set the list of {@link Scan} objects.
241    *
242    * @param scans The list of {@link Scan} used to define the input
243    */
244   protected void setScans(List<Scan> scans) {
245     this.scans = scans;
246   }
247 
248   /**
249    * Allows subclasses to set the {@link TableRecordReader}.
250    *
251    * @param tableRecordReader A different {@link TableRecordReader}
252    *          implementation.
253    */
254   protected void setTableRecordReader(TableRecordReader tableRecordReader) {
255     this.tableRecordReader = tableRecordReader;
256   }
257 }