View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.classification.InterfaceStability;
28  import org.apache.hadoop.hbase.client.HTable;
29  import org.apache.hadoop.hbase.client.Result;
30  import org.apache.hadoop.hbase.client.Scan;
31  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.apache.hadoop.hbase.util.Pair;
34  import org.apache.hadoop.mapreduce.InputFormat;
35  import org.apache.hadoop.mapreduce.InputSplit;
36  import org.apache.hadoop.mapreduce.JobContext;
37  import org.apache.hadoop.mapreduce.RecordReader;
38  import org.apache.hadoop.mapreduce.TaskAttemptContext;
39  
40  import java.util.Map;
41  import java.util.HashMap;
42  import java.util.Iterator;
43  /**
44   * A base for {@link MultiTableInputFormat}s. Receives a list of
45   * {@link Scan} instances that define the input tables and
46   * filters etc. Subclasses may use other TableRecordReader implementations.
47   */
48  @InterfaceAudience.Public
49  @InterfaceStability.Evolving
50  public abstract class MultiTableInputFormatBase extends
51      InputFormat<ImmutableBytesWritable, Result> {
52  
53    final Log LOG = LogFactory.getLog(MultiTableInputFormatBase.class);
54  
55    /** Holds the set of scans used to define the input. */
56    private List<Scan> scans;
57  
58    /** The reader scanning the table, can be a custom one. */
59    private TableRecordReader tableRecordReader = null;
60  
61    /**
62     * Builds a TableRecordReader. If no TableRecordReader was provided, uses the
63     * default.
64     *
65     * @param split The split to work with.
66     * @param context The current context.
67     * @return The newly created record reader.
68     * @throws IOException When creating the reader fails.
69     * @throws InterruptedException when record reader initialization fails
70     * @see org.apache.hadoop.mapreduce.InputFormat#createRecordReader(
71     *      org.apache.hadoop.mapreduce.InputSplit,
72     *      org.apache.hadoop.mapreduce.TaskAttemptContext)
73     */
74    @Override
75    public RecordReader<ImmutableBytesWritable, Result> createRecordReader(
76        InputSplit split, TaskAttemptContext context)
77        throws IOException, InterruptedException {
78      TableSplit tSplit = (TableSplit) split;
79  
80      if (tSplit.getTableName() == null) {
81        throw new IOException("Cannot create a record reader because of a"
82            + " previous error. Please look at the previous logs lines from"
83            + " the task's full log for more details.");
84      }
85      HTable table =
86          new HTable(context.getConfiguration(), tSplit.getTableName());
87  
88      TableRecordReader trr = this.tableRecordReader;
89      // if no table record reader was provided use default
90      if (trr == null) {
91        trr = new TableRecordReader();
92      }
93      Scan sc = tSplit.getScan();
94      sc.setStartRow(tSplit.getStartRow());
95      sc.setStopRow(tSplit.getEndRow());
96      trr.setScan(sc);
97      trr.setHTable(table);
98      trr.initialize(split, context);
99      return trr;
100   }
101 
102   /**
103    * Calculates the splits that will serve as input for the map tasks. The
104    * number of splits matches the number of regions in a table.
105    *
106    * @param context The current job context.
107    * @return The list of input splits.
108    * @throws IOException When creating the list of splits fails.
109    * @see org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce.JobContext)
110    */
111 
112 
113   @Override
114   public List<InputSplit> getSplits(JobContext context) throws IOException {
115     if (scans.isEmpty()) {
116       throw new IOException("No scans were provided.");
117     }
118 
119     Map<String, List<Scan>> tableMaps = new HashMap<String, List<Scan>>();
120     for (Scan scan : scans) {
121       byte[] tableName = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME);
122       if (tableName == null)
123         throw new IOException("A scan object did not have a table name");
124       String tableNameStr = Bytes.toString(tableName);
125 
126       List<Scan> scanList = tableMaps.get(tableNameStr);
127 
128       if (scanList == null) {
129         scanList = new ArrayList<Scan>();
130         tableMaps.put(tableNameStr, scanList);
131       }
132       scanList.add(scan);
133     }
134 
135     List<InputSplit> splits = new ArrayList<InputSplit>();
136     Iterator iter = tableMaps.entrySet().iterator();
137     while (iter.hasNext()) {
138       Map.Entry<String, List<Scan>> entry = (Map.Entry<String, List<Scan>>) iter.next();
139       String tableNameStr = entry.getKey();
140       List<Scan> scanList = entry.getValue();
141       HTable table = new HTable(context.getConfiguration(), tableNameStr);
142       Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
143       for (Scan scan : scanList) {
144         if (keys == null || keys.getFirst() == null ||
145                 keys.getFirst().length == 0) {
146           throw new IOException("Expecting at least one region for table : "
147                   + tableNameStr);
148         }
149         int count = 0;
150         byte[] startRow = scan.getStartRow();
151         byte[] stopRow = scan.getStopRow();
152         for (int i = 0; i < keys.getFirst().length; i++) {
153           if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
154             continue;
155           }
156 
157           // determine if the given start and stop keys fall into the range
158           if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
159                   Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
160                   (stopRow.length == 0 || Bytes.compareTo(stopRow,
161                   keys.getFirst()[i]) > 0)) {
162             byte[] splitStart = startRow.length == 0 ||
163                     Bytes.compareTo(keys.getFirst()[i],
164                     startRow) >= 0 ? keys.getFirst()[i] : startRow;
165             byte[] splitStop = (stopRow.length == 0 ||
166                     Bytes.compareTo(keys.getSecond()[i],
167                     stopRow) <= 0) && keys.getSecond()[i].length > 0 ?
168                     keys.getSecond()[i] : stopRow;
169             String regionLocation = table.getRegionLocation(
170                     keys.getFirst()[i], false).getHostname();
171             InputSplit split = new TableSplit(Bytes.toBytes(tableNameStr), scan,
172                     splitStart, splitStop, regionLocation);
173             splits.add(split);
174             if (LOG.isDebugEnabled())
175               LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
176           }
177         }
178       }
179       table.close();
180     }
181 
182     return splits;
183   }
184 
185   /**
186    * Test if the given region is to be included in the InputSplit while
187    * splitting the regions of a table.
188    * <p>
189    * This optimization is effective when there is a specific reasoning to
190    * exclude an entire region from the M-R job, (and hence, not contributing to
191    * the InputSplit), given the start and end keys of the same. <br>
192    * Useful when we need to remember the last-processed top record and revisit
193    * the [last, current) interval for M-R processing, continuously. In addition
194    * to reducing InputSplits, reduces the load on the region server as well, due
195    * to the ordering of the keys. <br>
196    * <br>
197    * Note: It is possible that <code>endKey.length() == 0 </code> , for the last
198    * (recent) region. <br>
199    * Override this method, if you want to bulk exclude regions altogether from
200    * M-R. By default, no region is excluded( i.e. all regions are included).
201    *
202    * @param startKey Start key of the region
203    * @param endKey End key of the region
204    * @return true, if this region needs to be included as part of the input
205    *         (default).
206    */
207   protected boolean includeRegionInSplit(final byte[] startKey,
208       final byte[] endKey) {
209     return true;
210   }
211 
212   /**
213    * Allows subclasses to get the list of {@link Scan} objects.
214    */
215   protected List<Scan> getScans() {
216     return this.scans;
217   }
218 
219   /**
220    * Allows subclasses to set the list of {@link Scan} objects.
221    *
222    * @param scans The list of {@link Scan} used to define the input
223    */
224   protected void setScans(List<Scan> scans) {
225     this.scans = scans;
226   }
227 
228   /**
229    * Allows subclasses to set the {@link TableRecordReader}.
230    *
231    * @param tableRecordReader A different {@link TableRecordReader}
232    *          implementation.
233    */
234   protected void setTableRecordReader(TableRecordReader tableRecordReader) {
235     this.tableRecordReader = tableRecordReader;
236   }
237 }