View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.util.Map;
24  import java.util.TreeMap;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.io.HLogLink;
33  import org.apache.hadoop.hbase.regionserver.HRegion;
34  import org.apache.hadoop.hbase.regionserver.wal.HLog;
35  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
36  import org.apache.hadoop.hbase.util.Bytes;
37  
38  /**
39   * If the snapshot has references to one or more log files,
40   * those must be split (each log contains multiple tables and regions)
41   * and must be placed in the region/recovered.edits folder.
42   * (recovered.edits files will be played on region startup)
43   *
44   * In case of Restore: the log can just be split in the recovered.edits folder.
45   * In case of Clone: each entry in the log must be modified to use the new region name.
46   * (region names are encoded with: tableName, startKey, regionIdTimeStamp)
47   *
48   * We can't use the normal split code, because the HLogKey contains the
49   * table name and the region name, and in case of "clone from snapshot"
50   * region name and table name will be different and must be replaced in
51   * the recovered.edits.
52   */
53  @InterfaceAudience.Private
54  class SnapshotLogSplitter implements Closeable {
55    static final Log LOG = LogFactory.getLog(SnapshotLogSplitter.class);
56  
57    private final class LogWriter implements Closeable {
58      private HLog.Writer writer;
59      private Path logFile;
60      private long seqId;
61  
62      public LogWriter(final Configuration conf, final FileSystem fs,
63          final Path logDir, long seqId) throws IOException {
64        logFile = new Path(logDir, logFileName(seqId, true));
65        this.writer = HLog.createWriter(fs, logFile, conf);
66        this.seqId = seqId;
67      }
68  
69      public void close() throws IOException {
70        writer.close();
71  
72        Path finalFile = new Path(logFile.getParent(), logFileName(seqId, false));
73        LOG.debug("LogWriter tmpLogFile=" + logFile + " -> logFile=" + finalFile);
74        fs.rename(logFile, finalFile);
75      }
76  
77      public void append(final HLog.Entry entry) throws IOException {
78        writer.append(entry);
79        if (seqId < entry.getKey().getLogSeqNum()) {
80          seqId = entry.getKey().getLogSeqNum();
81        }
82      }
83  
84      private String logFileName(long seqId, boolean temp) {
85        String fileName = String.format("%019d", seqId);
86        if (temp) fileName += HLog.RECOVERED_LOG_TMPFILE_SUFFIX;
87        return fileName;
88      }
89    }
90  
91    private final Map<byte[], LogWriter> regionLogWriters =
92        new TreeMap<byte[], LogWriter>(Bytes.BYTES_COMPARATOR);
93  
94    private final Map<byte[], byte[]> regionsMap;
95    private final Configuration conf;
96    private final byte[] snapshotTableName;
97    private final byte[] tableName;
98    private final Path tableDir;
99    private final FileSystem fs;
100 
101   /**
102    * @params tableName snapshot table name
103    * @params regionsMap maps original region names to the new ones.
104    */
105   public SnapshotLogSplitter(final Configuration conf, final FileSystem fs,
106       final Path tableDir, final byte[] snapshotTableName,
107       final Map<byte[], byte[]> regionsMap) {
108     this.regionsMap = regionsMap;
109     this.snapshotTableName = snapshotTableName;
110     this.tableName = Bytes.toBytes(tableDir.getName());
111     this.tableDir = tableDir;
112     this.conf = conf;
113     this.fs = fs;
114   }
115 
116   public void close() throws IOException {
117     for (LogWriter writer: regionLogWriters.values()) {
118       writer.close();
119     }
120   }
121 
122   public void splitLog(final String serverName, final String logfile) throws IOException {
123     LOG.debug("Restore log=" + logfile + " server=" + serverName +
124               " for snapshotTable=" + Bytes.toString(snapshotTableName) +
125               " to table=" + Bytes.toString(tableName));
126     splitLog(new HLogLink(conf, serverName, logfile).getAvailablePath(fs));
127   }
128 
129   public void splitRecoveredEdit(final Path editPath) throws IOException {
130     LOG.debug("Restore recover.edits=" + editPath +
131               " for snapshotTable=" + Bytes.toString(snapshotTableName) +
132               " to table=" + Bytes.toString(tableName));
133     splitLog(editPath);
134   }
135 
136   /**
137    * Split the snapshot HLog reference into regions recovered.edits.
138    *
139    * The HLogKey contains the table name and the region name,
140    * and they must be changed to the restored table names.
141    *
142    * @param logPath Snapshot HLog reference path
143    */
144   public void splitLog(final Path logPath) throws IOException {
145     HLog.Reader log = HLog.getReader(fs, logPath, conf);
146     try {
147       HLog.Entry entry;
148       LogWriter writer = null;
149       byte[] regionName = null;
150       byte[] newRegionName = null;
151       while ((entry = log.next()) != null) {
152         HLogKey key = entry.getKey();
153 
154         // We're interested only in the snapshot table that we're restoring
155         if (!Bytes.equals(key.getTablename(), snapshotTableName)) continue;
156 
157         // Writer for region.
158         if (!Bytes.equals(regionName, key.getEncodedRegionName())) {
159           regionName = key.getEncodedRegionName().clone();
160 
161           // Get the new region name in case of clone, or use the original one
162           newRegionName = regionsMap.get(regionName);
163           if (newRegionName == null) newRegionName = regionName;
164 
165           writer = getOrCreateWriter(newRegionName, key.getLogSeqNum());
166           LOG.debug("+ regionName=" + Bytes.toString(regionName));
167         }
168 
169         // Append Entry
170         key = new HLogKey(newRegionName, tableName,
171                           key.getLogSeqNum(), key.getWriteTime(), key.getClusterId());
172         writer.append(new HLog.Entry(key, entry.getEdit()));
173       }
174     } catch (IOException e) {
175       LOG.warn("Something wrong during the log split", e);
176     } finally {
177       log.close();
178     }
179   }
180 
181   /**
182    * Create a LogWriter for specified region if not already created.
183    */
184   private LogWriter getOrCreateWriter(final byte[] regionName, long seqId) throws IOException {
185     LogWriter writer = regionLogWriters.get(regionName);
186     if (writer == null) {
187       Path regionDir = HRegion.getRegionDir(tableDir, Bytes.toString(regionName));
188       Path dir = HLog.getRegionDirRecoveredEditsDir(regionDir);
189       fs.mkdirs(dir);
190 
191       writer = new LogWriter(conf, fs, dir, seqId);
192       regionLogWriters.put(regionName, writer);
193     }
194     return(writer);
195   }
196 }