001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Arrays;
024import java.util.Collections;
025import java.util.List;
026import java.util.NavigableSet;
027import java.util.TreeSet;
028import java.util.UUID;
029import java.util.regex.Matcher;
030import java.util.regex.Pattern;
031import org.apache.commons.lang3.ArrayUtils;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileAlreadyExistsException;
034import org.apache.hadoop.fs.FileStatus;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.fs.PathFilter;
038import org.apache.hadoop.hbase.Cell;
039import org.apache.hadoop.hbase.CellScanner;
040import org.apache.hadoop.hbase.CellUtil;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.Delete;
044import org.apache.hadoop.hbase.client.Durability;
045import org.apache.hadoop.hbase.client.Mutation;
046import org.apache.hadoop.hbase.client.Put;
047import org.apache.hadoop.hbase.client.RegionInfo;
048import org.apache.hadoop.hbase.client.Row;
049import org.apache.hadoop.hbase.regionserver.HRegion;
050import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
051import org.apache.hadoop.hbase.util.Bytes;
052import org.apache.hadoop.hbase.util.ConcurrentMapUtils.IOExceptionSupplier;
053import org.apache.hadoop.hbase.util.FSUtils;
054import org.apache.hadoop.hbase.util.Pair;
055import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
056import org.apache.yetus.audience.InterfaceAudience;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
061
062import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
063import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
064import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
065
066/**
067 * This class provides static methods to support WAL splitting related works
068 */
069@InterfaceAudience.Private
070public final class WALSplitUtil {
071  private static final Logger LOG = LoggerFactory.getLogger(WALSplitUtil.class);
072
073  private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");
074  private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";
075  private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";
076  private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";
077  private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();
078
079  private WALSplitUtil() {
080  }
081
082  /**
083   * Completes the work done by splitLogFile by archiving logs
084   * <p>
085   * It is invoked by SplitLogManager once it knows that one of the SplitLogWorkers have completed
086   * the splitLogFile() part. If the master crashes then this function might get called multiple
087   * times.
088   * <p>
089   * @param logfile
090   * @param conf
091   * @throws IOException
092   */
093  public static void finishSplitLogFile(String logfile, Configuration conf) throws IOException {
094    Path walDir = FSUtils.getWALRootDir(conf);
095    Path oldLogDir = new Path(walDir, HConstants.HREGION_OLDLOGDIR_NAME);
096    Path walPath;
097    if (FSUtils.isStartingWithPath(walDir, logfile)) {
098      walPath = new Path(logfile);
099    } else {
100      walPath = new Path(walDir, logfile);
101    }
102    finishSplitLogFile(walDir, oldLogDir, walPath, conf);
103  }
104
105  static void finishSplitLogFile(Path walDir, Path oldWALDir, Path walPath,
106      Configuration conf) throws IOException {
107    List<Path> processedLogs = new ArrayList<>();
108    List<Path> corruptedLogs = new ArrayList<>();
109    FileSystem walFS = walDir.getFileSystem(conf);
110    if (ZKSplitLog.isCorrupted(walDir, walPath.getName(), walFS)) {
111      corruptedLogs.add(walPath);
112    } else {
113      processedLogs.add(walPath);
114    }
115    archiveWALs(corruptedLogs, processedLogs, oldWALDir, walFS, conf);
116    Path stagingDir = ZKSplitLog.getSplitLogDir(walDir, walPath.getName());
117    walFS.delete(stagingDir, true);
118  }
119
120  /**
121   * Moves processed logs to a oldLogDir after successful processing Moves corrupted logs (any log
122   * that couldn't be successfully parsed to corruptDir (.corrupt) for later investigation
123   */
124  private static void archiveWALs(final List<Path> corruptedWALs, final List<Path> processedWALs,
125      final Path oldWALDir, final FileSystem walFS, final Configuration conf) throws IOException {
126    final Path corruptDir = new Path(FSUtils.getWALRootDir(conf), HConstants.CORRUPT_DIR_NAME);
127    if (conf.get("hbase.regionserver.hlog.splitlog.corrupt.dir") != null) {
128      LOG.warn("hbase.regionserver.hlog.splitlog.corrupt.dir is deprecated. Default to {}",
129        corruptDir);
130    }
131    if (!walFS.mkdirs(corruptDir)) {
132      LOG.info("Unable to mkdir {}", corruptDir);
133    }
134    walFS.mkdirs(oldWALDir);
135
136    // this method can get restarted or called multiple times for archiving
137    // the same log files.
138    for (Path corruptedWAL : corruptedWALs) {
139      Path p = new Path(corruptDir, corruptedWAL.getName());
140      if (walFS.exists(corruptedWAL)) {
141        if (!walFS.rename(corruptedWAL, p)) {
142          LOG.warn("Unable to move corrupted log {} to {}", corruptedWAL, p);
143        } else {
144          LOG.warn("Moved corrupted log {} to {}", corruptedWAL, p);
145        }
146      }
147    }
148
149    for (Path p : processedWALs) {
150      Path newPath = AbstractFSWAL.getWALArchivePath(oldWALDir, p);
151      if (walFS.exists(p)) {
152        if (!FSUtils.renameAndSetModifyTime(walFS, p, newPath)) {
153          LOG.warn("Unable to move {} to {}", p, newPath);
154        } else {
155          LOG.info("Archived processed log {} to {}", p, newPath);
156        }
157      }
158    }
159  }
160
161  /**
162   * Path to a file under RECOVERED_EDITS_DIR directory of the region found in <code>logEntry</code>
163   * named for the sequenceid in the passed <code>logEntry</code>: e.g.
164   * /hbase/some_table/2323432434/recovered.edits/2332. This method also ensures existence of
165   * RECOVERED_EDITS_DIR under the region creating it if necessary.
166   * @param tableName the table name
167   * @param encodedRegionName the encoded region name
168   * @param sedId the sequence id which used to generate file name
169   * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.
170   * @param tmpDirName of the directory used to sideline old recovered edits file
171   * @param conf configuration
172   * @return Path to file into which to dump split log edits.
173   * @throws IOException
174   */
175  @SuppressWarnings("deprecation")
176  @VisibleForTesting
177  static Path getRegionSplitEditsPath(TableName tableName, byte[] encodedRegionName, long sedId,
178      String fileNameBeingSplit, String tmpDirName, Configuration conf) throws IOException {
179    FileSystem walFS = FSUtils.getWALFileSystem(conf);
180    Path tableDir = FSUtils.getWALTableDir(conf, tableName);
181    String encodedRegionNameStr = Bytes.toString(encodedRegionName);
182    Path regionDir = HRegion.getRegionDir(tableDir, encodedRegionNameStr);
183    Path dir = getRegionDirRecoveredEditsDir(regionDir);
184
185    if (walFS.exists(dir) && walFS.isFile(dir)) {
186      Path tmp = new Path(tmpDirName);
187      if (!walFS.exists(tmp)) {
188        walFS.mkdirs(tmp);
189      }
190      tmp = new Path(tmp, HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionNameStr);
191      LOG.warn("Found existing old file: {}. It could be some "
192          + "leftover of an old installation. It should be a folder instead. "
193          + "So moving it to {}",
194        dir, tmp);
195      if (!walFS.rename(dir, tmp)) {
196        LOG.warn("Failed to sideline old file {}", dir);
197      }
198    }
199
200    if (!walFS.exists(dir) && !walFS.mkdirs(dir)) {
201      LOG.warn("mkdir failed on {}", dir);
202    }
203    // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.
204    // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure
205    // region's replayRecoveredEdits will not delete it
206    String fileName = formatRecoveredEditsFileName(sedId);
207    fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);
208    return new Path(dir, fileName);
209  }
210
211  private static String getTmpRecoveredEditsFileName(String fileName) {
212    return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;
213  }
214
215  /**
216   * Get the completed recovered edits file path, renaming it to be by last edit in the file from
217   * its first edit. Then we could use the name to skip recovered edits when doing
218   * {@link HRegion#replayRecoveredEditsIfAny}.
219   * @return dstPath take file's last edit log seq num as the name
220   */
221  static Path getCompletedRecoveredEditsFilePath(Path srcPath, long maximumEditWALSeqNum) {
222    String fileName = formatRecoveredEditsFileName(maximumEditWALSeqNum);
223    return new Path(srcPath.getParent(), fileName);
224  }
225
226  @VisibleForTesting
227  static String formatRecoveredEditsFileName(final long seqid) {
228    return String.format("%019d", seqid);
229  }
230
231  /**
232   * @param regionDir This regions directory in the filesystem.
233   * @return The directory that holds recovered edits files for the region <code>regionDir</code>
234   */
235  public static Path getRegionDirRecoveredEditsDir(final Path regionDir) {
236    return new Path(regionDir, HConstants.RECOVERED_EDITS_DIR);
237  }
238
239  /**
240   * Check whether there is recovered.edits in the region dir
241   * @param conf conf
242   * @param regionInfo the region to check
243   * @return true if recovered.edits exist in the region dir
244   */
245  public static boolean hasRecoveredEdits(final Configuration conf, final RegionInfo regionInfo)
246      throws IOException {
247    // No recovered.edits for non default replica regions
248    if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
249      return false;
250    }
251    // Only default replica region can reach here, so we can use regioninfo
252    // directly without converting it to default replica's regioninfo.
253    Path regionWALDir =
254        FSUtils.getWALRegionDir(conf, regionInfo.getTable(), regionInfo.getEncodedName());
255    Path regionDir = FSUtils.getRegionDirFromRootDir(FSUtils.getRootDir(conf), regionInfo);
256    Path wrongRegionWALDir =
257        FSUtils.getWrongWALRegionDir(conf, regionInfo.getTable(), regionInfo.getEncodedName());
258    FileSystem walFs = FSUtils.getWALFileSystem(conf);
259    FileSystem rootFs = FSUtils.getRootDirFileSystem(conf);
260    NavigableSet<Path> files = getSplitEditFilesSorted(walFs, regionWALDir);
261    if (!files.isEmpty()) {
262      return true;
263    }
264    files = getSplitEditFilesSorted(rootFs, regionDir);
265    if (!files.isEmpty()) {
266      return true;
267    }
268    files = getSplitEditFilesSorted(walFs, wrongRegionWALDir);
269    return !files.isEmpty();
270  }
271
272  /**
273   * This method will check 3 places for finding the max sequence id file. One is the expected
274   * place, another is the old place under the region directory, and the last one is the wrong one
275   * we introduced in HBASE-20734. See HBASE-22617 for more details.
276   * <p/>
277   * Notice that, you should always call this method instead of
278   * {@link #getMaxRegionSequenceId(FileSystem, Path)} until 4.0.0 release.
279   * @deprecated Only for compatibility, will be removed in 4.0.0.
280   */
281  @Deprecated
282  public static long getMaxRegionSequenceId(Configuration conf, RegionInfo region,
283      IOExceptionSupplier<FileSystem> rootFsSupplier, IOExceptionSupplier<FileSystem> walFsSupplier)
284      throws IOException {
285    FileSystem rootFs = rootFsSupplier.get();
286    FileSystem walFs = walFsSupplier.get();
287    Path regionWALDir = FSUtils.getWALRegionDir(conf, region.getTable(), region.getEncodedName());
288    // This is the old place where we store max sequence id file
289    Path regionDir = FSUtils.getRegionDirFromRootDir(FSUtils.getRootDir(conf), region);
290    // This is for HBASE-20734, where we use a wrong directory, see HBASE-22617 for more details.
291    Path wrongRegionWALDir =
292      FSUtils.getWrongWALRegionDir(conf, region.getTable(), region.getEncodedName());
293    long maxSeqId = getMaxRegionSequenceId(walFs, regionWALDir);
294    maxSeqId = Math.max(maxSeqId, getMaxRegionSequenceId(rootFs, regionDir));
295    maxSeqId = Math.max(maxSeqId, getMaxRegionSequenceId(walFs, wrongRegionWALDir));
296    return maxSeqId;
297  }
298
299  /**
300   * Returns sorted set of edit files made by splitter, excluding files with '.temp' suffix.
301   * @param walFS WAL FileSystem used to retrieving split edits files.
302   * @param regionDir WAL region dir to look for recovered edits files under.
303   * @return Files in passed <code>regionDir</code> as a sorted set.
304   * @throws IOException
305   */
306  public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem walFS,
307      final Path regionDir) throws IOException {
308    NavigableSet<Path> filesSorted = new TreeSet<>();
309    Path editsdir = getRegionDirRecoveredEditsDir(regionDir);
310    if (!walFS.exists(editsdir)) {
311      return filesSorted;
312    }
313    FileStatus[] files = FSUtils.listStatus(walFS, editsdir, new PathFilter() {
314      @Override
315      public boolean accept(Path p) {
316        boolean result = false;
317        try {
318          // Return files and only files that match the editfile names pattern.
319          // There can be other files in this directory other than edit files.
320          // In particular, on error, we'll move aside the bad edit file giving
321          // it a timestamp suffix. See moveAsideBadEditsFile.
322          Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());
323          result = walFS.isFile(p) && m.matches();
324          // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
325          // because it means splitwal thread is writting this file.
326          if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {
327            result = false;
328          }
329          // Skip SeqId Files
330          if (isSequenceIdFile(p)) {
331            result = false;
332          }
333        } catch (IOException e) {
334          LOG.warn("Failed isFile check on {}", p, e);
335        }
336        return result;
337      }
338    });
339    if (ArrayUtils.isNotEmpty(files)) {
340      Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));
341    }
342    return filesSorted;
343  }
344
345  /**
346   * Move aside a bad edits file.
347   * @param walFS WAL FileSystem used to rename bad edits file.
348   * @param edits Edits file to move aside.
349   * @return The name of the moved aside file.
350   * @throws IOException
351   */
352  public static Path moveAsideBadEditsFile(final FileSystem walFS, final Path edits)
353      throws IOException {
354    Path moveAsideName =
355        new Path(edits.getParent(), edits.getName() + "." + System.currentTimeMillis());
356    if (!walFS.rename(edits, moveAsideName)) {
357      LOG.warn("Rename failed from {} to {}", edits, moveAsideName);
358    }
359    return moveAsideName;
360  }
361
362  /**
363   * Is the given file a region open sequence id file.
364   */
365  @VisibleForTesting
366  public static boolean isSequenceIdFile(final Path file) {
367    return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)
368        || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);
369  }
370
371  private static FileStatus[] getSequenceIdFiles(FileSystem walFS, Path regionDir)
372      throws IOException {
373    // TODO: Why are we using a method in here as part of our normal region open where
374    // there is no splitting involved? Fix. St.Ack 01/20/2017.
375    Path editsDir = getRegionDirRecoveredEditsDir(regionDir);
376    try {
377      FileStatus[] files = walFS.listStatus(editsDir, WALSplitUtil::isSequenceIdFile);
378      return files != null ? files : new FileStatus[0];
379    } catch (FileNotFoundException e) {
380      return new FileStatus[0];
381    }
382  }
383
384  private static long getMaxSequenceId(FileStatus[] files) {
385    long maxSeqId = -1L;
386    for (FileStatus file : files) {
387      String fileName = file.getPath().getName();
388      try {
389        maxSeqId = Math.max(maxSeqId, Long
390            .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));
391      } catch (NumberFormatException ex) {
392        LOG.warn("Invalid SeqId File Name={}", fileName);
393      }
394    }
395    return maxSeqId;
396  }
397
398  /**
399   * Get the max sequence id which is stored in the region directory. -1 if none.
400   */
401  public static long getMaxRegionSequenceId(FileSystem walFS, Path regionDir) throws IOException {
402    return getMaxSequenceId(getSequenceIdFiles(walFS, regionDir));
403  }
404
405  /**
406   * Create a file with name as region's max sequence id
407   */
408  public static void writeRegionSequenceIdFile(FileSystem walFS, Path regionDir, long newMaxSeqId)
409      throws IOException {
410    FileStatus[] files = getSequenceIdFiles(walFS, regionDir);
411    long maxSeqId = getMaxSequenceId(files);
412    if (maxSeqId > newMaxSeqId) {
413      throw new IOException("The new max sequence id " + newMaxSeqId
414          + " is less than the old max sequence id " + maxSeqId);
415    }
416    // write a new seqId file
417    Path newSeqIdFile =
418        new Path(getRegionDirRecoveredEditsDir(regionDir), newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);
419    if (newMaxSeqId != maxSeqId) {
420      try {
421        if (!walFS.createNewFile(newSeqIdFile) && !walFS.exists(newSeqIdFile)) {
422          throw new IOException("Failed to create SeqId file:" + newSeqIdFile);
423        }
424        LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,
425          maxSeqId);
426      } catch (FileAlreadyExistsException ignored) {
427        // latest hdfs throws this exception. it's all right if newSeqIdFile already exists
428      }
429    }
430    // remove old ones
431    for (FileStatus status : files) {
432      if (!newSeqIdFile.equals(status.getPath())) {
433        walFS.delete(status.getPath(), false);
434      }
435    }
436  }
437
438  /** A struct used by getMutationsFromWALEntry */
439  public static class MutationReplay implements Comparable<MutationReplay> {
440    public MutationReplay(ClientProtos.MutationProto.MutationType type, Mutation mutation,
441        long nonceGroup, long nonce) {
442      this.type = type;
443      this.mutation = mutation;
444      if (this.mutation.getDurability() != Durability.SKIP_WAL) {
445        // using ASYNC_WAL for relay
446        this.mutation.setDurability(Durability.ASYNC_WAL);
447      }
448      this.nonceGroup = nonceGroup;
449      this.nonce = nonce;
450    }
451
452    private final ClientProtos.MutationProto.MutationType type;
453    public final Mutation mutation;
454    public final long nonceGroup;
455    public final long nonce;
456
457    @Override
458    public int compareTo(final MutationReplay d) {
459      return Row.COMPARATOR.compare(mutation, d.mutation);
460    }
461
462    @Override
463    public boolean equals(Object obj) {
464      if (!(obj instanceof MutationReplay)) {
465        return false;
466      } else {
467        return this.compareTo((MutationReplay) obj) == 0;
468      }
469    }
470
471    @Override
472    public int hashCode() {
473      return this.mutation.hashCode();
474    }
475
476    public ClientProtos.MutationProto.MutationType getType() {
477      return type;
478    }
479  }
480
481  /**
482   * This function is used to construct mutations from a WALEntry. It also reconstructs WALKey &amp;
483   * WALEdit from the passed in WALEntry
484   * @param entry
485   * @param cells
486   * @param logEntry pair of WALKey and WALEdit instance stores WALKey and WALEdit instances
487   *          extracted from the passed in WALEntry.
488   * @return list of Pair&lt;MutationType, Mutation&gt; to be replayed
489   * @throws IOException
490   */
491  public static List<MutationReplay> getMutationsFromWALEntry(AdminProtos.WALEntry entry,
492      CellScanner cells, Pair<WALKey, WALEdit> logEntry, Durability durability) throws IOException {
493    if (entry == null) {
494      // return an empty array
495      return Collections.emptyList();
496    }
497
498    long replaySeqId =
499        (entry.getKey().hasOrigSequenceNumber()) ? entry.getKey().getOrigSequenceNumber()
500            : entry.getKey().getLogSequenceNumber();
501    int count = entry.getAssociatedCellCount();
502    List<MutationReplay> mutations = new ArrayList<>();
503    Cell previousCell = null;
504    Mutation m = null;
505    WALKeyImpl key = null;
506    WALEdit val = null;
507    if (logEntry != null) {
508      val = new WALEdit();
509    }
510
511    for (int i = 0; i < count; i++) {
512      // Throw index out of bounds if our cell count is off
513      if (!cells.advance()) {
514        throw new ArrayIndexOutOfBoundsException("Expected=" + count + ", index=" + i);
515      }
516      Cell cell = cells.current();
517      if (val != null) val.add(cell);
518
519      boolean isNewRowOrType =
520          previousCell == null || previousCell.getTypeByte() != cell.getTypeByte()
521              || !CellUtil.matchingRows(previousCell, cell);
522      if (isNewRowOrType) {
523        // Create new mutation
524        if (CellUtil.isDelete(cell)) {
525          m = new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
526          // Deletes don't have nonces.
527          mutations.add(new MutationReplay(ClientProtos.MutationProto.MutationType.DELETE, m,
528              HConstants.NO_NONCE, HConstants.NO_NONCE));
529        } else {
530          m = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
531          // Puts might come from increment or append, thus we need nonces.
532          long nonceGroup =
533              entry.getKey().hasNonceGroup() ? entry.getKey().getNonceGroup() : HConstants.NO_NONCE;
534          long nonce = entry.getKey().hasNonce() ? entry.getKey().getNonce() : HConstants.NO_NONCE;
535          mutations.add(
536            new MutationReplay(ClientProtos.MutationProto.MutationType.PUT, m, nonceGroup, nonce));
537        }
538      }
539      if (CellUtil.isDelete(cell)) {
540        ((Delete) m).add(cell);
541      } else {
542        ((Put) m).add(cell);
543      }
544      m.setDurability(durability);
545      previousCell = cell;
546    }
547
548    // reconstruct WALKey
549    if (logEntry != null) {
550      org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALKey walKeyProto =
551          entry.getKey();
552      List<UUID> clusterIds = new ArrayList<>(walKeyProto.getClusterIdsCount());
553      for (HBaseProtos.UUID uuid : entry.getKey().getClusterIdsList()) {
554        clusterIds.add(new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits()));
555      }
556      key = new WALKeyImpl(walKeyProto.getEncodedRegionName().toByteArray(),
557          TableName.valueOf(walKeyProto.getTableName().toByteArray()), replaySeqId,
558          walKeyProto.getWriteTime(), clusterIds, walKeyProto.getNonceGroup(),
559          walKeyProto.getNonce(), null);
560      logEntry.setFirst(key);
561      logEntry.setSecond(val);
562    }
563
564    return mutations;
565  }
566}