Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.replication;
019
020import static org.junit.Assert.assertNull;
021import static org.junit.Assert.assertTrue;
022import static org.mockito.Mockito.mock;
023import static org.mockito.Mockito.when;
024
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.TreeMap;
032import org.apache.hadoop.hbase.Cell;
033import org.apache.hadoop.hbase.CellComparatorImpl;
034import org.apache.hadoop.hbase.HBaseClassTestRule;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.KeyValue;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.testclassification.ReplicationTests;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.apache.hadoop.hbase.wal.WAL.Entry;
044import org.apache.hadoop.hbase.wal.WALEdit;
045import org.apache.hadoop.hbase.wal.WALKeyImpl;
046import org.junit.Assert;
047import org.junit.ClassRule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050
051import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
052
053@Category({ ReplicationTests.class, SmallTests.class })
054public class TestReplicationWALEntryFilters {
055
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058    HBaseClassTestRule.forClass(TestReplicationWALEntryFilters.class);
059
060  static byte[] a = new byte[] { 'a' };
061  static byte[] b = new byte[] { 'b' };
062  static byte[] c = new byte[] { 'c' };
063  static byte[] d = new byte[] { 'd' };
064
065  @Test
066  public void testSystemTableWALEntryFilter() {
067    SystemTableWALEntryFilter filter = new SystemTableWALEntryFilter();
068
069    // meta
070    WALKeyImpl key1 =
071      new WALKeyImpl(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
072        TableName.META_TABLE_NAME, EnvironmentEdgeManager.currentTime());
073    Entry metaEntry = new Entry(key1, null);
074
075    assertNull(filter.filter(metaEntry));
076
077    // user table
078    WALKeyImpl key3 =
079      new WALKeyImpl(new byte[0], TableName.valueOf("foo"), EnvironmentEdgeManager.currentTime());
080    Entry userEntry = new Entry(key3, null);
081
082    assertEquals(userEntry, filter.filter(userEntry));
083  }
084
085  @Test
086  public void testScopeWALEntryFilter() {
087    WALEntryFilter filter = new ChainWALEntryFilter(new ScopeWALEntryFilter());
088
089    Entry userEntry = createEntry(null, a, b);
090    Entry userEntryA = createEntry(null, a);
091    Entry userEntryB = createEntry(null, b);
092    Entry userEntryEmpty = createEntry(null);
093
094    // no scopes
095    // now we will not filter out entries without a replication scope since serial replication still
096    // need the sequence id, but the cells will all be filtered out.
097    assertTrue(filter.filter(userEntry).getEdit().isEmpty());
098
099    // empty scopes
100    // ditto
101    TreeMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
102    userEntry = createEntry(scopes, a, b);
103    assertTrue(filter.filter(userEntry).getEdit().isEmpty());
104
105    // different scope
106    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
107    scopes.put(c, HConstants.REPLICATION_SCOPE_GLOBAL);
108    userEntry = createEntry(scopes, a, b);
109    // all kvs should be filtered
110    assertEquals(userEntryEmpty, filter.filter(userEntry));
111
112    // local scope
113    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
114    scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
115    userEntry = createEntry(scopes, a, b);
116    assertEquals(userEntryEmpty, filter.filter(userEntry));
117    scopes.put(b, HConstants.REPLICATION_SCOPE_LOCAL);
118    assertEquals(userEntryEmpty, filter.filter(userEntry));
119
120    // only scope a
121    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
122    scopes.put(a, HConstants.REPLICATION_SCOPE_GLOBAL);
123    userEntry = createEntry(scopes, a, b);
124    assertEquals(userEntryA, filter.filter(userEntry));
125    scopes.put(b, HConstants.REPLICATION_SCOPE_LOCAL);
126    assertEquals(userEntryA, filter.filter(userEntry));
127
128    // only scope b
129    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
130    scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
131    userEntry = createEntry(scopes, a, b);
132    assertEquals(userEntryB, filter.filter(userEntry));
133    scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
134    assertEquals(userEntryB, filter.filter(userEntry));
135
136    // scope a and b
137    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
138    scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
139    userEntry = createEntry(scopes, a, b);
140    assertEquals(userEntryB, filter.filter(userEntry));
141    scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
142    assertEquals(userEntryB, filter.filter(userEntry));
143  }
144
145  WALEntryFilter nullFilter = new WALEntryFilter() {
146    @Override
147    public Entry filter(Entry entry) {
148      return null;
149    }
150  };
151
152  WALEntryFilter passFilter = new WALEntryFilter() {
153    @Override
154    public Entry filter(Entry entry) {
155      return entry;
156    }
157  };
158
159  public static class FilterSomeCellsWALCellFilter implements WALEntryFilter, WALCellFilter {
160    @Override
161    public Entry filter(Entry entry) {
162      return entry;
163    }
164
165    @Override
166    public Cell filterCell(Entry entry, Cell cell) {
167      if (
168        Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()).equals("a")
169      ) {
170        return null;
171      } else {
172        return cell;
173      }
174    }
175  }
176
177  public static class FilterAllCellsWALCellFilter implements WALEntryFilter, WALCellFilter {
178    @Override
179    public Entry filter(Entry entry) {
180      return entry;
181    }
182
183    @Override
184    public Cell filterCell(Entry entry, Cell cell) {
185      return null;
186    }
187  }
188
189  @Test
190  public void testChainWALEntryWithCellFilter() {
191    Entry userEntry = createEntry(null, a, b, c);
192    ChainWALEntryFilter filterSomeCells =
193      new ChainWALEntryFilter(new FilterSomeCellsWALCellFilter());
194    // since WALCellFilter filter cells with rowkey 'a'
195    assertEquals(createEntry(null, b, c), filterSomeCells.filter(userEntry));
196
197    Entry userEntry2 = createEntry(null, b, c, d);
198    // since there is no cell to get filtered, nothing should get filtered
199    assertEquals(userEntry2, filterSomeCells.filter(userEntry2));
200
201    // since we filter all the cells, we should get empty entry
202    ChainWALEntryFilter filterAllCells = new ChainWALEntryFilter(new FilterAllCellsWALCellFilter());
203    assertEquals(createEntry(null), filterAllCells.filter(userEntry));
204  }
205
206  @Test
207  public void testChainWALEmptyEntryWithCellFilter() {
208    Entry userEntry = createEntry(null, a, b, c);
209    ChainWALEmptyEntryFilter filterSomeCells =
210      new ChainWALEmptyEntryFilter(new FilterSomeCellsWALCellFilter());
211    // since WALCellFilter filter cells with rowkey 'a'
212    assertEquals(createEntry(null, b, c), filterSomeCells.filter(userEntry));
213
214    Entry userEntry2 = createEntry(null, b, c, d);
215    // since there is no cell to get filtered, nothing should get filtered
216    assertEquals(userEntry2, filterSomeCells.filter(userEntry2));
217
218    ChainWALEmptyEntryFilter filterAllCells =
219      new ChainWALEmptyEntryFilter(new FilterAllCellsWALCellFilter());
220    assertEquals(createEntry(null), filterAllCells.filter(userEntry));
221    // let's set the filter empty entry flag to true now for the above case
222    filterAllCells.setFilterEmptyEntry(true);
223    // since WALCellFilter filter all cells, whole entry should be filtered
224    assertEquals(null, filterAllCells.filter(userEntry));
225  }
226
227  @Test
228  public void testChainWALEntryFilter() {
229    Entry userEntry = createEntry(null, a, b, c);
230
231    ChainWALEntryFilter filter = new ChainWALEntryFilter(passFilter);
232    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
233
234    filter = new ChainWALEntryFilter(passFilter, passFilter);
235    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
236
237    filter = new ChainWALEntryFilter(passFilter, passFilter, passFilter);
238    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
239
240    filter = new ChainWALEntryFilter(nullFilter);
241    assertEquals(null, filter.filter(userEntry));
242
243    filter = new ChainWALEntryFilter(nullFilter, passFilter);
244    assertEquals(null, filter.filter(userEntry));
245
246    filter = new ChainWALEntryFilter(passFilter, nullFilter);
247    assertEquals(null, filter.filter(userEntry));
248
249    filter = new ChainWALEntryFilter(nullFilter, passFilter, nullFilter);
250    assertEquals(null, filter.filter(userEntry));
251
252    filter = new ChainWALEntryFilter(nullFilter, nullFilter);
253    assertEquals(null, filter.filter(userEntry));
254
255    // flatten
256    filter = new ChainWALEntryFilter(
257      new ChainWALEntryFilter(passFilter, new ChainWALEntryFilter(passFilter, passFilter),
258        new ChainWALEntryFilter(passFilter), new ChainWALEntryFilter(passFilter)),
259      new ChainWALEntryFilter(passFilter));
260    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
261
262    filter = new ChainWALEntryFilter(
263      new ChainWALEntryFilter(passFilter,
264        new ChainWALEntryFilter(passFilter, new ChainWALEntryFilter(nullFilter))),
265      new ChainWALEntryFilter(passFilter));
266    assertEquals(null, filter.filter(userEntry));
267  }
268
269  @Test
270  public void testNamespaceTableCfWALEntryFilter() {
271    ReplicationPeer peer = mock(ReplicationPeer.class);
272    ReplicationPeerConfigBuilder peerConfigBuilder = ReplicationPeerConfig.newBuilder();
273
274    // 1. replicate_all flag is false, no namespaces and table-cfs config
275    peerConfigBuilder.setReplicateAllUserTables(false).setNamespaces(null).setTableCFsMap(null);
276    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
277    Entry userEntry = createEntry(null, a, b, c);
278    ChainWALEntryFilter filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
279    assertEquals(null, filter.filter(userEntry));
280
281    // 2. replicate_all flag is false, and only config table-cfs in peer
282    // empty map
283    userEntry = createEntry(null, a, b, c);
284    Map<TableName, List<String>> tableCfs = new HashMap<>();
285    peerConfigBuilder.setReplicateAllUserTables(false).setTableCFsMap(tableCfs);
286    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
287    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
288    assertEquals(null, filter.filter(userEntry));
289
290    // table bar
291    userEntry = createEntry(null, a, b, c);
292    tableCfs = new HashMap<>();
293    tableCfs.put(TableName.valueOf("bar"), null);
294    peerConfigBuilder.setReplicateAllUserTables(false).setTableCFsMap(tableCfs);
295    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
296    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
297    assertEquals(null, filter.filter(userEntry));
298
299    // table foo:a
300    userEntry = createEntry(null, a, b, c);
301    tableCfs = new HashMap<>();
302    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a"));
303    peerConfigBuilder.setReplicateAllUserTables(false).setTableCFsMap(tableCfs);
304    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
305    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
306    assertEquals(createEntry(null, a), filter.filter(userEntry));
307
308    // table foo:a,c
309    userEntry = createEntry(null, a, b, c, d);
310    tableCfs = new HashMap<>();
311    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a", "c"));
312    peerConfigBuilder.setReplicateAllUserTables(false).setTableCFsMap(tableCfs);
313    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
314    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
315    assertEquals(createEntry(null, a, c), filter.filter(userEntry));
316
317    // 3. replicate_all flag is false, and only config namespaces in peer
318    when(peer.getTableCFs()).thenReturn(null);
319    // empty set
320    Set<String> namespaces = new HashSet<>();
321    peerConfigBuilder.setReplicateAllUserTables(false).setNamespaces(namespaces)
322      .setTableCFsMap(null);
323    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
324    userEntry = createEntry(null, a, b, c);
325    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
326    assertEquals(null, filter.filter(userEntry));
327
328    // namespace default
329    namespaces.add("default");
330    peerConfigBuilder.setReplicateAllUserTables(false).setNamespaces(namespaces);
331    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
332    userEntry = createEntry(null, a, b, c);
333    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
334    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
335
336    // namespace ns1
337    namespaces = new HashSet<>();
338    namespaces.add("ns1");
339    peerConfigBuilder.setReplicateAllUserTables(false).setNamespaces(namespaces);
340    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
341    userEntry = createEntry(null, a, b, c);
342    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
343    assertEquals(null, filter.filter(userEntry));
344
345    // 4. replicate_all flag is false, and config namespaces and table-cfs both
346    // Namespaces config should not confict with table-cfs config
347    namespaces = new HashSet<>();
348    tableCfs = new HashMap<>();
349    namespaces.add("ns1");
350    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a", "c"));
351    peerConfigBuilder.setReplicateAllUserTables(false).setNamespaces(namespaces)
352      .setTableCFsMap(tableCfs);
353    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
354    userEntry = createEntry(null, a, b, c);
355    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
356    assertEquals(createEntry(null, a, c), filter.filter(userEntry));
357
358    namespaces = new HashSet<>();
359    tableCfs = new HashMap<>();
360    namespaces.add("default");
361    tableCfs.put(TableName.valueOf("ns1:foo"), Lists.newArrayList("a", "c"));
362    peerConfigBuilder.setReplicateAllUserTables(false).setNamespaces(namespaces)
363      .setTableCFsMap(tableCfs);
364    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
365    userEntry = createEntry(null, a, b, c);
366    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
367    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
368
369    namespaces = new HashSet<>();
370    tableCfs = new HashMap<>();
371    namespaces.add("ns1");
372    tableCfs.put(TableName.valueOf("bar"), null);
373    peerConfigBuilder.setReplicateAllUserTables(false).setNamespaces(namespaces)
374      .setTableCFsMap(tableCfs);
375    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
376    userEntry = createEntry(null, a, b, c);
377    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
378    assertEquals(null, filter.filter(userEntry));
379  }
380
381  @Test
382  public void testNamespaceTableCfWALEntryFilter2() {
383    ReplicationPeer peer = mock(ReplicationPeer.class);
384    ReplicationPeerConfigBuilder peerConfigBuilder = ReplicationPeerConfig.newBuilder();
385
386    // 1. replicate_all flag is true
387    // and no exclude namespaces and no exclude table-cfs config
388    peerConfigBuilder.setReplicateAllUserTables(true).setExcludeNamespaces(null)
389      .setExcludeTableCFsMap(null);
390    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
391    Entry userEntry = createEntry(null, a, b, c);
392    ChainWALEntryFilter filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
393    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
394
395    // 2. replicate_all flag is true, and only config exclude namespaces
396    // empty set
397    Set<String> namespaces = new HashSet<String>();
398    peerConfigBuilder.setExcludeNamespaces(namespaces).setExcludeTableCFsMap(null);
399    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
400    userEntry = createEntry(null, a, b, c);
401    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
402    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
403
404    // exclude namespace default
405    namespaces.add("default");
406    peerConfigBuilder.setExcludeNamespaces(namespaces).setExcludeTableCFsMap(null);
407    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
408    userEntry = createEntry(null, a, b, c);
409    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
410    assertEquals(null, filter.filter(userEntry));
411
412    // exclude namespace ns1
413    namespaces = new HashSet<String>();
414    namespaces.add("ns1");
415    peerConfigBuilder.setExcludeNamespaces(namespaces).setExcludeTableCFsMap(null);
416    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
417    userEntry = createEntry(null, a, b, c);
418    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
419    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
420
421    // 3. replicate_all flag is true, and only config exclude table-cfs
422    // empty table-cfs map
423    Map<TableName, List<String>> tableCfs = new HashMap<TableName, List<String>>();
424    peerConfigBuilder.setExcludeNamespaces(null).setExcludeTableCFsMap(tableCfs);
425    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
426    userEntry = createEntry(null, a, b, c);
427    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
428    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
429
430    // exclude table bar
431    tableCfs = new HashMap<TableName, List<String>>();
432    tableCfs.put(TableName.valueOf("bar"), null);
433    peerConfigBuilder.setExcludeNamespaces(null).setExcludeTableCFsMap(tableCfs);
434    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
435    userEntry = createEntry(null, a, b, c);
436    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
437    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
438
439    // exclude table foo:a
440    tableCfs = new HashMap<TableName, List<String>>();
441    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a"));
442    peerConfigBuilder.setExcludeNamespaces(null).setExcludeTableCFsMap(tableCfs);
443    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
444    userEntry = createEntry(null, a, b, c);
445    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
446    assertEquals(createEntry(null, b, c), filter.filter(userEntry));
447
448    // 4. replicate_all flag is true, and config exclude namespaces and table-cfs both
449    // exclude ns1 and table foo:a,c
450    namespaces = new HashSet<String>();
451    tableCfs = new HashMap<TableName, List<String>>();
452    namespaces.add("ns1");
453    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a", "c"));
454    peerConfigBuilder.setExcludeNamespaces(namespaces).setExcludeTableCFsMap(tableCfs);
455    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
456    userEntry = createEntry(null, a, b, c);
457    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
458    assertEquals(createEntry(null, b), filter.filter(userEntry));
459
460    // exclude namespace default and table ns1:bar
461    namespaces = new HashSet<String>();
462    tableCfs = new HashMap<TableName, List<String>>();
463    namespaces.add("default");
464    tableCfs.put(TableName.valueOf("ns1:bar"), new ArrayList<String>());
465    peerConfigBuilder.setExcludeNamespaces(namespaces).setExcludeTableCFsMap(tableCfs);
466    when(peer.getPeerConfig()).thenReturn(peerConfigBuilder.build());
467    userEntry = createEntry(null, a, b, c);
468    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
469    assertEquals(null, filter.filter(userEntry));
470  }
471
472  private Entry createEntry(TreeMap<byte[], Integer> scopes, byte[]... kvs) {
473    WALKeyImpl key1 = new WALKeyImpl(new byte[0], TableName.valueOf("foo"),
474      EnvironmentEdgeManager.currentTime(), scopes);
475    WALEdit edit1 = new WALEdit();
476
477    for (byte[] kv : kvs) {
478      edit1.add(new KeyValue(kv, kv, kv));
479    }
480    return new Entry(key1, edit1);
481  }
482
483  private void assertEquals(Entry e1, Entry e2) {
484    Assert.assertEquals(e1 == null, e2 == null);
485    if (e1 == null) {
486      return;
487    }
488
489    // do not compare WALKeys
490
491    // compare kvs
492    Assert.assertEquals(e1.getEdit() == null, e2.getEdit() == null);
493    if (e1.getEdit() == null) {
494      return;
495    }
496    List<Cell> cells1 = e1.getEdit().getCells();
497    List<Cell> cells2 = e2.getEdit().getCells();
498    Assert.assertEquals(cells1.size(), cells2.size());
499    for (int i = 0; i < cells1.size(); i++) {
500      CellComparatorImpl.COMPARATOR.compare(cells1.get(i), cells2.get(i));
501    }
502  }
503}