View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.Cell;
29  import org.apache.hadoop.hbase.CellComparator;
30  import org.apache.hadoop.hbase.CellUtil;
31  import org.apache.hadoop.hbase.KeyValue;
32  import org.apache.hadoop.hbase.KeyValue.Type;
33  import org.apache.hadoop.hbase.Tag;
34  import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.util.Pair;
37  import org.apache.hadoop.hbase.util.Triple;
38  
39  /**
40   * Similar to ScanDeletTracker but tracks the visibility expression also before
41   * deciding if a Cell can be considered deleted
42   */
43  @InterfaceAudience.Private
44  public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
45  
46    private static final Log LOG = LogFactory.getLog(VisibilityScanDeleteTracker.class);
47  
48    // Its better to track the visibility tags in delete based on each type.  Create individual
49    // data structures for tracking each of them.  This would ensure that there is no tracking based
50    // on time and also would handle all cases where deletefamily or deletecolumns is specified with
51    // Latest_timestamp.  In such cases the ts in the delete marker and the masking
52    // put will not be same. So going with individual data structures for different delete
53    // type would solve this problem and also ensure that the combination of different type
54    // of deletes with diff ts would also work fine
55    // Track per TS
56    private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamily =
57        new ArrayList<Triple<List<Tag>, Byte, Long>>();
58    // Delete family version with different ts and different visibility expression could come.
59    // Need to track it per ts.
60    private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamilyVersion =
61        new ArrayList<Triple<List<Tag>, Byte, Long>>();
62    private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
63    // Tracking as List<List> is to handle same ts cell but different visibility tag. 
64    // TODO : Need to handle puts with same ts but different vis tags.
65    private List<Pair<List<Tag>, Byte>> visiblityTagsDeleteColumnVersion =
66        new ArrayList<Pair<List<Tag>, Byte>>();
67  
68    public VisibilityScanDeleteTracker() {
69      super();
70    }
71  
72    @Override
73    public void add(Cell delCell) {
74      //Cannot call super.add because need to find if the delete needs to be considered
75      long timestamp = delCell.getTimestamp();
76      byte type = delCell.getTypeByte();
77      if (type == KeyValue.Type.DeleteFamily.getCode()) {
78        hasFamilyStamp = true;
79        boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
80        if (!hasVisTag && timestamp > familyStamp) {
81          familyStamp = timestamp;
82        }
83        return;
84      } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
85        familyVersionStamps.add(timestamp);
86        extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamilyVersion);
87        return;
88      }
89      // new column, or more general delete type
90      if (deleteBuffer != null) {
91        if (!(CellUtil.matchingQualifier(delCell, deleteBuffer, deleteOffset, deleteLength))) {
92          // A case where there are deletes for a column qualifier but there are
93          // no corresponding puts for them. Rare case.
94          visibilityTagsDeleteColumns = null;
95          visiblityTagsDeleteColumnVersion = null;
96        } else if (type == KeyValue.Type.Delete.getCode() && (deleteTimestamp != timestamp)) {
97          // there is a timestamp change which means we could clear the list
98          // when ts is same and the vis tags are different we need to collect
99          // them all. Interesting part is that in the normal case of puts if
100         // there are 2 cells with same ts and diff vis tags only one of them is
101         // returned. Handling with a single List<Tag> would mean that only one
102         // of the cell would be considered. Doing this as a precaution.
103         // Rare cases.
104         visiblityTagsDeleteColumnVersion = null;
105       }
106     }
107     deleteBuffer = delCell.getQualifierArray();
108     deleteOffset = delCell.getQualifierOffset();
109     deleteLength = delCell.getQualifierLength();
110     deleteType = type;
111     deleteTimestamp = timestamp;
112     extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
113   }
114 
115   private boolean extractDeleteCellVisTags(Cell delCell, Type type) {
116     // If tag is present in the delete
117     boolean hasVisTag = false;
118     if (delCell.getTagsLength() > 0) {
119       switch (type) {
120       case DeleteFamily:
121         List<Tag> delTags = new ArrayList<Tag>();
122         if (visibilityTagsDeleteFamily != null) {
123           Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
124           if (!delTags.isEmpty()) {
125             visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>(
126                 delTags, deleteCellVisTagsFormat, delCell.getTimestamp()));
127             hasVisTag = true;
128           }
129         }
130         break;
131       case DeleteFamilyVersion:
132         delTags = new ArrayList<Tag>();
133         Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
134         if (!delTags.isEmpty()) {
135           visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags,
136               deleteCellVisTagsFormat, delCell.getTimestamp()));
137           hasVisTag = true;
138         }
139         break;
140       case DeleteColumn:
141         if (visibilityTagsDeleteColumns == null) {
142           visibilityTagsDeleteColumns = new ArrayList<Pair<List<Tag>, Byte>>();
143         }
144         delTags = new ArrayList<Tag>();
145         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
146         if (!delTags.isEmpty()) {
147           visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
148               deleteCellVisTagsFormat));
149           hasVisTag = true;
150         }
151         break;
152       case Delete:
153         if (visiblityTagsDeleteColumnVersion == null) {
154           visiblityTagsDeleteColumnVersion = new ArrayList<Pair<List<Tag>, Byte>>();
155         }
156         delTags = new ArrayList<Tag>();
157         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
158         if (!delTags.isEmpty()) {
159           visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
160               deleteCellVisTagsFormat));
161           hasVisTag = true;
162         }
163         break;
164       default:
165         throw new IllegalArgumentException("Invalid delete type");
166       }
167     } else {
168       switch (type) {
169       case DeleteFamily:
170         visibilityTagsDeleteFamily = null;
171         break;
172       case DeleteFamilyVersion:
173         visibilityTagsDeleteFamilyVersion = null;
174         break;
175       case DeleteColumn:
176         visibilityTagsDeleteColumns = null;
177         break;
178       case Delete:
179         visiblityTagsDeleteColumnVersion = null;
180         break;
181       default:
182         throw new IllegalArgumentException("Invalid delete type");
183       }
184     }
185     return hasVisTag;
186   }
187 
188   @Override
189   public DeleteResult isDeleted(Cell cell) {
190     long timestamp = cell.getTimestamp();
191     try {
192       if (hasFamilyStamp) {
193         if (visibilityTagsDeleteFamily != null) {
194           for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) {
195             // visibilityTagsDeleteFamily is ArrayList
196             Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i);
197             if (timestamp <= triple.getThird()) {
198               List<Tag> putVisTags = new ArrayList<Tag>();
199               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
200               boolean matchFound = VisibilityLabelServiceManager
201                   .getInstance()
202                   .getVisibilityLabelService()
203                   .matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
204                       triple.getSecond());
205               if (matchFound) {
206                 // A return type of FAMILY_DELETED will cause skip for all remaining cells from this
207                 // family. We would like to match visibility expression on every put cells after
208                 // this and only remove those matching with the family delete visibility. So we are
209                 // returning FAMILY_VERSION_DELETED from here.
210                 return DeleteResult.FAMILY_VERSION_DELETED;
211               }
212             }
213           }
214         } else {
215           if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp<=familyStamp) {
216             // No tags
217             return DeleteResult.FAMILY_VERSION_DELETED;
218           }
219         }
220       }
221       if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
222         if (visibilityTagsDeleteFamilyVersion != null) {
223           for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) {
224             // visibilityTagsDeleteFamilyVersion is ArrayList
225             Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i);
226             if (timestamp == triple.getThird()) {
227               List<Tag> putVisTags = new ArrayList<Tag>();
228               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
229               boolean matchFound = VisibilityLabelServiceManager
230                   .getInstance()
231                   .getVisibilityLabelService()
232                   .matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
233                       triple.getSecond());
234               if (matchFound) {
235                 return DeleteResult.FAMILY_VERSION_DELETED;
236               }
237             }
238           }
239         } else {
240           if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
241             // No tags
242             return DeleteResult.FAMILY_VERSION_DELETED;
243           }
244         }
245       }
246       if (deleteBuffer != null) {
247         int ret = CellComparator.compareQualifiers(cell, deleteBuffer, deleteOffset, deleteLength);
248         if (ret == 0) {
249           if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
250             if (visibilityTagsDeleteColumns != null) {
251               for (Pair<List<Tag>, Byte> tags : visibilityTagsDeleteColumns) {
252                 List<Tag> putVisTags = new ArrayList<Tag>();
253                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
254                 boolean matchFound = VisibilityLabelServiceManager
255                     .getInstance()
256                     .getVisibilityLabelService()
257                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
258                         tags.getSecond());
259                 if (matchFound) {
260                   return DeleteResult.VERSION_DELETED;
261                 }
262               }
263             } else {
264               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
265                 // No tags
266                 return DeleteResult.VERSION_DELETED;
267               }
268             }
269           }
270           // Delete (aka DeleteVersion)
271           // If the timestamp is the same, keep this one
272           if (timestamp == deleteTimestamp) {
273             if (visiblityTagsDeleteColumnVersion != null) {
274               for (Pair<List<Tag>, Byte> tags : visiblityTagsDeleteColumnVersion) {
275                 List<Tag> putVisTags = new ArrayList<Tag>();
276                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
277                 boolean matchFound = VisibilityLabelServiceManager
278                     .getInstance()
279                     .getVisibilityLabelService()
280                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
281                         tags.getSecond());
282                 if (matchFound) {
283                   return DeleteResult.VERSION_DELETED;
284                 }
285               }
286             } else {
287               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
288                 // No tags
289                 return DeleteResult.VERSION_DELETED;
290               }
291             }
292           }
293         } else if (ret > 0) {
294           // Next column case.
295           deleteBuffer = null;
296           visibilityTagsDeleteColumns = null;
297           visiblityTagsDeleteColumnVersion = null;
298         } else {
299           throw new IllegalStateException("isDeleted failed: deleteBuffer="
300               + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier="
301               + Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
302                   cell.getQualifierLength())
303               + ", timestamp=" + timestamp + ", comparison result: " + ret);
304         }
305       }
306     } catch (IOException e) {
307       LOG.error("Error in isDeleted() check! Will treat cell as not deleted", e);
308     }
309     return DeleteResult.NOT_DELETED;
310   }
311 
312   @Override
313   public void reset() {
314     super.reset();
315     visibilityTagsDeleteColumns = null;
316     visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>();
317     visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte, Long>>();
318     visiblityTagsDeleteColumnVersion = null;
319   }
320 }