View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.regionserver.querymatcher.ScanDeleteTracker;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.CellComparator;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.KeyValue.Type;
34  import org.apache.hadoop.hbase.Tag;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.util.Pair;
37  import org.apache.hadoop.hbase.util.Triple;
38  
39  /**
40   * Similar to ScanDeletTracker but tracks the visibility expression also before
41   * deciding if a Cell can be considered deleted
42   */
43  @InterfaceAudience.Private
44  public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
45  
46    private static final Log LOG = LogFactory.getLog(VisibilityScanDeleteTracker.class);
47  
48    // Its better to track the visibility tags in delete based on each type.  Create individual
49    // data structures for tracking each of them.  This would ensure that there is no tracking based
50    // on time and also would handle all cases where deletefamily or deletecolumns is specified with
51    // Latest_timestamp.  In such cases the ts in the delete marker and the masking
52    // put will not be same. So going with individual data structures for different delete
53    // type would solve this problem and also ensure that the combination of different type
54    // of deletes with diff ts would also work fine
55    // Track per TS
56    private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamily =
57        new ArrayList<Triple<List<Tag>, Byte, Long>>();
58    // Delete family version with different ts and different visibility expression could come.
59    // Need to track it per ts.
60    private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamilyVersion =
61        new ArrayList<Triple<List<Tag>, Byte, Long>>();
62    private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
63    // Tracking as List<List> is to handle same ts cell but different visibility tag. 
64    // TODO : Need to handle puts with same ts but different vis tags.
65    private List<Pair<List<Tag>, Byte>> visiblityTagsDeleteColumnVersion =
66        new ArrayList<Pair<List<Tag>, Byte>>();
67  
68    public VisibilityScanDeleteTracker() {
69      super();
70    }
71  
72    @Override
73    public void add(Cell delCell) {
74      //Cannot call super.add because need to find if the delete needs to be considered
75      long timestamp = delCell.getTimestamp();
76      byte type = delCell.getTypeByte();
77      if (type == KeyValue.Type.DeleteFamily.getCode()) {
78        hasFamilyStamp = true;
79        boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
80        if (!hasVisTag && timestamp > familyStamp) {
81          familyStamp = timestamp;
82        }
83        return;
84      } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
85        familyVersionStamps.add(timestamp);
86        extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamilyVersion);
87        return;
88      }
89      // new column, or more general delete type
90      if (deleteBuffer != null) {
91        if (!(CellUtil.matchingQualifier(delCell, deleteBuffer, deleteOffset, deleteLength))) {
92          // A case where there are deletes for a column qualifier but there are
93          // no corresponding puts for them. Rare case.
94          visibilityTagsDeleteColumns = null;
95          visiblityTagsDeleteColumnVersion = null;
96        } else if (type == KeyValue.Type.Delete.getCode() && (deleteTimestamp != timestamp)) {
97          // there is a timestamp change which means we could clear the list
98          // when ts is same and the vis tags are different we need to collect
99          // them all. Interesting part is that in the normal case of puts if
100         // there are 2 cells with same ts and diff vis tags only one of them is
101         // returned. Handling with a single List<Tag> would mean that only one
102         // of the cell would be considered. Doing this as a precaution.
103         // Rare cases.
104         visiblityTagsDeleteColumnVersion = null;
105       }
106     }
107     deleteBuffer = delCell.getQualifierArray();
108     deleteOffset = delCell.getQualifierOffset();
109     deleteLength = delCell.getQualifierLength();
110     deleteType = type;
111     deleteTimestamp = timestamp;
112     extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
113   }
114 
115   private boolean extractDeleteCellVisTags(Cell delCell, Type type) {
116     // If tag is present in the delete
117     boolean hasVisTag = false;
118     if (delCell.getTagsLength() > 0) {
119       Byte deleteCellVisTagsFormat = null;
120       switch (type) {
121       case DeleteFamily:
122         List<Tag> delTags = new ArrayList<Tag>();
123         if (visibilityTagsDeleteFamily == null) {
124           visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>();
125         }
126         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
127         if (!delTags.isEmpty()) {
128           visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>(delTags,
129               deleteCellVisTagsFormat, delCell.getTimestamp()));
130           hasVisTag = true;
131         }
132         break;
133       case DeleteFamilyVersion:
134         if(visibilityTagsDeleteFamilyVersion == null) {
135           visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte, Long>>();
136         }
137         delTags = new ArrayList<Tag>();
138         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
139         if (!delTags.isEmpty()) {
140           visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags,
141               deleteCellVisTagsFormat, delCell.getTimestamp()));
142           hasVisTag = true;
143         }
144         break;
145       case DeleteColumn:
146         if (visibilityTagsDeleteColumns == null) {
147           visibilityTagsDeleteColumns = new ArrayList<Pair<List<Tag>, Byte>>();
148         }
149         delTags = new ArrayList<Tag>();
150         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
151         if (!delTags.isEmpty()) {
152           visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
153               deleteCellVisTagsFormat));
154           hasVisTag = true;
155         }
156         break;
157       case Delete:
158         if (visiblityTagsDeleteColumnVersion == null) {
159           visiblityTagsDeleteColumnVersion = new ArrayList<Pair<List<Tag>, Byte>>();
160         }
161         delTags = new ArrayList<Tag>();
162         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
163         if (!delTags.isEmpty()) {
164           visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
165               deleteCellVisTagsFormat));
166           hasVisTag = true;
167         }
168         break;
169       default:
170         throw new IllegalArgumentException("Invalid delete type");
171       }
172     }
173     return hasVisTag;
174   }
175 
176   @Override
177   public DeleteResult isDeleted(Cell cell) {
178     long timestamp = cell.getTimestamp();
179     try {
180       if (hasFamilyStamp) {
181         if (visibilityTagsDeleteFamily != null) {
182           if (!visibilityTagsDeleteFamily.isEmpty()) {
183             for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) {
184               // visibilityTagsDeleteFamily is ArrayList
185               Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i);
186               if (timestamp <= triple.getThird()) {
187                 List<Tag> putVisTags = new ArrayList<Tag>();
188                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
189                 boolean matchFound = VisibilityLabelServiceManager.getInstance()
190                     .getVisibilityLabelService().matchVisibility(putVisTags, putCellVisTagsFormat,
191                       triple.getFirst(), triple.getSecond());
192                 if (matchFound) {
193                   // A return type of FAMILY_DELETED will cause skip for all remaining cells from
194                   // this
195                   // family. We would like to match visibility expression on every put cells after
196                   // this and only remove those matching with the family delete visibility. So we
197                   // are
198                   // returning FAMILY_VERSION_DELETED from here.
199                   return DeleteResult.FAMILY_VERSION_DELETED;
200                 }
201               }
202             }
203           } else {
204             if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp <= familyStamp) {
205               // No tags
206               return DeleteResult.FAMILY_VERSION_DELETED;
207             }
208           }
209         } else {
210           if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp <= familyStamp) {
211             // No tags
212             return DeleteResult.FAMILY_VERSION_DELETED;
213           }
214         }
215       }
216       if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
217         if (visibilityTagsDeleteFamilyVersion != null) {
218           if (!visibilityTagsDeleteFamilyVersion.isEmpty()) {
219             for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) {
220               // visibilityTagsDeleteFamilyVersion is ArrayList
221               Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i);
222               if (timestamp == triple.getThird()) {
223                 List<Tag> putVisTags = new ArrayList<Tag>();
224                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
225                 boolean matchFound = VisibilityLabelServiceManager.getInstance()
226                     .getVisibilityLabelService().matchVisibility(putVisTags, putCellVisTagsFormat,
227                       triple.getFirst(), triple.getSecond());
228                 if (matchFound) {
229                   return DeleteResult.FAMILY_VERSION_DELETED;
230                 }
231               }
232             }
233           } else {
234             if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
235               // No tags
236               return DeleteResult.FAMILY_VERSION_DELETED;
237             }
238           }
239         } else {
240           if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
241             // No tags
242             return DeleteResult.FAMILY_VERSION_DELETED;
243           }
244         }
245       }
246       if (deleteBuffer != null) {
247         int ret = CellComparator.compareQualifiers(cell, deleteBuffer, deleteOffset, deleteLength);
248         if (ret == 0) {
249           if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
250             if (visibilityTagsDeleteColumns != null) {
251               if (!visibilityTagsDeleteColumns.isEmpty()) {
252                 for (Pair<List<Tag>, Byte> tags : visibilityTagsDeleteColumns) {
253                   List<Tag> putVisTags = new ArrayList<Tag>();
254                   Byte putCellVisTagsFormat =
255                       VisibilityUtils.extractVisibilityTags(cell, putVisTags);
256                   boolean matchFound = VisibilityLabelServiceManager.getInstance()
257                       .getVisibilityLabelService().matchVisibility(putVisTags, putCellVisTagsFormat,
258                         tags.getFirst(), tags.getSecond());
259                   if (matchFound) {
260                     return DeleteResult.VERSION_DELETED;
261                   }
262                 }
263               } else {
264                 if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
265                   // No tags
266                   return DeleteResult.VERSION_DELETED;
267                 }
268               }
269             } else {
270               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
271                 // No tags
272                 return DeleteResult.VERSION_DELETED;
273               }
274             }
275           }
276           // Delete (aka DeleteVersion)
277           // If the timestamp is the same, keep this one
278           if (timestamp == deleteTimestamp) {
279             if (visiblityTagsDeleteColumnVersion != null) {
280               if (!visiblityTagsDeleteColumnVersion.isEmpty()) {
281                 for (Pair<List<Tag>, Byte> tags : visiblityTagsDeleteColumnVersion) {
282                   List<Tag> putVisTags = new ArrayList<Tag>();
283                   Byte putCellVisTagsFormat =
284                       VisibilityUtils.extractVisibilityTags(cell, putVisTags);
285                   boolean matchFound = VisibilityLabelServiceManager.getInstance()
286                       .getVisibilityLabelService().matchVisibility(putVisTags, putCellVisTagsFormat,
287                         tags.getFirst(), tags.getSecond());
288                   if (matchFound) {
289                     return DeleteResult.VERSION_DELETED;
290                   }
291                 }
292               } else {
293                 if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
294                   // No tags
295                   return DeleteResult.VERSION_DELETED;
296                 }
297               }
298             } else {
299               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
300                 // No tags
301                 return DeleteResult.VERSION_DELETED;
302               }
303             }
304           }
305         } else if (ret > 0) {
306           // Next column case.
307           deleteBuffer = null;
308           // Can nullify this because we are moving to the next column
309           visibilityTagsDeleteColumns = null;
310           visiblityTagsDeleteColumnVersion = null;
311         } else {
312           throw new IllegalStateException("isDeleted failed: deleteBuffer="
313               + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier="
314               + Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
315                   cell.getQualifierLength())
316               + ", timestamp=" + timestamp + ", comparison result: " + ret);
317         }
318       }
319     } catch (IOException e) {
320       LOG.error("Error in isDeleted() check! Will treat cell as not deleted", e);
321     }
322     return DeleteResult.NOT_DELETED;
323   }
324 
325   @Override
326   public void reset() {
327     super.reset();
328     // clear only here
329     visibilityTagsDeleteColumns = null;
330     visibilityTagsDeleteFamily = null;
331     visibilityTagsDeleteFamilyVersion = null;
332     visiblityTagsDeleteColumnVersion = null;
333   }
334 }