View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.Cell;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.KeyValue.Type;
31  import org.apache.hadoop.hbase.Tag;
32  import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.hbase.util.Pair;
35  import org.apache.hadoop.hbase.util.Triple;
36  
37  /**
38   * Similar to ScanDeletTracker but tracks the visibility expression also before
39   * deciding if a Cell can be considered deleted
40   */
41  @InterfaceAudience.Private
42  public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
43  
44    private static final Log LOG = LogFactory.getLog(VisibilityScanDeleteTracker.class);
45  
46    // Its better to track the visibility tags in delete based on each type.  Create individual
47    // data structures for tracking each of them.  This would ensure that there is no tracking based
48    // on time and also would handle all cases where deletefamily or deletecolumns is specified with
49    // Latest_timestamp.  In such cases the ts in the delete marker and the masking
50    // put will not be same. So going with individual data structures for different delete
51    // type would solve this problem and also ensure that the combination of different type
52    // of deletes with diff ts would also work fine
53    // Track per TS
54    private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamily =
55        new ArrayList<Triple<List<Tag>, Byte, Long>>();
56    // Delete family version with different ts and different visibility expression could come.
57    // Need to track it per ts.
58    private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamilyVersion =
59        new ArrayList<Triple<List<Tag>, Byte, Long>>();
60    private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
61    // Tracking as List<List> is to handle same ts cell but different visibility tag. 
62    // TODO : Need to handle puts with same ts but different vis tags.
63    private List<Pair<List<Tag>, Byte>> visiblityTagsDeleteColumnVersion =
64        new ArrayList<Pair<List<Tag>, Byte>>();
65  
66    public VisibilityScanDeleteTracker() {
67      super();
68    }
69  
70    @Override
71    public void add(Cell delCell) {
72      //Cannot call super.add because need to find if the delete needs to be considered
73      long timestamp = delCell.getTimestamp();
74      int qualifierOffset = delCell.getQualifierOffset();
75      int qualifierLength = delCell.getQualifierLength();
76      byte type = delCell.getTypeByte();
77      if (type == KeyValue.Type.DeleteFamily.getCode()) {
78        hasFamilyStamp = true;
79        boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
80        if (!hasVisTag && timestamp > familyStamp) {
81          familyStamp = timestamp;
82        }
83        return;
84      } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
85        familyVersionStamps.add(timestamp);
86        extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamilyVersion);
87        return;
88      }
89      // new column, or more general delete type
90      if (deleteBuffer != null) {
91        if (Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, delCell.getQualifierArray(),
92            qualifierOffset, qualifierLength) != 0) {
93          // A case where there are deletes for a column qualifier but there are
94          // no corresponding puts for them. Rare case.
95          visibilityTagsDeleteColumns = null;
96          visiblityTagsDeleteColumnVersion = null;
97        } else if (type == KeyValue.Type.Delete.getCode() && (deleteTimestamp != timestamp)) {
98          // there is a timestamp change which means we could clear the list
99          // when ts is same and the vis tags are different we need to collect
100         // them all. Interesting part is that in the normal case of puts if
101         // there are 2 cells with same ts and diff vis tags only one of them is
102         // returned. Handling with a single List<Tag> would mean that only one
103         // of the cell would be considered. Doing this as a precaution.
104         // Rare cases.
105         visiblityTagsDeleteColumnVersion = null;
106       }
107     }
108     deleteBuffer = delCell.getQualifierArray();
109     deleteOffset = qualifierOffset;
110     deleteLength = qualifierLength;
111     deleteType = type;
112     deleteTimestamp = timestamp;
113     extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
114   }
115 
116   private boolean extractDeleteCellVisTags(Cell delCell, Type type) {
117     // If tag is present in the delete
118     boolean hasVisTag = false;
119     if (delCell.getTagsLength() > 0) {
120       switch (type) {
121       case DeleteFamily:
122         List<Tag> delTags = new ArrayList<Tag>();
123         if (visibilityTagsDeleteFamily != null) {
124           Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
125           if (!delTags.isEmpty()) {
126             visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>(
127                 delTags, deleteCellVisTagsFormat, delCell.getTimestamp()));
128             hasVisTag = true;
129           }
130         }
131         break;
132       case DeleteFamilyVersion:
133         delTags = new ArrayList<Tag>();
134         Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
135         if (!delTags.isEmpty()) {
136           visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags,
137               deleteCellVisTagsFormat, delCell.getTimestamp()));
138           hasVisTag = true;
139         }
140         break;
141       case DeleteColumn:
142         if (visibilityTagsDeleteColumns == null) {
143           visibilityTagsDeleteColumns = new ArrayList<Pair<List<Tag>, Byte>>();
144         }
145         delTags = new ArrayList<Tag>();
146         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
147         if (!delTags.isEmpty()) {
148           visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
149               deleteCellVisTagsFormat));
150           hasVisTag = true;
151         }
152         break;
153       case Delete:
154         if (visiblityTagsDeleteColumnVersion == null) {
155           visiblityTagsDeleteColumnVersion = new ArrayList<Pair<List<Tag>, Byte>>();
156         }
157         delTags = new ArrayList<Tag>();
158         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
159         if (!delTags.isEmpty()) {
160           visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
161               deleteCellVisTagsFormat));
162           hasVisTag = true;
163         }
164         break;
165       default:
166         throw new IllegalArgumentException("Invalid delete type");
167       }
168     } else {
169       switch (type) {
170       case DeleteFamily:
171         visibilityTagsDeleteFamily = null;
172         break;
173       case DeleteFamilyVersion:
174         visibilityTagsDeleteFamilyVersion = null;
175         break;
176       case DeleteColumn:
177         visibilityTagsDeleteColumns = null;
178         break;
179       case Delete:
180         visiblityTagsDeleteColumnVersion = null;
181         break;
182       default:
183         throw new IllegalArgumentException("Invalid delete type");
184       }
185     }
186     return hasVisTag;
187   }
188 
189   @Override
190   public DeleteResult isDeleted(Cell cell) {
191     long timestamp = cell.getTimestamp();
192     int qualifierOffset = cell.getQualifierOffset();
193     int qualifierLength = cell.getQualifierLength();
194     try {
195       if (hasFamilyStamp) {
196         if (visibilityTagsDeleteFamily != null) {
197           for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) {
198             // visibilityTagsDeleteFamily is ArrayList
199             Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i);
200             if (timestamp <= triple.getThird()) {
201               List<Tag> putVisTags = new ArrayList<Tag>();
202               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
203               boolean matchFound = VisibilityLabelServiceManager
204                   .getInstance()
205                   .getVisibilityLabelService()
206                   .matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
207                       triple.getSecond());
208               if (matchFound) {
209                 // A return type of FAMILY_DELETED will cause skip for all remaining cells from this
210                 // family. We would like to match visibility expression on every put cells after
211                 // this and only remove those matching with the family delete visibility. So we are
212                 // returning FAMILY_VERSION_DELETED from here.
213                 return DeleteResult.FAMILY_VERSION_DELETED;
214               }
215             }
216           }
217         } else {
218           if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp<=familyStamp) {
219             // No tags
220             return DeleteResult.FAMILY_VERSION_DELETED;
221           }
222         }
223       }
224       if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
225         if (visibilityTagsDeleteFamilyVersion != null) {
226           for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) {
227             // visibilityTagsDeleteFamilyVersion is ArrayList
228             Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i);
229             if (timestamp == triple.getThird()) {
230               List<Tag> putVisTags = new ArrayList<Tag>();
231               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
232               boolean matchFound = VisibilityLabelServiceManager
233                   .getInstance()
234                   .getVisibilityLabelService()
235                   .matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
236                       triple.getSecond());
237               if (matchFound) {
238                 return DeleteResult.FAMILY_VERSION_DELETED;
239               }
240             }
241           }
242         } else {
243           if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
244             // No tags
245             return DeleteResult.FAMILY_VERSION_DELETED;
246           }
247         }
248       }
249       if (deleteBuffer != null) {
250         int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength,
251             cell.getQualifierArray(), qualifierOffset, qualifierLength);
252 
253         if (ret == 0) {
254           if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
255             if (visibilityTagsDeleteColumns != null) {
256               for (Pair<List<Tag>, Byte> tags : visibilityTagsDeleteColumns) {
257                 List<Tag> putVisTags = new ArrayList<Tag>();
258                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
259                 boolean matchFound = VisibilityLabelServiceManager
260                     .getInstance()
261                     .getVisibilityLabelService()
262                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
263                         tags.getSecond());
264                 if (matchFound) {
265                   return DeleteResult.VERSION_DELETED;
266                 }
267               }
268             } else {
269               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
270                 // No tags
271                 return DeleteResult.VERSION_DELETED;
272               }
273             }
274           }
275           // Delete (aka DeleteVersion)
276           // If the timestamp is the same, keep this one
277           if (timestamp == deleteTimestamp) {
278             if (visiblityTagsDeleteColumnVersion != null) {
279               for (Pair<List<Tag>, Byte> tags : visiblityTagsDeleteColumnVersion) {
280                 List<Tag> putVisTags = new ArrayList<Tag>();
281                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
282                 boolean matchFound = VisibilityLabelServiceManager
283                     .getInstance()
284                     .getVisibilityLabelService()
285                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
286                         tags.getSecond());
287                 if (matchFound) {
288                   return DeleteResult.VERSION_DELETED;
289                 }
290               }
291             } else {
292               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
293                 // No tags
294                 return DeleteResult.VERSION_DELETED;
295               }
296             }
297           }
298         } else if (ret < 0) {
299           // Next column case.
300           deleteBuffer = null;
301           visibilityTagsDeleteColumns = null;
302           visiblityTagsDeleteColumnVersion = null;
303         } else {
304           throw new IllegalStateException("isDeleted failed: deleteBuffer="
305               + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier="
306               + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength)
307               + ", timestamp=" + timestamp + ", comparison result: " + ret);
308         }
309       }
310     } catch (IOException e) {
311       LOG.error("Error in isDeleted() check! Will treat cell as not deleted", e);
312     }
313     return DeleteResult.NOT_DELETED;
314   }
315 
316   @Override
317   public void reset() {
318     super.reset();
319     visibilityTagsDeleteColumns = null;
320     visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>();
321     visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte, Long>>();
322     visiblityTagsDeleteColumnVersion = null;
323   }
324 }