View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.HashMap;
24  import java.util.Iterator;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.Map.Entry;
28  import java.util.Set;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.KeyValue.Type;
36  import org.apache.hadoop.hbase.Tag;
37  import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.Pair;
40  
41  /**
42   * Similar to ScanDeletTracker but tracks the visibility expression also before
43   * deciding if a Cell can be considered deleted
44   */
45  @InterfaceAudience.Private
46  public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
47  
48    private static final Log LOG = LogFactory.getLog(VisibilityScanDeleteTracker.class);
49  
50    // Its better to track the visibility tags in delete based on each type.  Create individual
51    // data structures for tracking each of them.  This would ensure that there is no tracking based
52    // on time and also would handle all cases where deletefamily or deletecolumns is specified with
53    // Latest_timestamp.  In such cases the ts in the delete marker and the masking
54    // put will not be same. So going with individual data structures for different delete
55    // type would solve this problem and also ensure that the combination of different type
56    // of deletes with diff ts would also work fine
57    // Track per TS
58    private Map<Long, Pair<List<Tag>, Byte>> visibilityTagsDeleteFamily =
59        new HashMap<Long, Pair<List<Tag>, Byte>>();
60    // Delete family version with different ts and different visibility expression could come.
61    // Need to track it per ts.
62    private Map<Long,Pair<List<Tag>, Byte>> visibilityTagsDeleteFamilyVersion =
63        new HashMap<Long, Pair<List<Tag>, Byte>>();
64    private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
65    // Tracking as List<List> is to handle same ts cell but different visibility tag. 
66    // TODO : Need to handle puts with same ts but different vis tags.
67    private List<Pair<List<Tag>, Byte>> visiblityTagsDeleteColumnVersion =
68        new ArrayList<Pair<List<Tag>, Byte>>();
69  
70    public VisibilityScanDeleteTracker() {
71      super();
72    }
73  
74    @Override
75    public void add(Cell delCell) {
76      //Cannot call super.add because need to find if the delete needs to be considered
77      long timestamp = delCell.getTimestamp();
78      int qualifierOffset = delCell.getQualifierOffset();
79      int qualifierLength = delCell.getQualifierLength();
80      byte type = delCell.getTypeByte();
81      if (type == KeyValue.Type.DeleteFamily.getCode()) {
82        hasFamilyStamp = true;
83        //familyStamps.add(delCell.getTimestamp());
84        extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
85        return;
86      } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
87        familyVersionStamps.add(timestamp);
88        extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamilyVersion);
89        return;
90      }
91      // new column, or more general delete type
92      if (deleteBuffer != null) {
93        if (Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, delCell.getQualifierArray(),
94            qualifierOffset, qualifierLength) != 0) {
95          // A case where there are deletes for a column qualifier but there are
96          // no corresponding puts for them. Rare case.
97          visibilityTagsDeleteColumns = null;
98          visiblityTagsDeleteColumnVersion = null;
99        } else if (type == KeyValue.Type.Delete.getCode() && (deleteTimestamp != timestamp)) {
100         // there is a timestamp change which means we could clear the list
101         // when ts is same and the vis tags are different we need to collect
102         // them all. Interesting part is that in the normal case of puts if
103         // there are 2 cells with same ts and diff vis tags only one of them is
104         // returned. Handling with a single List<Tag> would mean that only one
105         // of the cell would be considered. Doing this as a precaution.
106         // Rare cases.
107         visiblityTagsDeleteColumnVersion = null;
108       }
109     }
110     deleteBuffer = delCell.getQualifierArray();
111     deleteOffset = qualifierOffset;
112     deleteLength = qualifierLength;
113     deleteType = type;
114     deleteTimestamp = timestamp;
115     extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
116   }
117 
118   private void extractDeleteCellVisTags(Cell delCell, Type type) {
119     // If tag is present in the delete
120     if (delCell.getTagsLength() > 0) {
121       switch (type) {
122       case DeleteFamily:
123         List<Tag> delTags = new ArrayList<Tag>();
124         if (visibilityTagsDeleteFamily != null) {
125           Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
126           if (!delTags.isEmpty()) {
127             visibilityTagsDeleteFamily.put(delCell.getTimestamp(), new Pair<List<Tag>, Byte>(
128                 delTags, deleteCellVisTagsFormat));
129           }
130         }
131         break;
132       case DeleteFamilyVersion:
133         delTags = new ArrayList<Tag>();
134         Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
135         if (!delTags.isEmpty()) {
136           visibilityTagsDeleteFamilyVersion.put(delCell.getTimestamp(), new Pair<List<Tag>, Byte>(
137               delTags, deleteCellVisTagsFormat));
138         }
139         break;
140       case DeleteColumn:
141         if (visibilityTagsDeleteColumns == null) {
142           visibilityTagsDeleteColumns = new ArrayList<Pair<List<Tag>, Byte>>();
143         }
144         delTags = new ArrayList<Tag>();
145         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
146         if (!delTags.isEmpty()) {
147           visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
148               deleteCellVisTagsFormat));
149         }
150         break;
151       case Delete:
152         if (visiblityTagsDeleteColumnVersion == null) {
153           visiblityTagsDeleteColumnVersion = new ArrayList<Pair<List<Tag>, Byte>>();
154         }
155         delTags = new ArrayList<Tag>();
156         deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
157         if (!delTags.isEmpty()) {
158           visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
159               deleteCellVisTagsFormat));
160         }
161         break;
162       default:
163         throw new IllegalArgumentException("Invalid delete type");
164       }
165     } else {
166       switch (type) {
167       case DeleteFamily:
168         visibilityTagsDeleteFamily = null;
169         break;
170       case DeleteFamilyVersion:
171         visibilityTagsDeleteFamilyVersion = null;
172         break;
173       case DeleteColumn:
174         visibilityTagsDeleteColumns = null;
175         break;
176       case Delete:
177         visiblityTagsDeleteColumnVersion = null;
178         break;
179       default:
180         throw new IllegalArgumentException("Invalid delete type");
181       }
182     }
183   }
184 
185   @Override
186   public DeleteResult isDeleted(Cell cell) {
187     long timestamp = cell.getTimestamp();
188     int qualifierOffset = cell.getQualifierOffset();
189     int qualifierLength = cell.getQualifierLength();
190     try {
191       if (hasFamilyStamp) {
192         if (visibilityTagsDeleteFamily != null) {
193           Set<Entry<Long, Pair<List<Tag>, Byte>>> deleteFamilies = visibilityTagsDeleteFamily
194               .entrySet();
195           Iterator<Entry<Long, Pair<List<Tag>, Byte>>> iterator = deleteFamilies.iterator();
196           while (iterator.hasNext()) {
197             Entry<Long, Pair<List<Tag>, Byte>> entry = iterator.next();
198             if (timestamp <= entry.getKey()) {
199               List<Tag> putVisTags = new ArrayList<Tag>();
200               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
201               boolean matchFound = VisibilityLabelServiceManager
202                   .getInstance()
203                   .getVisibilityLabelService()
204                   .matchVisibility(putVisTags, putCellVisTagsFormat, entry.getValue().getFirst(),
205                       entry.getValue().getSecond());
206               if (matchFound) {
207                 return DeleteResult.FAMILY_VERSION_DELETED;
208               }
209             }
210           }
211         } else {
212           if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
213             // No tags
214             return DeleteResult.FAMILY_VERSION_DELETED;
215           }
216         }
217       }
218       if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
219         if (visibilityTagsDeleteFamilyVersion != null) {
220           Pair<List<Tag>, Byte> tags = visibilityTagsDeleteFamilyVersion.get(Long
221               .valueOf(timestamp));
222           if (tags != null) {
223             List<Tag> putVisTags = new ArrayList<Tag>();
224             Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
225             boolean matchFound = VisibilityLabelServiceManager
226                 .getInstance()
227                 .getVisibilityLabelService()
228                 .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
229                     tags.getSecond());
230             if (matchFound) {
231               return DeleteResult.FAMILY_VERSION_DELETED;
232             }
233           }
234         } else {
235           if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
236             // No tags
237             return DeleteResult.FAMILY_VERSION_DELETED;
238           }
239         }
240       }
241       if (deleteBuffer != null) {
242         int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength,
243             cell.getQualifierArray(), qualifierOffset, qualifierLength);
244 
245         if (ret == 0) {
246           if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
247             if (visibilityTagsDeleteColumns != null) {
248               for (Pair<List<Tag>, Byte> tags : visibilityTagsDeleteColumns) {
249                 List<Tag> putVisTags = new ArrayList<Tag>();
250                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
251                 boolean matchFound = VisibilityLabelServiceManager
252                     .getInstance()
253                     .getVisibilityLabelService()
254                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
255                         tags.getSecond());
256                 if (matchFound) {
257                   return DeleteResult.VERSION_DELETED;
258                 }
259               }
260             } else {
261               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
262                 // No tags
263                 return DeleteResult.VERSION_DELETED;
264               }
265             }
266           }
267           // Delete (aka DeleteVersion)
268           // If the timestamp is the same, keep this one
269           if (timestamp == deleteTimestamp) {
270             if (visiblityTagsDeleteColumnVersion != null) {
271               for (Pair<List<Tag>, Byte> tags : visiblityTagsDeleteColumnVersion) {
272                 List<Tag> putVisTags = new ArrayList<Tag>();
273                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
274                 boolean matchFound = VisibilityLabelServiceManager
275                     .getInstance()
276                     .getVisibilityLabelService()
277                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
278                         tags.getSecond());
279                 if (matchFound) {
280                   return DeleteResult.VERSION_DELETED;
281                 }
282               }
283             } else {
284               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
285                 // No tags
286                 return DeleteResult.VERSION_DELETED;
287               }
288             }
289           }
290         } else if (ret < 0) {
291           // Next column case.
292           deleteBuffer = null;
293           visibilityTagsDeleteColumns = null;
294           visiblityTagsDeleteColumnVersion = null;
295         } else {
296           throw new IllegalStateException("isDeleted failed: deleteBuffer="
297               + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier="
298               + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength)
299               + ", timestamp=" + timestamp + ", comparison result: " + ret);
300         }
301       }
302     } catch (IOException e) {
303       LOG.error("Error in isDeleted() check! Will treat cell as not deleted", e);
304     }
305     return DeleteResult.NOT_DELETED;
306   }
307 
308   @Override
309   public void reset() {
310     super.reset();
311     visibilityTagsDeleteColumns = null;
312     visibilityTagsDeleteFamily = new HashMap<Long, Pair<List<Tag>, Byte>>();
313     visibilityTagsDeleteFamilyVersion = new HashMap<Long, Pair<List<Tag>, Byte>>();
314     visiblityTagsDeleteColumnVersion = null;
315   }
316 }