001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.visibility;
019
020import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE;
021
022import com.google.protobuf.ByteString;
023
024import java.io.ByteArrayOutputStream;
025import java.io.DataOutputStream;
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.Collections;
029import java.util.HashMap;
030import java.util.Iterator;
031import java.util.List;
032import java.util.Map;
033import java.util.Map.Entry;
034import java.util.Optional;
035import java.util.Set;
036
037import org.apache.commons.lang3.StringUtils;
038import org.apache.hadoop.conf.Configuration;
039import org.apache.hadoop.hbase.ArrayBackedTag;
040import org.apache.hadoop.hbase.Cell;
041import org.apache.hadoop.hbase.PrivateCellUtil;
042import org.apache.hadoop.hbase.Tag;
043import org.apache.hadoop.hbase.TagType;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
045import org.apache.hadoop.hbase.exceptions.DeserializationException;
046import org.apache.hadoop.hbase.filter.Filter;
047import org.apache.hadoop.hbase.io.util.StreamUtils;
048import org.apache.hadoop.hbase.ipc.RpcServer;
049import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
050import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations;
051import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations;
052import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
053import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
054import org.apache.hadoop.hbase.regionserver.Region;
055import org.apache.hadoop.hbase.security.AccessDeniedException;
056import org.apache.hadoop.hbase.security.User;
057import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode;
058import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode;
059import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode;
060import org.apache.hadoop.hbase.security.visibility.expression.Operator;
061import org.apache.hadoop.hbase.util.ByteRange;
062import org.apache.hadoop.hbase.util.Bytes;
063import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
064import org.apache.hadoop.util.ReflectionUtils;
065import org.apache.yetus.audience.InterfaceAudience;
066import org.slf4j.Logger;
067import org.slf4j.LoggerFactory;
068
069/**
070 * Utility method to support visibility
071 */
072@InterfaceAudience.Private
073public class VisibilityUtils {
074
075  private static final Logger LOG = LoggerFactory.getLogger(VisibilityUtils.class);
076
077  public static final String VISIBILITY_LABEL_GENERATOR_CLASS =
078      "hbase.regionserver.scan.visibility.label.generator.class";
079  public static final String SYSTEM_LABEL = "system";
080  public static final Tag SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG = new ArrayBackedTag(
081      TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE,
082      VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG_VAL);
083  private static final String COMMA = ",";
084
085  private static final ExpressionParser EXP_PARSER = new ExpressionParser();
086  private static final ExpressionExpander EXP_EXPANDER = new ExpressionExpander();
087
088  /**
089   * Creates the labels data to be written to zookeeper.
090   * @param existingLabels
091   * @return Bytes form of labels and their ordinal details to be written to zookeeper.
092   */
093  public static byte[] getDataToWriteToZooKeeper(Map<String, Integer> existingLabels) {
094    VisibilityLabelsRequest.Builder visReqBuilder = VisibilityLabelsRequest.newBuilder();
095    for (Entry<String, Integer> entry : existingLabels.entrySet()) {
096      VisibilityLabel.Builder visLabBuilder = VisibilityLabel.newBuilder();
097      visLabBuilder.setLabel(ByteString.copyFrom(Bytes.toBytes(entry.getKey())));
098      visLabBuilder.setOrdinal(entry.getValue());
099      visReqBuilder.addVisLabel(visLabBuilder.build());
100    }
101    return ProtobufUtil.prependPBMagic(visReqBuilder.build().toByteArray());
102  }
103
104  /**
105   * Creates the user auth data to be written to zookeeper.
106   * @param userAuths
107   * @return Bytes form of user auths details to be written to zookeeper.
108   */
109  public static byte[] getUserAuthsDataToWriteToZooKeeper(Map<String, List<Integer>> userAuths) {
110    MultiUserAuthorizations.Builder builder = MultiUserAuthorizations.newBuilder();
111    for (Entry<String, List<Integer>> entry : userAuths.entrySet()) {
112      UserAuthorizations.Builder userAuthsBuilder = UserAuthorizations.newBuilder();
113      userAuthsBuilder.setUser(ByteString.copyFrom(Bytes.toBytes(entry.getKey())));
114      for (Integer label : entry.getValue()) {
115        userAuthsBuilder.addAuth(label);
116      }
117      builder.addUserAuths(userAuthsBuilder.build());
118    }
119    return ProtobufUtil.prependPBMagic(builder.build().toByteArray());
120  }
121
122  /**
123   * Reads back from the zookeeper. The data read here is of the form written by
124   * writeToZooKeeper(Map&lt;byte[], Integer&gt; entries).
125   * 
126   * @param data
127   * @return Labels and their ordinal details
128   * @throws DeserializationException
129   */
130  public static List<VisibilityLabel> readLabelsFromZKData(byte[] data)
131      throws DeserializationException {
132    if (ProtobufUtil.isPBMagicPrefix(data)) {
133      int pblen = ProtobufUtil.lengthOfPBMagic();
134      try {
135        VisibilityLabelsRequest.Builder builder = VisibilityLabelsRequest.newBuilder();
136        ProtobufUtil.mergeFrom(builder, data, pblen, data.length - pblen);
137        return builder.getVisLabelList();
138      } catch (IOException e) {
139        throw new DeserializationException(e);
140      }
141    }
142    return null;
143  }
144
145  /**
146   * Reads back User auth data written to zookeeper.
147   * @param data
148   * @return User auth details
149   * @throws DeserializationException
150   */
151  public static MultiUserAuthorizations readUserAuthsFromZKData(byte[] data) 
152      throws DeserializationException {
153    if (ProtobufUtil.isPBMagicPrefix(data)) {
154      int pblen = ProtobufUtil.lengthOfPBMagic();
155      try {
156        MultiUserAuthorizations.Builder builder = MultiUserAuthorizations.newBuilder();
157        ProtobufUtil.mergeFrom(builder, data, pblen, data.length - pblen);
158        return builder.build();
159      } catch (IOException e) {
160        throw new DeserializationException(e);
161      }
162    }
163    return null;
164  }
165
166  /**
167   * @param conf The configuration to use
168   * @return Stack of ScanLabelGenerator instances. ScanLabelGenerator classes can be specified in
169   *         Configuration as comma separated list using key
170   *         "hbase.regionserver.scan.visibility.label.generator.class"
171   * @throws IllegalArgumentException
172   *           when any of the specified ScanLabelGenerator class can not be loaded.
173   */
174  public static List<ScanLabelGenerator> getScanLabelGenerators(Configuration conf) {
175    // There can be n SLG specified as comma separated in conf
176    String slgClassesCommaSeparated = conf.get(VISIBILITY_LABEL_GENERATOR_CLASS);
177    // We have only System level SLGs now. The order of execution will be same as the order in the
178    // comma separated config value
179    List<ScanLabelGenerator> slgs = new ArrayList<>();
180    if (StringUtils.isNotEmpty(slgClassesCommaSeparated)) {
181      String[] slgClasses = slgClassesCommaSeparated.split(COMMA);
182      for (String slgClass : slgClasses) {
183        Class<? extends ScanLabelGenerator> slgKlass;
184        try {
185          slgKlass = (Class<? extends ScanLabelGenerator>) conf.getClassByName(slgClass.trim());
186          slgs.add(ReflectionUtils.newInstance(slgKlass, conf));
187        } catch (ClassNotFoundException e) {
188          throw new IllegalArgumentException("Unable to find " + slgClass, e);
189        }
190      }
191    }
192    // If no SLG is specified in conf, by default we'll add two SLGs
193    // 1. FeedUserAuthScanLabelGenerator
194    // 2. DefinedSetFilterScanLabelGenerator
195    // This stacking will achieve the following default behavior:
196    // 1. If there is no Auths in the scan, we will obtain the global defined set for the user
197    //    from the labels table.
198    // 2. If there is Auths in the scan, we will examine the passed in Auths and filter out the
199    //    labels that the user is not entitled to. Then use the resulting label set.
200    if (slgs.isEmpty()) {
201      slgs.add(ReflectionUtils.newInstance(FeedUserAuthScanLabelGenerator.class, conf));
202      slgs.add(ReflectionUtils.newInstance(DefinedSetFilterScanLabelGenerator.class, conf));
203    }
204    return slgs;
205  }
206
207  /**
208   * Extract the visibility tags of the given Cell into the given List
209   * @param cell - the cell
210   * @param tags - the array that will be populated if visibility tags are present
211   * @return The visibility tags serialization format
212   */
213  public static Byte extractVisibilityTags(Cell cell, List<Tag> tags) {
214    Byte serializationFormat = null;
215    Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
216    while (tagsIterator.hasNext()) {
217      Tag tag = tagsIterator.next();
218      if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
219        serializationFormat = Tag.getValueAsByte(tag);
220      } else if (tag.getType() == VISIBILITY_TAG_TYPE) {
221        tags.add(tag);
222      }
223    }
224    return serializationFormat;
225  }
226
227  /**
228   * Extracts and partitions the visibility tags and nonVisibility Tags
229   *
230   * @param cell - the cell for which we would extract and partition the
231   * visibility and non visibility tags
232   * @param visTags
233   *          - all the visibilty tags of type TagType.VISIBILITY_TAG_TYPE would
234   *          be added to this list
235   * @param nonVisTags - all the non visibility tags would be added to this list
236   * @return - the serailization format of the tag. Can be null if no tags are found or
237   * if there is no visibility tag found
238   */
239  public static Byte extractAndPartitionTags(Cell cell, List<Tag> visTags,
240      List<Tag> nonVisTags) {
241    Byte serializationFormat = null;
242    Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
243    while (tagsIterator.hasNext()) {
244      Tag tag = tagsIterator.next();
245      if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
246        serializationFormat = Tag.getValueAsByte(tag);
247      } else if (tag.getType() == VISIBILITY_TAG_TYPE) {
248        visTags.add(tag);
249      } else {
250        // ignore string encoded visibility expressions, will be added in replication handling
251        nonVisTags.add(tag);
252      }
253    }
254    return serializationFormat;
255  }
256
257  public static boolean isVisibilityTagsPresent(Cell cell) {
258    Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
259    while (tagsIterator.hasNext()) {
260      Tag tag = tagsIterator.next();
261      if (tag.getType() == VISIBILITY_TAG_TYPE) {
262        return true;
263      }
264    }
265    return false;
266  }
267
268  public static Filter createVisibilityLabelFilter(Region region, Authorizations authorizations)
269      throws IOException {
270    Map<ByteRange, Integer> cfVsMaxVersions = new HashMap<>();
271    for (ColumnFamilyDescriptor hcd : region.getTableDescriptor().getColumnFamilies()) {
272      cfVsMaxVersions.put(new SimpleMutableByteRange(hcd.getName()), hcd.getMaxVersions());
273    }
274    VisibilityLabelService vls = VisibilityLabelServiceManager.getInstance()
275        .getVisibilityLabelService();
276    Filter visibilityLabelFilter = new VisibilityLabelFilter(
277        vls.getVisibilityExpEvaluator(authorizations), cfVsMaxVersions);
278    return visibilityLabelFilter;
279  }
280
281  /**
282   * @return User who called RPC method. For non-RPC handling, falls back to system user
283   * @throws IOException When there is IOE in getting the system user (During non-RPC handling).
284   */
285  public static User getActiveUser() throws IOException {
286    Optional<User> optionalUser = RpcServer.getRequestUser();
287    User user;
288    if (optionalUser.isPresent()) {
289      user = optionalUser.get();
290    } else {
291      user = User.getCurrent();
292    }
293    if (LOG.isTraceEnabled()) {
294      LOG.trace("Current active user name is " + user.getShortName());
295    }
296    return user;
297  }
298
299  public static List<Tag> createVisibilityExpTags(String visExpression,
300      boolean withSerializationFormat, boolean checkAuths, Set<Integer> auths,
301      VisibilityLabelOrdinalProvider ordinalProvider) throws IOException {
302    ExpressionNode node = null;
303    try {
304      node = EXP_PARSER.parse(visExpression);
305    } catch (ParseException e) {
306      throw new IOException(e);
307    }
308    node = EXP_EXPANDER.expand(node);
309    List<Tag> tags = new ArrayList<>();
310    ByteArrayOutputStream baos = new ByteArrayOutputStream();
311    DataOutputStream dos = new DataOutputStream(baos);
312    List<Integer> labelOrdinals = new ArrayList<>();
313    // We will be adding this tag before the visibility tags and the presence of this
314    // tag indicates we are supporting deletes with cell visibility
315    if (withSerializationFormat) {
316      tags.add(VisibilityUtils.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG);
317    }
318    if (node.isSingleNode()) {
319      getLabelOrdinals(node, labelOrdinals, auths, checkAuths, ordinalProvider);
320      writeLabelOrdinalsToStream(labelOrdinals, dos);
321      tags.add(new ArrayBackedTag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
322      baos.reset();
323    } else {
324      NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
325      if (nlNode.getOperator() == Operator.OR) {
326        for (ExpressionNode child : nlNode.getChildExps()) {
327          getLabelOrdinals(child, labelOrdinals, auths, checkAuths, ordinalProvider);
328          writeLabelOrdinalsToStream(labelOrdinals, dos);
329          tags.add(new ArrayBackedTag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
330          baos.reset();
331          labelOrdinals.clear();
332        }
333      } else {
334        getLabelOrdinals(nlNode, labelOrdinals, auths, checkAuths, ordinalProvider);
335        writeLabelOrdinalsToStream(labelOrdinals, dos);
336        tags.add(new ArrayBackedTag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
337        baos.reset();
338      }
339    }
340    return tags;
341  }
342
343  private static void getLabelOrdinals(ExpressionNode node, List<Integer> labelOrdinals,
344      Set<Integer> auths, boolean checkAuths, VisibilityLabelOrdinalProvider ordinalProvider)
345      throws IOException, InvalidLabelException {
346    if (node.isSingleNode()) {
347      String identifier = null;
348      int labelOrdinal = 0;
349      if (node instanceof LeafExpressionNode) {
350        identifier = ((LeafExpressionNode) node).getIdentifier();
351        if (LOG.isTraceEnabled()) {
352          LOG.trace("The identifier is " + identifier);
353        }
354        labelOrdinal = ordinalProvider.getLabelOrdinal(identifier);
355        checkAuths(auths, labelOrdinal, identifier, checkAuths);
356      } else {
357        // This is a NOT node.
358        LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node)
359            .getChildExps().get(0);
360        identifier = lNode.getIdentifier();
361        labelOrdinal = ordinalProvider.getLabelOrdinal(identifier);
362        checkAuths(auths, labelOrdinal, identifier, checkAuths);
363        labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal.
364      }
365      if (labelOrdinal == 0) {
366        throw new InvalidLabelException("Invalid visibility label " + identifier);
367      }
368      labelOrdinals.add(labelOrdinal);
369    } else {
370      List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps();
371      for (ExpressionNode child : childExps) {
372        getLabelOrdinals(child, labelOrdinals, auths, checkAuths, ordinalProvider);
373      }
374    }
375  }
376
377  /**
378   * This will sort the passed labels in ascending oder and then will write one after the other to
379   * the passed stream.
380   * @param labelOrdinals
381   *          Unsorted label ordinals
382   * @param dos
383   *          Stream where to write the labels.
384   * @throws IOException
385   *           When IOE during writes to Stream.
386   */
387  private static void writeLabelOrdinalsToStream(List<Integer> labelOrdinals, DataOutputStream dos)
388      throws IOException {
389    Collections.sort(labelOrdinals);
390    for (Integer labelOrdinal : labelOrdinals) {
391      StreamUtils.writeRawVInt32(dos, labelOrdinal);
392    }
393  }
394
395  private static void checkAuths(Set<Integer> auths, int labelOrdinal, String identifier,
396      boolean checkAuths) throws IOException {
397    if (checkAuths) {
398      if (auths == null || (!auths.contains(labelOrdinal))) {
399        throw new AccessDeniedException("Visibility label " + identifier
400            + " not authorized for the user " + VisibilityUtils.getActiveUser().getShortName());
401      }
402    }
403  }
404}