View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.security.visibility;
19  
20  import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE;
21  
22  import java.io.ByteArrayOutputStream;
23  import java.io.DataOutputStream;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.HashMap;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Map.Entry;
32  import java.util.Set;
33  
34  import org.apache.commons.lang.StringUtils;
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.hbase.Cell;
40  import org.apache.hadoop.hbase.CellUtil;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.Tag;
43  import org.apache.hadoop.hbase.TagType;
44  import org.apache.hadoop.hbase.exceptions.DeserializationException;
45  import org.apache.hadoop.hbase.filter.Filter;
46  import org.apache.hadoop.hbase.io.util.StreamUtils;
47  import org.apache.hadoop.hbase.ipc.RequestContext;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations;
50  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations;
51  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
52  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
53  import org.apache.hadoop.hbase.regionserver.HRegion;
54  import org.apache.hadoop.hbase.security.AccessDeniedException;
55  import org.apache.hadoop.hbase.security.User;
56  import org.apache.hadoop.hbase.security.access.AccessControlLists;
57  import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode;
58  import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode;
59  import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode;
60  import org.apache.hadoop.hbase.security.visibility.expression.Operator;
61  import org.apache.hadoop.hbase.util.ByteRange;
62  import org.apache.hadoop.hbase.util.ByteStringer;
63  import org.apache.hadoop.hbase.util.Bytes;
64  import org.apache.hadoop.hbase.util.SimpleByteRange;
65  import org.apache.hadoop.hbase.util.Pair;
66  import org.apache.hadoop.util.ReflectionUtils;
67  
68  import com.google.protobuf.InvalidProtocolBufferException;
69  
70  /**
71   * Utility method to support visibility
72   */
73  @InterfaceAudience.Private
74  public class VisibilityUtils {
75  
76    private static final Log LOG = LogFactory.getLog(VisibilityUtils.class);
77  
78    public static final String VISIBILITY_LABEL_GENERATOR_CLASS =
79        "hbase.regionserver.scan.visibility.label.generator.class";
80    public static final String SYSTEM_LABEL = "system";
81    public static final Tag SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG = new Tag(
82        TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE,
83        VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG_VAL);
84    private static final String COMMA = ",";
85  
86    private static final ExpressionParser EXP_PARSER = new ExpressionParser();
87    private static final ExpressionExpander EXP_EXPANDER = new ExpressionExpander();
88  
89    /**
90     * Creates the labels data to be written to zookeeper.
91     * @param existingLabels
92     * @return Bytes form of labels and their ordinal details to be written to zookeeper.
93     */
94    public static byte[] getDataToWriteToZooKeeper(Map<String, Integer> existingLabels) {
95      VisibilityLabelsRequest.Builder visReqBuilder = VisibilityLabelsRequest.newBuilder();
96      for (Entry<String, Integer> entry : existingLabels.entrySet()) {
97        VisibilityLabel.Builder visLabBuilder = VisibilityLabel.newBuilder();
98        visLabBuilder.setLabel(ByteStringer.wrap(Bytes.toBytes(entry.getKey())));
99        visLabBuilder.setOrdinal(entry.getValue());
100       visReqBuilder.addVisLabel(visLabBuilder.build());
101     }
102     return ProtobufUtil.prependPBMagic(visReqBuilder.build().toByteArray());
103   }
104 
105   /**
106    * Get the super users and groups defined in the configuration.
107    * The user running the hbase server is always included.
108    * @param conf
109    * @return Pair of super user list and super group list.
110    * @throws IOException
111    */
112   public static Pair<List<String>, List<String>> getSystemAndSuperUsers(Configuration conf)
113       throws IOException {
114     ArrayList<String> superUsers = new ArrayList<String>();
115     ArrayList<String> superGroups = new ArrayList<String>();
116     User user = User.getCurrent();
117     if (user == null) {
118       throw new IOException("Unable to obtain the current user, "
119           + "authorization checks for internal operations will not work correctly!");
120     }
121     if (LOG.isTraceEnabled()) {
122       LOG.trace("Current user name is " + user.getShortName());
123     }
124     String currentUser = user.getShortName();
125     String[] superUserList = conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0]);
126     for (String name : superUserList) {
127       if (AccessControlLists.isGroupPrincipal(name)) {
128         superGroups.add(AccessControlLists.getGroupName(name));
129       } else {
130         superUsers.add(name);
131       }
132     }
133     superUsers.add(currentUser);
134     return new Pair<List<String>, List<String>>(superUsers, superGroups);
135   }
136 
137   /**
138    * Creates the user auth data to be written to zookeeper.
139    * @param userAuths
140    * @return Bytes form of user auths details to be written to zookeeper.
141    */
142   public static byte[] getUserAuthsDataToWriteToZooKeeper(Map<String, List<Integer>> userAuths) {
143     MultiUserAuthorizations.Builder builder = MultiUserAuthorizations.newBuilder();
144     for (Entry<String, List<Integer>> entry : userAuths.entrySet()) {
145       UserAuthorizations.Builder userAuthsBuilder = UserAuthorizations.newBuilder();
146       userAuthsBuilder.setUser(ByteStringer.wrap(Bytes.toBytes(entry.getKey())));
147       for (Integer label : entry.getValue()) {
148         userAuthsBuilder.addAuth(label);
149       }
150       builder.addUserAuths(userAuthsBuilder.build());
151     }
152     return ProtobufUtil.prependPBMagic(builder.build().toByteArray());
153   }
154 
155   /**
156    * Reads back from the zookeeper. The data read here is of the form written by
157    * writeToZooKeeper(Map<byte[], Integer> entries).
158    * 
159    * @param data
160    * @return Labels and their ordinal details
161    * @throws DeserializationException
162    */
163   public static List<VisibilityLabel> readLabelsFromZKData(byte[] data)
164       throws DeserializationException {
165     if (ProtobufUtil.isPBMagicPrefix(data)) {
166       int pblen = ProtobufUtil.lengthOfPBMagic();
167       try {
168         VisibilityLabelsRequest request = VisibilityLabelsRequest.newBuilder()
169             .mergeFrom(data, pblen, data.length - pblen).build();
170         return request.getVisLabelList();
171       } catch (InvalidProtocolBufferException e) {
172         throw new DeserializationException(e);
173       }
174     }
175     return null;
176   }
177 
178   /**
179    * Reads back User auth data written to zookeeper.
180    * @param data
181    * @return User auth details
182    * @throws DeserializationException
183    */
184   public static MultiUserAuthorizations readUserAuthsFromZKData(byte[] data) 
185       throws DeserializationException {
186     if (ProtobufUtil.isPBMagicPrefix(data)) {
187       int pblen = ProtobufUtil.lengthOfPBMagic();
188       try {
189         MultiUserAuthorizations multiUserAuths = MultiUserAuthorizations.newBuilder()
190             .mergeFrom(data, pblen, data.length - pblen).build();
191         return multiUserAuths;
192       } catch (InvalidProtocolBufferException e) {
193         throw new DeserializationException(e);
194       }
195     }
196     return null;
197   }
198 
199   /**
200    * @param conf The configuration to use
201    * @return Stack of ScanLabelGenerator instances. ScanLabelGenerator classes can be specified in
202    *         Configuration as comma separated list using key
203    *         "hbase.regionserver.scan.visibility.label.generator.class"
204    * @throws IllegalArgumentException
205    *           when any of the specified ScanLabelGenerator class can not be loaded.
206    */
207   public static List<ScanLabelGenerator> getScanLabelGenerators(Configuration conf) {
208     // There can be n SLG specified as comma separated in conf
209     String slgClassesCommaSeparated = conf.get(VISIBILITY_LABEL_GENERATOR_CLASS);
210     // We have only System level SLGs now. The order of execution will be same as the order in the
211     // comma separated config value
212     List<ScanLabelGenerator> slgs = new ArrayList<ScanLabelGenerator>();
213     if (StringUtils.isNotEmpty(slgClassesCommaSeparated)) {
214       String[] slgClasses = slgClassesCommaSeparated.split(COMMA);
215       for (String slgClass : slgClasses) {
216         Class<? extends ScanLabelGenerator> slgKlass;
217         try {
218           slgKlass = (Class<? extends ScanLabelGenerator>) conf.getClassByName(slgClass.trim());
219           slgs.add(ReflectionUtils.newInstance(slgKlass, conf));
220         } catch (ClassNotFoundException e) {
221           throw new IllegalArgumentException("Unable to find " + slgClass, e);
222         }
223       }
224     }
225     // If no SLG is specified in conf, by default we'll add two SLGs
226     // 1. FeedUserAuthScanLabelGenerator
227     // 2. DefinedSetFilterScanLabelGenerator
228     // This stacking will achieve the following default behavior:
229     // 1. If there is no Auths in the scan, we will obtain the global defined set for the user
230     //    from the labels table.
231     // 2. If there is Auths in the scan, we will examine the passed in Auths and filter out the
232     //    labels that the user is not entitled to. Then use the resulting label set.
233     if (slgs.isEmpty()) {
234       slgs.add(ReflectionUtils.newInstance(FeedUserAuthScanLabelGenerator.class, conf));
235       slgs.add(ReflectionUtils.newInstance(DefinedSetFilterScanLabelGenerator.class, conf));
236     }
237     return slgs;
238   }
239 
240   /**
241    * Extract the visibility tags of the given Cell into the given List
242    * @param cell - the cell
243    * @param tags - the array that will be populated if visibility tags are present
244    * @return The visibility tags serialization format
245    */
246   public static Byte extractVisibilityTags(Cell cell, List<Tag> tags) {
247     Byte serializationFormat = null;
248     if (cell.getTagsLengthUnsigned() > 0) {
249       Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
250           cell.getTagsLengthUnsigned());
251       while (tagsIterator.hasNext()) {
252         Tag tag = tagsIterator.next();
253         if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
254           serializationFormat = tag.getBuffer()[tag.getTagOffset()];
255         } else if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
256           tags.add(tag);
257         }
258       }
259     }
260     return serializationFormat;
261   }
262 
263   /**
264    * Extracts and partitions the visibility tags and nonVisibility Tags
265    *
266    * @param cell - the cell for which we would extract and partition the
267    * visibility and non visibility tags
268    * @param visTags
269    *          - all the visibilty tags of type TagType.VISIBILITY_TAG_TYPE would
270    *          be added to this list
271    * @param nonVisTags - all the non visibility tags would be added to this list
272    * @return - the serailization format of the tag. Can be null if no tags are found or
273    * if there is no visibility tag found
274    */
275   public static Byte extractAndPartitionTags(Cell cell, List<Tag> visTags,
276       List<Tag> nonVisTags) {
277     Byte serializationFormat = null;
278     if (cell.getTagsLength() > 0) {
279       Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
280           cell.getTagsLength());
281       while (tagsIterator.hasNext()) {
282         Tag tag = tagsIterator.next();
283         if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
284           serializationFormat = tag.getBuffer()[tag.getTagOffset()];
285         } else if (tag.getType() == VISIBILITY_TAG_TYPE) {
286           visTags.add(tag);
287         } else {
288           // ignore string encoded visibility expressions, will be added in replication handling
289           nonVisTags.add(tag);
290         }
291       }
292     }
293     return serializationFormat;
294   }
295 
296   public static boolean isVisibilityTagsPresent(Cell cell) {
297     if (cell.getTagsLengthUnsigned() == 0) {
298       return false;
299     }
300     Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
301         cell.getTagsLengthUnsigned());
302     while (tagsIterator.hasNext()) {
303       Tag tag = tagsIterator.next();
304       if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
305         return true;
306       }
307     }
308     return false;
309   }
310 
311   public static Filter createVisibilityLabelFilter(HRegion region, Authorizations authorizations)
312       throws IOException {
313     Map<ByteRange, Integer> cfVsMaxVersions = new HashMap<ByteRange, Integer>();
314     for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) {
315       cfVsMaxVersions.put(new SimpleByteRange(hcd.getName()), hcd.getMaxVersions());
316     }
317     VisibilityLabelService vls = VisibilityLabelServiceManager.getInstance()
318         .getVisibilityLabelService();
319     Filter visibilityLabelFilter = new VisibilityLabelFilter(
320         vls.getVisibilityExpEvaluator(authorizations), cfVsMaxVersions);
321     return visibilityLabelFilter;
322   }
323 
324   /**
325    * @return User who called RPC method. For non-RPC handling, falls back to system user
326    * @throws IOException When there is IOE in getting the system user (During non-RPC handling).
327    */
328   public static User getActiveUser() throws IOException {
329     User user = RequestContext.getRequestUser();
330     if (!RequestContext.isInRequestContext()) {
331       user = User.getCurrent();
332     }
333     if (LOG.isTraceEnabled()) {
334       LOG.trace("Current active user name is " + user.getShortName());
335     }
336     return user;
337   }
338 
339   public static List<Tag> createVisibilityExpTags(String visExpression,
340       boolean withSerializationFormat, boolean checkAuths, Set<Integer> auths,
341       VisibilityLabelOrdinalProvider ordinalProvider) throws IOException {
342     ExpressionNode node = null;
343     try {
344       node = EXP_PARSER.parse(visExpression);
345     } catch (ParseException e) {
346       throw new IOException(e);
347     }
348     node = EXP_EXPANDER.expand(node);
349     List<Tag> tags = new ArrayList<Tag>();
350     ByteArrayOutputStream baos = new ByteArrayOutputStream();
351     DataOutputStream dos = new DataOutputStream(baos);
352     List<Integer> labelOrdinals = new ArrayList<Integer>();
353     // We will be adding this tag before the visibility tags and the presence of this
354     // tag indicates we are supporting deletes with cell visibility
355     if (withSerializationFormat) {
356       tags.add(VisibilityUtils.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG);
357     }
358     if (node.isSingleNode()) {
359       getLabelOrdinals(node, labelOrdinals, auths, checkAuths, ordinalProvider);
360       writeLabelOrdinalsToStream(labelOrdinals, dos);
361       tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
362       baos.reset();
363     } else {
364       NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
365       if (nlNode.getOperator() == Operator.OR) {
366         for (ExpressionNode child : nlNode.getChildExps()) {
367           getLabelOrdinals(child, labelOrdinals, auths, checkAuths, ordinalProvider);
368           writeLabelOrdinalsToStream(labelOrdinals, dos);
369           tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
370           baos.reset();
371           labelOrdinals.clear();
372         }
373       } else {
374         getLabelOrdinals(nlNode, labelOrdinals, auths, checkAuths, ordinalProvider);
375         writeLabelOrdinalsToStream(labelOrdinals, dos);
376         tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
377         baos.reset();
378       }
379     }
380     return tags;
381   }
382 
383   private static void getLabelOrdinals(ExpressionNode node, List<Integer> labelOrdinals,
384       Set<Integer> auths, boolean checkAuths, VisibilityLabelOrdinalProvider ordinalProvider)
385       throws IOException, InvalidLabelException {
386     if (node.isSingleNode()) {
387       String identifier = null;
388       int labelOrdinal = 0;
389       if (node instanceof LeafExpressionNode) {
390         identifier = ((LeafExpressionNode) node).getIdentifier();
391         if (LOG.isTraceEnabled()) {
392           LOG.trace("The identifier is " + identifier);
393         }
394         labelOrdinal = ordinalProvider.getLabelOrdinal(identifier);
395         checkAuths(auths, labelOrdinal, identifier, checkAuths);
396       } else {
397         // This is a NOT node.
398         LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node)
399             .getChildExps().get(0);
400         identifier = lNode.getIdentifier();
401         labelOrdinal = ordinalProvider.getLabelOrdinal(identifier);
402         checkAuths(auths, labelOrdinal, identifier, checkAuths);
403         labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal.
404       }
405       if (labelOrdinal == 0) {
406         throw new InvalidLabelException("Invalid visibility label " + identifier);
407       }
408       labelOrdinals.add(labelOrdinal);
409     } else {
410       List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps();
411       for (ExpressionNode child : childExps) {
412         getLabelOrdinals(child, labelOrdinals, auths, checkAuths, ordinalProvider);
413       }
414     }
415   }
416 
417   /**
418    * This will sort the passed labels in ascending oder and then will write one after the other to
419    * the passed stream.
420    * 
421    * @param labelOrdinals
422    *          Unsorted label ordinals
423    * @param dos
424    *          Stream where to write the labels.
425    * @throws IOException
426    *           When IOE during writes to Stream.
427    */
428   private static void writeLabelOrdinalsToStream(List<Integer> labelOrdinals, DataOutputStream dos)
429       throws IOException {
430     Collections.sort(labelOrdinals);
431     for (Integer labelOrdinal : labelOrdinals) {
432       StreamUtils.writeRawVInt32(dos, labelOrdinal);
433     }
434   }
435 
436   private static void checkAuths(Set<Integer> auths, int labelOrdinal, String identifier,
437       boolean checkAuths) throws IOException {
438     if (checkAuths) {
439       if (auths == null || (!auths.contains(labelOrdinal))) {
440         throw new AccessDeniedException("Visibility label " + identifier
441             + " not authorized for the user " + VisibilityUtils.getActiveUser().getShortName());
442       }
443     }
444   }
445 }