001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas.policies;
019
020import static org.mockito.Mockito.mock;
021import static org.mockito.Mockito.when;
022
023import java.util.ArrayList;
024import java.util.List;
025import org.apache.hadoop.fs.FileStatus;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
031import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
032import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
033import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement;
034import org.apache.hadoop.hbase.regionserver.RegionServerServices;
035import org.apache.hadoop.hbase.testclassification.SmallTests;
036import org.junit.Before;
037import org.junit.ClassRule;
038import org.junit.Test;
039import org.junit.experimental.categories.Category;
040
041@Category(SmallTests.class)
042public class TestBulkLoadCheckingViolationPolicyEnforcement {
043
044  @ClassRule
045  public static final HBaseClassTestRule CLASS_RULE =
046    HBaseClassTestRule.forClass(TestBulkLoadCheckingViolationPolicyEnforcement.class);
047
048  FileSystem fs;
049  RegionServerServices rss;
050  TableName tableName;
051  SpaceViolationPolicyEnforcement policy;
052
053  @Before
054  public void setup() {
055    fs = mock(FileSystem.class);
056    rss = mock(RegionServerServices.class);
057    tableName = TableName.valueOf("foo");
058    policy = new DefaultViolationPolicyEnforcement();
059  }
060
061  @Test
062  public void testFilesUnderLimit() throws Exception {
063    final List<String> paths = new ArrayList<>();
064    final List<FileStatus> statuses = new ArrayList<>();
065    final long length = 100L * 1024L;
066    for (int i = 0; i < 5; i++) {
067      String path = "/" + i;
068      FileStatus status = mock(FileStatus.class);
069      when(fs.getFileStatus(new Path(path))).thenReturn(status);
070      when(status.getLen()).thenReturn(length);
071      when(status.isFile()).thenReturn(true);
072      paths.add(path);
073      statuses.add(status);
074    }
075
076    // Quota is not in violation now
077    SpaceQuotaSnapshot snapshot =
078      new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6);
079
080    policy.initialize(rss, tableName, snapshot);
081
082    policy.computeBulkLoadSize(fs, paths);
083  }
084
085  @Test(expected = IllegalArgumentException.class)
086  public void testFileIsNotAFile() throws Exception {
087    final List<String> paths = new ArrayList<>();
088    String path = "/1";
089    FileStatus status = mock(FileStatus.class);
090    when(fs.getFileStatus(new Path(path))).thenReturn(status);
091    when(status.getLen()).thenReturn(1000L);
092    when(status.isFile()).thenReturn(false);
093    paths.add(path);
094
095    // Quota is not in violation now
096    SpaceQuotaSnapshot snapshot =
097      new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE);
098
099    policy.initialize(rss, tableName, snapshot);
100
101    // If the file to bulk load isn't a file, this should throw an exception
102    policy.computeBulkLoadSize(fs, paths);
103  }
104
105  @Test(expected = SpaceLimitingException.class)
106  public void testOneFileInBatchOverLimit() throws Exception {
107    final List<String> paths = new ArrayList<>();
108    final List<FileStatus> statuses = new ArrayList<>();
109    final long length = 1000L * 1024L;
110    for (int i = 0; i < 5; i++) {
111      String path = "/" + i;
112      FileStatus status = mock(FileStatus.class);
113      when(fs.getFileStatus(new Path(path))).thenReturn(status);
114      when(status.getLen()).thenReturn(length);
115      when(status.isFile()).thenReturn(true);
116      paths.add(path);
117      statuses.add(status);
118    }
119
120    // Quota is not in violation now
121    SpaceQuotaSnapshot snapshot =
122      new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L);
123
124    policy.initialize(rss, tableName, snapshot);
125
126    policy.computeBulkLoadSize(fs, paths);
127  }
128
129  @Test(expected = SpaceLimitingException.class)
130  public void testSumOfFilesOverLimit() throws Exception {
131    final List<String> paths = new ArrayList<>();
132    final List<FileStatus> statuses = new ArrayList<>();
133    final long length = 1024L;
134    for (int i = 0; i < 5; i++) {
135      String path = "/" + i;
136      FileStatus status = mock(FileStatus.class);
137      when(fs.getFileStatus(new Path(path))).thenReturn(status);
138      when(status.getLen()).thenReturn(length);
139      when(status.isFile()).thenReturn(true);
140      paths.add(path);
141      statuses.add(status);
142    }
143
144    // Quota is not in violation now, but 5*1024 files would push us to violation
145    SpaceQuotaSnapshot snapshot =
146      new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 5000L);
147
148    policy.initialize(rss, tableName, snapshot);
149
150    policy.computeBulkLoadSize(fs, paths);
151  }
152}